diff options
Diffstat (limited to 'third_party/libwebrtc/modules/video_coding/timing')
28 files changed, 5074 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/timing/BUILD.gn b/third_party/libwebrtc/modules/video_coding/timing/BUILD.gn new file mode 100644 index 0000000000..38348e6967 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/BUILD.gn @@ -0,0 +1,153 @@ +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_library("codec_timer") { + sources = [ + "codec_timer.cc", + "codec_timer.h", + ] + deps = [ "../../../rtc_base:rtc_numerics" ] +} + +rtc_library("inter_frame_delay") { + sources = [ + "inter_frame_delay.cc", + "inter_frame_delay.h", + ] + deps = [ + "../..:module_api_public", + "../../../api/units:frequency", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:rtc_numerics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("frame_delay_variation_kalman_filter") { + sources = [ + "frame_delay_variation_kalman_filter.cc", + "frame_delay_variation_kalman_filter.h", + ] + deps = [ + "../../../api/units:data_size", + "../../../api/units:time_delta", + ] + visibility = [ + ":jitter_estimator", + ":timing_unittests", + ] +} + +rtc_library("jitter_estimator") { + sources = [ + "jitter_estimator.cc", + "jitter_estimator.h", + ] + deps = [ + ":frame_delay_variation_kalman_filter", + ":rtt_filter", + "../../../api:field_trials_view", + "../../../api/units:data_size", + "../../../api/units:frequency", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:checks", + "../../../rtc_base:logging", + "../../../rtc_base:rolling_accumulator", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base:safe_conversions", + "../../../rtc_base/experiments:field_trial_parser", + "../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("rtt_filter") { + sources = [ + "rtt_filter.cc", + "rtt_filter.h", + ] + deps = [ "../../../api/units:time_delta" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/container:inlined_vector", + ] +} + +rtc_library("timestamp_extrapolator") { + sources = [ + "timestamp_extrapolator.cc", + "timestamp_extrapolator.h", + ] + deps = [ + "../../../api/units:timestamp", + "../../../modules:module_api_public", + "../../../rtc_base:rtc_numerics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("timing_module") { + sources = [ + "timing.cc", + "timing.h", + ] + deps = [ + ":codec_timer", + ":timestamp_extrapolator", + "../../../api:field_trials_view", + "../../../api/units:time_delta", + "../../../api/video:video_frame", + "../../../api/video:video_rtp_headers", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base/experiments:field_trial_parser", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("timing_unittests") { + testonly = true + sources = [ + "frame_delay_variation_kalman_filter_unittest.cc", + "inter_frame_delay_unittest.cc", + "jitter_estimator_unittest.cc", + "rtt_filter_unittest.cc", + "timestamp_extrapolator_unittest.cc", + "timing_unittest.cc", + ] + deps = [ + ":frame_delay_variation_kalman_filter", + ":inter_frame_delay", + ":jitter_estimator", + ":rtt_filter", + ":timestamp_extrapolator", + ":timing_module", + "../../../api:array_view", + "../../../api:field_trials", + "../../../api/units:data_size", + "../../../api/units:frequency", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:histogram_percentile_counter", + "../../../rtc_base:timeutils", + "../../../system_wrappers:system_wrappers", + "../../../test:scoped_key_value_config", + "../../../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} diff --git a/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc new file mode 100644 index 0000000000..f57d42d40a --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/codec_timer.h" + +#include <cstdint> + +namespace webrtc { + +namespace { + +// The first kIgnoredSampleCount samples will be ignored. +const int kIgnoredSampleCount = 5; +// Return the `kPercentile` value in RequiredDecodeTimeMs(). +const float kPercentile = 0.95f; +// The window size in ms. +const int64_t kTimeLimitMs = 10000; + +} // anonymous namespace + +CodecTimer::CodecTimer() : ignored_sample_count_(0), filter_(kPercentile) {} +CodecTimer::~CodecTimer() = default; + +void CodecTimer::AddTiming(int64_t decode_time_ms, int64_t now_ms) { + // Ignore the first `kIgnoredSampleCount` samples. + if (ignored_sample_count_ < kIgnoredSampleCount) { + ++ignored_sample_count_; + return; + } + + // Insert new decode time value. + filter_.Insert(decode_time_ms); + history_.emplace(decode_time_ms, now_ms); + + // Pop old decode time values. + while (!history_.empty() && + now_ms - history_.front().sample_time_ms > kTimeLimitMs) { + filter_.Erase(history_.front().decode_time_ms); + history_.pop(); + } +} + +// Get the 95th percentile observed decode time within a time window. +int64_t CodecTimer::RequiredDecodeTimeMs() const { + return filter_.GetPercentileValue(); +} + +CodecTimer::Sample::Sample(int64_t decode_time_ms, int64_t sample_time_ms) + : decode_time_ms(decode_time_ms), sample_time_ms(sample_time_ms) {} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/codec_timer.h b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.h new file mode 100644 index 0000000000..9f12d82e98 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_CODEC_TIMER_H_ +#define MODULES_VIDEO_CODING_TIMING_CODEC_TIMER_H_ + +#include <queue> + +#include "rtc_base/numerics/percentile_filter.h" + +namespace webrtc { + +class CodecTimer { + public: + CodecTimer(); + ~CodecTimer(); + + // Add a new decode time to the filter. + void AddTiming(int64_t new_decode_time_ms, int64_t now_ms); + + // Get the required decode time in ms. It is the 95th percentile observed + // decode time within a time window. + int64_t RequiredDecodeTimeMs() const; + + private: + struct Sample { + Sample(int64_t decode_time_ms, int64_t sample_time_ms); + int64_t decode_time_ms; + int64_t sample_time_ms; + }; + + // The number of samples ignored so far. + int ignored_sample_count_; + // Queue with history of latest decode time values. + std::queue<Sample> history_; + // `filter_` contains the same values as `history_`, but in a data structure + // that allows efficient retrieval of the percentile value. + PercentileFilter<int64_t> filter_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_CODEC_TIMER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build new file mode 100644 index 0000000000..59498feb91 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build @@ -0,0 +1,221 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("codec_timer_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.cc b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.cc new file mode 100644 index 0000000000..ec6aa3445a --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/frame_delay_variation_kalman_filter.h" + +#include "api/units/data_size.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +namespace { +// TODO(brandtr): The value below corresponds to 8 Gbps. Is that reasonable? +constexpr double kMaxBandwidth = 0.000001; // Unit: [1 / bytes per ms]. +} // namespace + +FrameDelayVariationKalmanFilter::FrameDelayVariationKalmanFilter() { + // TODO(brandtr): Is there a factor 1000 missing here? + estimate_[0] = 1 / (512e3 / 8); // Unit: [1 / bytes per ms] + estimate_[1] = 0; // Unit: [ms] + + // Initial estimate covariance. + estimate_cov_[0][0] = 1e-4; // Unit: [(1 / bytes per ms)^2] + estimate_cov_[1][1] = 1e2; // Unit: [ms^2] + estimate_cov_[0][1] = estimate_cov_[1][0] = 0; + + // Process noise covariance. + process_noise_cov_diag_[0] = 2.5e-10; // Unit: [(1 / bytes per ms)^2] + process_noise_cov_diag_[1] = 1e-10; // Unit: [ms^2] +} + +void FrameDelayVariationKalmanFilter::PredictAndUpdate( + double frame_delay_variation_ms, + double frame_size_variation_bytes, + double max_frame_size_bytes, + double var_noise) { + // Sanity checks. + if (max_frame_size_bytes < 1) { + return; + } + if (var_noise <= 0.0) { + return; + } + + // This member function follows the data flow in + // https://en.wikipedia.org/wiki/Kalman_filter#Details. + + // 1) Estimate prediction: `x = F*x`. + // For this model, there is no need to explicitly predict the estimate, since + // the state transition matrix is the identity. + + // 2) Estimate covariance prediction: `P = F*P*F' + Q`. + // Again, since the state transition matrix is the identity, this update + // is performed by simply adding the process noise covariance. + estimate_cov_[0][0] += process_noise_cov_diag_[0]; + estimate_cov_[1][1] += process_noise_cov_diag_[1]; + + // 3) Innovation: `y = z - H*x`. + // This is the part of the measurement that cannot be explained by the current + // estimate. + double innovation = + frame_delay_variation_ms - + GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes); + + // 4) Innovation variance: `s = H*P*H' + r`. + double estim_cov_times_obs[2]; + estim_cov_times_obs[0] = + estimate_cov_[0][0] * frame_size_variation_bytes + estimate_cov_[0][1]; + estim_cov_times_obs[1] = + estimate_cov_[1][0] * frame_size_variation_bytes + estimate_cov_[1][1]; + double observation_noise_stddev = + (300.0 * exp(-fabs(frame_size_variation_bytes) / + (1e0 * max_frame_size_bytes)) + + 1) * + sqrt(var_noise); + if (observation_noise_stddev < 1.0) { + observation_noise_stddev = 1.0; + } + // TODO(brandtr): Shouldn't we add observation_noise_stddev^2 here? Otherwise, + // the dimensional analysis fails. + double innovation_var = frame_size_variation_bytes * estim_cov_times_obs[0] + + estim_cov_times_obs[1] + observation_noise_stddev; + if ((innovation_var < 1e-9 && innovation_var >= 0) || + (innovation_var > -1e-9 && innovation_var <= 0)) { + RTC_DCHECK_NOTREACHED(); + return; + } + + // 5) Optimal Kalman gain: `K = P*H'/s`. + // How much to trust the model vs. how much to trust the measurement. + double kalman_gain[2]; + kalman_gain[0] = estim_cov_times_obs[0] / innovation_var; + kalman_gain[1] = estim_cov_times_obs[1] / innovation_var; + + // 6) Estimate update: `x = x + K*y`. + // Optimally weight the new information in the innovation and add it to the + // old estimate. + estimate_[0] += kalman_gain[0] * innovation; + estimate_[1] += kalman_gain[1] * innovation; + + // (This clamping is not part of the linear Kalman filter.) + if (estimate_[0] < kMaxBandwidth) { + estimate_[0] = kMaxBandwidth; + } + + // 7) Estimate covariance update: `P = (I - K*H)*P` + double t00 = estimate_cov_[0][0]; + double t01 = estimate_cov_[0][1]; + estimate_cov_[0][0] = + (1 - kalman_gain[0] * frame_size_variation_bytes) * t00 - + kalman_gain[0] * estimate_cov_[1][0]; + estimate_cov_[0][1] = + (1 - kalman_gain[0] * frame_size_variation_bytes) * t01 - + kalman_gain[0] * estimate_cov_[1][1]; + estimate_cov_[1][0] = estimate_cov_[1][0] * (1 - kalman_gain[1]) - + kalman_gain[1] * frame_size_variation_bytes * t00; + estimate_cov_[1][1] = estimate_cov_[1][1] * (1 - kalman_gain[1]) - + kalman_gain[1] * frame_size_variation_bytes * t01; + + // Covariance matrix, must be positive semi-definite. + RTC_DCHECK(estimate_cov_[0][0] + estimate_cov_[1][1] >= 0 && + estimate_cov_[0][0] * estimate_cov_[1][1] - + estimate_cov_[0][1] * estimate_cov_[1][0] >= + 0 && + estimate_cov_[0][0] >= 0); +} + +double FrameDelayVariationKalmanFilter::GetFrameDelayVariationEstimateSizeBased( + double frame_size_variation_bytes) const { + // Unit: [1 / bytes per millisecond] * [bytes] = [milliseconds]. + return estimate_[0] * frame_size_variation_bytes; +} + +double FrameDelayVariationKalmanFilter::GetFrameDelayVariationEstimateTotal( + double frame_size_variation_bytes) const { + double frame_transmission_delay_ms = + GetFrameDelayVariationEstimateSizeBased(frame_size_variation_bytes); + double link_queuing_delay_ms = estimate_[1]; + return frame_transmission_delay_ms + link_queuing_delay_ms; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.h b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.h new file mode 100644 index 0000000000..a65ceefa10 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_VARIATION_KALMAN_FILTER_H_ +#define MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_VARIATION_KALMAN_FILTER_H_ + +#include "api/units/data_size.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +// This class uses a linear Kalman filter (see +// https://en.wikipedia.org/wiki/Kalman_filter) to estimate the frame delay +// variation (i.e., the difference in transmission time between a frame and the +// prior frame) for a frame, given its size variation in bytes (i.e., the +// difference in size between a frame and the prior frame). The idea is that, +// given a fixed link bandwidth, a larger frame (in bytes) would take +// proportionally longer to arrive than a correspondingly smaller frame. Using +// the variations of frame delay and frame size, the underlying bandwidth and +// queuing delay variation of the network link can be estimated. +// +// The filter takes as input the frame delay variation, the difference between +// the actual inter-frame arrival time and the expected inter-frame arrival time +// (based on RTP timestamp), and frame size variation, the inter-frame size +// delta for a single frame. The frame delay variation is seen as the +// measurement and the frame size variation is used in the observation model. +// The hidden state of the filter is the link bandwidth and queuing delay +// buildup. The estimated state can be used to get the expected frame delay +// variation for a frame, given its frame size variation. This information can +// then be used to estimate the frame delay variation coming from network +// jitter. +// +// Mathematical details: +// * The state (`x` in Wikipedia notation) is a 2x1 vector comprising the +// reciprocal of link bandwidth [1 / bytes per ms] and the +// link queuing delay buildup [ms]. +// * The state transition matrix (`F`) is the 2x2 identity matrix, meaning that +// link bandwidth and link queuing delay buildup are modeled as independent. +// * The measurement (`z`) is the (scalar) frame delay variation [ms]. +// * The observation matrix (`H`) is a 1x2 vector set as +// `{frame_size_variation [bytes], 1.0}`. +// * The state estimate covariance (`P`) is a symmetric 2x2 matrix. +// * The process noise covariance (`Q`) is a constant 2x2 diagonal matrix +// [(1 / bytes per ms)^2, ms^2]. +// * The observation noise covariance (`r`) is a scalar [ms^2] that is +// determined externally to this class. +class FrameDelayVariationKalmanFilter { + public: + FrameDelayVariationKalmanFilter(); + ~FrameDelayVariationKalmanFilter() = default; + + // Predicts and updates the filter, given a new pair of frame delay variation + // and frame size variation. + // + // Inputs: + // `frame_delay_variation_ms`: + // Frame delay variation as calculated by the `InterFrameDelay` estimator. + // + // `frame_size_variation_bytes`: + // Frame size variation, i.e., the current frame size minus the previous + // frame size (in bytes). Note that this quantity may be negative. + // + // `max_frame_size_bytes`: + // Filtered largest frame size received since the last reset. + // + // `var_noise`: + // Variance of the estimated random jitter. + // + // TODO(bugs.webrtc.org/14381): For now use doubles as input parameters as + // units defined in api/units have insufficient underlying precision for + // jitter estimation. + void PredictAndUpdate(double frame_delay_variation_ms, + double frame_size_variation_bytes, + double max_frame_size_bytes, + double var_noise); + + // Given a frame size variation, returns the estimated frame delay variation + // explained by the link bandwidth alone. + double GetFrameDelayVariationEstimateSizeBased( + double frame_size_variation_bytes) const; + + // Given a frame size variation, returns the estimated frame delay variation + // explained by both link bandwidth and link queuing delay buildup. + double GetFrameDelayVariationEstimateTotal( + double frame_size_variation_bytes) const; + + private: + // State estimate (bandwidth [1 / bytes per ms], queue buildup [ms]). + double estimate_[2]; + double estimate_cov_[2][2]; // Estimate covariance. + + // Process noise covariance. This is a diagonal matrix, so we only store the + // diagonal entries. + double process_noise_cov_diag_[2]; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_VARIATION_KALMAN_FILTER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_gn/moz.build new file mode 100644 index 0000000000..ecdcfc93e1 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_gn/moz.build @@ -0,0 +1,221 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("frame_delay_variation_kalman_filter_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_unittest.cc new file mode 100644 index 0000000000..6103f3a1bc --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_unittest.cc @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/frame_delay_variation_kalman_filter.h" + +#include "test/gtest.h" + +namespace webrtc { +namespace { + +// This test verifies that the initial filter state (link bandwidth, link +// propagation delay) is such that a frame of size zero would take no time to +// propagate. +TEST(FrameDelayVariationKalmanFilterTest, + InitializedFilterWithZeroSizeFrameTakesNoTimeToPropagate) { + FrameDelayVariationKalmanFilter filter; + + // A zero-sized frame... + double frame_size_variation_bytes = 0.0; + + // ...should take no time to propagate due to it's size... + EXPECT_EQ(filter.GetFrameDelayVariationEstimateSizeBased( + frame_size_variation_bytes), + 0.0); + + // ...and no time due to the initial link propagation delay being zero. + EXPECT_EQ( + filter.GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes), + 0.0); +} + +// TODO(brandtr): Look into if there is a factor 1000 missing here? It seems +// unreasonable to have an initial link bandwidth of 512 _mega_bits per second? +TEST(FrameDelayVariationKalmanFilterTest, + InitializedFilterWithSmallSizeFrameTakesFixedTimeToPropagate) { + FrameDelayVariationKalmanFilter filter; + + // A 1000-byte frame... + double frame_size_variation_bytes = 1000.0; + // ...should take around `1000.0 / (512e3 / 8.0) = 0.015625 ms` to transmit. + double expected_frame_delay_variation_estimate_ms = 1000.0 / (512e3 / 8.0); + + EXPECT_EQ(filter.GetFrameDelayVariationEstimateSizeBased( + frame_size_variation_bytes), + expected_frame_delay_variation_estimate_ms); + EXPECT_EQ( + filter.GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes), + expected_frame_delay_variation_estimate_ms); +} + +TEST(FrameDelayVariationKalmanFilterTest, + NegativeNoiseVarianceDoesNotUpdateFilter) { + FrameDelayVariationKalmanFilter filter; + + // Negative variance... + double var_noise = -0.1; + filter.PredictAndUpdate(/*frame_delay_variation_ms=*/3, + /*frame_size_variation_bytes=*/200.0, + /*max_frame_size_bytes=*/2000, var_noise); + + // ...does _not_ update the filter. + EXPECT_EQ(filter.GetFrameDelayVariationEstimateTotal( + /*frame_size_variation_bytes=*/0.0), + 0.0); + + // Positive variance... + var_noise = 0.1; + filter.PredictAndUpdate(/*frame_delay_variation_ms=*/3, + /*frame_size_variation_bytes=*/200.0, + /*max_frame_size_bytes=*/2000, var_noise); + + // ...does update the filter. + EXPECT_GT(filter.GetFrameDelayVariationEstimateTotal( + /*frame_size_variation_bytes=*/0.0), + 0.0); +} + +TEST(FrameDelayVariationKalmanFilterTest, + VerifyConvergenceWithAlternatingDeviations) { + FrameDelayVariationKalmanFilter filter; + + // One frame every 33 ms. + int framerate_fps = 30; + // Let's assume approximately 10% delay variation. + double frame_delay_variation_ms = 3; + // With a bitrate of 512 kbps, each frame will be around 2000 bytes. + double max_frame_size_bytes = 2000; + // And again, let's assume 10% size deviation. + double frame_size_variation_bytes = 200; + double var_noise = 0.1; + int test_duration_s = 60; + + for (int i = 0; i < test_duration_s * framerate_fps; ++i) { + // For simplicity, assume alternating variations. + double sign = (i % 2 == 0) ? 1.0 : -1.0; + filter.PredictAndUpdate(sign * frame_delay_variation_ms, + sign * frame_size_variation_bytes, + max_frame_size_bytes, var_noise); + } + + // Verify that the filter has converged within a margin of 0.1 ms. + EXPECT_NEAR( + filter.GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes), + frame_delay_variation_ms, 0.1); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc new file mode 100644 index 0000000000..bed9f875ee --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/inter_frame_delay.h" + +#include "absl/types/optional.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "modules/include/module_common_types_public.h" + +namespace webrtc { + +namespace { +constexpr Frequency k90kHz = Frequency::KiloHertz(90); +} + +InterFrameDelay::InterFrameDelay() { + Reset(); +} + +// Resets the delay estimate. +void InterFrameDelay::Reset() { + prev_wall_clock_ = absl::nullopt; + prev_rtp_timestamp_unwrapped_ = 0; +} + +// Calculates the delay of a frame with the given timestamp. +// This method is called when the frame is complete. +absl::optional<TimeDelta> InterFrameDelay::CalculateDelay( + uint32_t rtp_timestamp, + Timestamp now) { + int64_t rtp_timestamp_unwrapped = unwrapper_.Unwrap(rtp_timestamp); + if (!prev_wall_clock_) { + // First set of data, initialization, wait for next frame. + prev_wall_clock_ = now; + prev_rtp_timestamp_unwrapped_ = rtp_timestamp_unwrapped; + return TimeDelta::Zero(); + } + + // Account for reordering in jitter variance estimate in the future? + // Note that this also captures incomplete frames which are grabbed for + // decoding after a later frame has been complete, i.e. real packet losses. + uint32_t cropped_last = static_cast<uint32_t>(prev_rtp_timestamp_unwrapped_); + if (rtp_timestamp_unwrapped < prev_rtp_timestamp_unwrapped_ || + !IsNewerTimestamp(rtp_timestamp, cropped_last)) { + return absl::nullopt; + } + + // Compute the compensated timestamp difference. + int64_t d_rtp_ticks = rtp_timestamp_unwrapped - prev_rtp_timestamp_unwrapped_; + TimeDelta dts = d_rtp_ticks / k90kHz; + TimeDelta dt = now - *prev_wall_clock_; + + // frameDelay is the difference of dT and dTS -- i.e. the difference of the + // wall clock time difference and the timestamp difference between two + // following frames. + TimeDelta delay = dt - dts; + + prev_rtp_timestamp_unwrapped_ = rtp_timestamp_unwrapped; + prev_wall_clock_ = now; + return delay; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h new file mode 100644 index 0000000000..03b5f78cc1 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_H_ +#define MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_H_ + +#include <stdint.h> + +#include "absl/types/optional.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" + +namespace webrtc { + +class InterFrameDelay { + public: + InterFrameDelay(); + + // Resets the estimate. Zeros are given as parameters. + void Reset(); + + // Calculates the delay of a frame with the given timestamp. + // This method is called when the frame is complete. + absl::optional<TimeDelta> CalculateDelay(uint32_t rtp_timestamp, + Timestamp now); + + private: + // The previous rtp timestamp passed to the delay estimate + int64_t prev_rtp_timestamp_unwrapped_; + RtpTimestampUnwrapper unwrapper_; + + // The previous wall clock timestamp used by the delay estimate + absl::optional<Timestamp> prev_wall_clock_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build new file mode 100644 index 0000000000..765d42aade --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build @@ -0,0 +1,221 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("inter_frame_delay_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc new file mode 100644 index 0000000000..183b378ced --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/inter_frame_delay.h" + +#include <limits> + +#include "absl/types/optional.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +// Test is for frames at 30fps. At 30fps, RTP timestamps will increase by +// 90000 / 30 = 3000 ticks per frame. +constexpr Frequency k30Fps = Frequency::Hertz(30); +constexpr TimeDelta kFrameDelay = 1 / k30Fps; +constexpr uint32_t kRtpTicksPerFrame = Frequency::KiloHertz(90) / k30Fps; +constexpr Timestamp kStartTime = Timestamp::Millis(1337); + +} // namespace + +using ::testing::Eq; +using ::testing::Optional; + +TEST(InterFrameDelayTest, OldRtpTimestamp) { + InterFrameDelay inter_frame_delay; + EXPECT_THAT(inter_frame_delay.CalculateDelay(180000, kStartTime), + Optional(TimeDelta::Zero())); + EXPECT_THAT(inter_frame_delay.CalculateDelay(90000, kStartTime), + Eq(absl::nullopt)); +} + +TEST(InterFrameDelayTest, NegativeWrapAroundIsSameAsOldRtpTimestamp) { + InterFrameDelay inter_frame_delay; + uint32_t rtp = 1500; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, kStartTime), + Optional(TimeDelta::Zero())); + // RTP has wrapped around backwards. + rtp -= 3000; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, kStartTime), + Eq(absl::nullopt)); +} + +TEST(InterFrameDelayTest, CorrectDelayForFrames) { + InterFrameDelay inter_frame_delay; + // Use a fake clock to simplify time keeping. + SimulatedClock clock(kStartTime); + + // First frame is always delay 0. + uint32_t rtp = 90000; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Perfectly timed frame has 0 delay. + clock.AdvanceTime(kFrameDelay); + rtp += kRtpTicksPerFrame; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Slightly early frame will have a negative delay. + clock.AdvanceTime(kFrameDelay - TimeDelta::Millis(3)); + rtp += kRtpTicksPerFrame; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-TimeDelta::Millis(3))); + + // Slightly late frame will have positive delay. + clock.AdvanceTime(kFrameDelay + TimeDelta::Micros(5125)); + rtp += kRtpTicksPerFrame; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Micros(5125))); + + // Simulate faster frame RTP at the same clock delay. The frame arrives late, + // since the RTP timestamp is faster than the delay, and thus is positive. + clock.AdvanceTime(kFrameDelay); + rtp += kRtpTicksPerFrame / 2; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(kFrameDelay / 2.0)); + + // Simulate slower frame RTP at the same clock delay. The frame is early, + // since the RTP timestamp advanced more than the delay, and thus is negative. + clock.AdvanceTime(kFrameDelay); + rtp += 1.5 * kRtpTicksPerFrame; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-kFrameDelay / 2.0)); +} + +TEST(InterFrameDelayTest, PositiveWrapAround) { + InterFrameDelay inter_frame_delay; + // Use a fake clock to simplify time keeping. + SimulatedClock clock(kStartTime); + + // First frame is behind the max RTP by 1500. + uint32_t rtp = std::numeric_limits<uint32_t>::max() - 1500; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Rtp wraps around, now 1499. + rtp += kRtpTicksPerFrame; + + // Frame delay should be as normal, in this case simulated as 1ms late. + clock.AdvanceTime(kFrameDelay + TimeDelta::Millis(1)); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Millis(1))); +} + +TEST(InterFrameDelayTest, MultipleWrapArounds) { + // Simulate a long pauses which cause wrap arounds multiple times. + constexpr Frequency k90Khz = Frequency::KiloHertz(90); + constexpr uint32_t kHalfRtp = std::numeric_limits<uint32_t>::max() / 2; + constexpr TimeDelta kWrapAroundDelay = kHalfRtp / k90Khz; + + InterFrameDelay inter_frame_delay; + // Use a fake clock to simplify time keeping. + SimulatedClock clock(kStartTime); + uint32_t rtp = 0; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + rtp += kHalfRtp; + clock.AdvanceTime(kWrapAroundDelay); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + // 1st wrap around. + rtp += kHalfRtp + 1; + clock.AdvanceTime(kWrapAroundDelay + TimeDelta::Millis(1)); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Millis(1) - (1 / k90Khz))); + + rtp += kHalfRtp; + clock.AdvanceTime(kWrapAroundDelay); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + // 2nd wrap arounds. + rtp += kHalfRtp + 1; + clock.AdvanceTime(kWrapAroundDelay - TimeDelta::Millis(1)); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-TimeDelta::Millis(1) - (1 / k90Khz))); + + // Ensure short delay (large RTP delay) between wrap-arounds has correct + // jitter. + rtp += kHalfRtp; + clock.AdvanceTime(TimeDelta::Millis(10)); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-(kWrapAroundDelay - TimeDelta::Millis(10)))); + // 3nd wrap arounds, this time with large RTP delay. + rtp += kHalfRtp + 1; + clock.AdvanceTime(TimeDelta::Millis(10)); + EXPECT_THAT( + inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-(kWrapAroundDelay - TimeDelta::Millis(10) + (1 / k90Khz)))); +} + +TEST(InterFrameDelayTest, NegativeWrapAroundAfterPositiveWrapAround) { + InterFrameDelay inter_frame_delay; + // Use a fake clock to simplify time keeping. + SimulatedClock clock(kStartTime); + uint32_t rtp = std::numeric_limits<uint32_t>::max() - 1500; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Rtp wraps around, now 1499. + rtp += kRtpTicksPerFrame; + // Frame delay should be as normal, in this case simulated as 1ms late. + clock.AdvanceTime(kFrameDelay); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Wrap back. + rtp -= kRtpTicksPerFrame; + // Frame delay should be as normal, in this case simulated as 1ms late. + clock.AdvanceTime(kFrameDelay); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Eq(absl::nullopt)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc new file mode 100644 index 0000000000..62757787a1 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc @@ -0,0 +1,476 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/jitter_estimator.h" + +#include <math.h> +#include <string.h> + +#include <algorithm> +#include <cstdint> + +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "modules/video_coding/timing/rtt_filter.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace { + +// Number of frames to wait for before post processing estimate. Also used in +// the frame rate estimator ramp-up. +constexpr size_t kFrameProcessingStartupCount = 30; + +// Number of frames to wait for before enabling the frame size filters. +constexpr size_t kFramesUntilSizeFiltering = 5; + +// Initial value for frame size filters. +constexpr double kInitialAvgAndMaxFrameSizeBytes = 500.0; + +// Time constant for average frame size filter. +constexpr double kPhi = 0.97; +// Time constant for max frame size filter. +constexpr double kPsi = 0.9999; +// Default constants for percentile frame size filter. +constexpr double kDefaultMaxFrameSizePercentile = 0.95; +constexpr int kDefaultFrameSizeWindow = 30 * 10; + +// Outlier rejection constants. +constexpr double kNumStdDevDelayClamp = 3.5; +constexpr double kNumStdDevDelayOutlier = 15.0; +constexpr double kNumStdDevSizeOutlier = 3.0; +constexpr double kCongestionRejectionFactor = -0.25; + +// Rampup constant for deviation noise filters. +constexpr size_t kAlphaCountMax = 400; + +// Noise threshold constants. +// ~Less than 1% chance (look up in normal distribution table)... +constexpr double kNoiseStdDevs = 2.33; +// ...of getting 30 ms freezes +constexpr double kNoiseStdDevOffset = 30.0; + +// Jitter estimate clamping limits. +constexpr TimeDelta kMinJitterEstimate = TimeDelta::Millis(1); +constexpr TimeDelta kMaxJitterEstimate = TimeDelta::Seconds(10); + +// A constant describing the delay from the jitter buffer to the delay on the +// receiving side which is not accounted for by the jitter buffer nor the +// decoding delay estimate. +constexpr TimeDelta OPERATING_SYSTEM_JITTER = TimeDelta::Millis(10); + +// Time constant for reseting the NACK count. +constexpr TimeDelta kNackCountTimeout = TimeDelta::Seconds(60); + +// RTT mult activation. +constexpr size_t kNackLimit = 3; + +// Frame rate estimate clamping limit. +constexpr Frequency kMaxFramerateEstimate = Frequency::Hertz(200); + +} // namespace + +constexpr char JitterEstimator::Config::kFieldTrialsKey[]; + +JitterEstimator::Config JitterEstimator::Config::ParseAndValidate( + absl::string_view field_trial) { + Config config; + config.Parser()->Parse(field_trial); + + // The `MovingPercentileFilter` RTC_CHECKs on the validity of the + // percentile and window length, so we'd better validate the field trial + // provided values here. + if (config.max_frame_size_percentile) { + double original = *config.max_frame_size_percentile; + config.max_frame_size_percentile = std::min(std::max(0.0, original), 1.0); + if (config.max_frame_size_percentile != original) { + RTC_LOG(LS_ERROR) << "Skipping invalid max_frame_size_percentile=" + << original; + } + } + if (config.frame_size_window && config.frame_size_window < 1) { + RTC_LOG(LS_ERROR) << "Skipping invalid frame_size_window=" + << *config.frame_size_window; + config.frame_size_window = 1; + } + + // General sanity checks. + if (config.num_stddev_delay_clamp && config.num_stddev_delay_clamp < 0.0) { + RTC_LOG(LS_ERROR) << "Skipping invalid num_stddev_delay_clamp=" + << *config.num_stddev_delay_clamp; + config.num_stddev_delay_clamp = 0.0; + } + if (config.num_stddev_delay_outlier && + config.num_stddev_delay_outlier < 0.0) { + RTC_LOG(LS_ERROR) << "Skipping invalid num_stddev_delay_outlier=" + << *config.num_stddev_delay_outlier; + config.num_stddev_delay_outlier = 0.0; + } + if (config.num_stddev_size_outlier && config.num_stddev_size_outlier < 0.0) { + RTC_LOG(LS_ERROR) << "Skipping invalid num_stddev_size_outlier=" + << *config.num_stddev_size_outlier; + config.num_stddev_size_outlier = 0.0; + } + + return config; +} + +JitterEstimator::JitterEstimator(Clock* clock, + const FieldTrialsView& field_trials) + : config_(Config::ParseAndValidate( + field_trials.Lookup(Config::kFieldTrialsKey))), + avg_frame_size_median_bytes_(static_cast<size_t>( + config_.frame_size_window.value_or(kDefaultFrameSizeWindow))), + max_frame_size_bytes_percentile_( + config_.max_frame_size_percentile.value_or( + kDefaultMaxFrameSizePercentile), + static_cast<size_t>( + config_.frame_size_window.value_or(kDefaultFrameSizeWindow))), + fps_counter_(30), // TODO(sprang): Use an estimator with limit based + // on time, rather than number of samples. + clock_(clock) { + Reset(); +} + +JitterEstimator::~JitterEstimator() = default; + +// Resets the JitterEstimate. +void JitterEstimator::Reset() { + avg_frame_size_bytes_ = kInitialAvgAndMaxFrameSizeBytes; + max_frame_size_bytes_ = kInitialAvgAndMaxFrameSizeBytes; + var_frame_size_bytes2_ = 100; + avg_frame_size_median_bytes_.Reset(); + max_frame_size_bytes_percentile_.Reset(); + last_update_time_ = absl::nullopt; + prev_estimate_ = absl::nullopt; + prev_frame_size_ = absl::nullopt; + avg_noise_ms_ = 0.0; + var_noise_ms2_ = 4.0; + alpha_count_ = 1; + filter_jitter_estimate_ = TimeDelta::Zero(); + latest_nack_ = Timestamp::Zero(); + nack_count_ = 0; + startup_frame_size_sum_bytes_ = 0; + startup_frame_size_count_ = 0; + startup_count_ = 0; + rtt_filter_.Reset(); + fps_counter_.Reset(); + + kalman_filter_ = FrameDelayVariationKalmanFilter(); +} + +// Updates the estimates with the new measurements. +void JitterEstimator::UpdateEstimate(TimeDelta frame_delay, + DataSize frame_size) { + if (frame_size.IsZero()) { + return; + } + // Can't use DataSize since this can be negative. + double delta_frame_bytes = + frame_size.bytes() - prev_frame_size_.value_or(DataSize::Zero()).bytes(); + if (startup_frame_size_count_ < kFramesUntilSizeFiltering) { + startup_frame_size_sum_bytes_ += frame_size.bytes(); + startup_frame_size_count_++; + } else if (startup_frame_size_count_ == kFramesUntilSizeFiltering) { + // Give the frame size filter. + avg_frame_size_bytes_ = startup_frame_size_sum_bytes_ / + static_cast<double>(startup_frame_size_count_); + startup_frame_size_count_++; + } + + double avg_frame_size_bytes = + kPhi * avg_frame_size_bytes_ + (1 - kPhi) * frame_size.bytes(); + double deviation_size_bytes = 2 * sqrt(var_frame_size_bytes2_); + if (frame_size.bytes() < avg_frame_size_bytes_ + deviation_size_bytes) { + // Only update the average frame size if this sample wasn't a key frame. + avg_frame_size_bytes_ = avg_frame_size_bytes; + } + + double delta_bytes = frame_size.bytes() - avg_frame_size_bytes; + var_frame_size_bytes2_ = std::max( + kPhi * var_frame_size_bytes2_ + (1 - kPhi) * (delta_bytes * delta_bytes), + 1.0); + + // Update non-linear IIR estimate of max frame size. + max_frame_size_bytes_ = + std::max<double>(kPsi * max_frame_size_bytes_, frame_size.bytes()); + + // Maybe update percentile estimates of frame sizes. + if (config_.avg_frame_size_median) { + avg_frame_size_median_bytes_.Insert(frame_size.bytes()); + } + if (config_.MaxFrameSizePercentileEnabled()) { + max_frame_size_bytes_percentile_.Insert(frame_size.bytes()); + } + + if (!prev_frame_size_) { + prev_frame_size_ = frame_size; + return; + } + prev_frame_size_ = frame_size; + + // Cap frame_delay based on the current time deviation noise. + double num_stddev_delay_clamp = + config_.num_stddev_delay_clamp.value_or(kNumStdDevDelayClamp); + TimeDelta max_time_deviation = + TimeDelta::Millis(num_stddev_delay_clamp * sqrt(var_noise_ms2_) + 0.5); + frame_delay.Clamp(-max_time_deviation, max_time_deviation); + + double delay_deviation_ms = + frame_delay.ms() - + kalman_filter_.GetFrameDelayVariationEstimateTotal(delta_frame_bytes); + + // Outlier rejection: these conditions depend on filtered versions of the + // delay and frame size _means_, respectively, together with a configurable + // number of standard deviations. If a sample is large with respect to the + // corresponding mean and dispersion (defined by the number of + // standard deviations and the sample standard deviation), it is deemed an + // outlier. This "empirical rule" is further described in + // https://en.wikipedia.org/wiki/68-95-99.7_rule. Note that neither of the + // estimated means are true sample means, which implies that they are possibly + // not normally distributed. Hence, this rejection method is just a heuristic. + double num_stddev_delay_outlier = + config_.num_stddev_delay_outlier.value_or(kNumStdDevDelayOutlier); + // Delay outlier rejection is two-sided. + bool abs_delay_is_not_outlier = + fabs(delay_deviation_ms) < + num_stddev_delay_outlier * sqrt(var_noise_ms2_); + // The reasoning above means, in particular, that we should use the sample + // mean-style `avg_frame_size_bytes_` estimate, as opposed to the + // median-filtered version, even if configured to use latter for the + // calculation in `CalculateEstimate()`. + // Size outlier rejection is one-sided. + double num_stddev_size_outlier = + config_.num_stddev_size_outlier.value_or(kNumStdDevSizeOutlier); + bool size_is_positive_outlier = + frame_size.bytes() > + avg_frame_size_bytes_ + + num_stddev_size_outlier * sqrt(var_frame_size_bytes2_); + + // Only update the Kalman filter if the sample is not considered an extreme + // outlier. Even if it is an extreme outlier from a delay point of view, if + // the frame size also is large the deviation is probably due to an incorrect + // line slope. + if (abs_delay_is_not_outlier || size_is_positive_outlier) { + // Prevent updating with frames which have been congested by a large frame, + // and therefore arrives almost at the same time as that frame. + // This can occur when we receive a large frame (key frame) which has been + // delayed. The next frame is of normal size (delta frame), and thus deltaFS + // will be << 0. This removes all frame samples which arrives after a key + // frame. + double congestion_rejection_factor = + config_.congestion_rejection_factor.value_or( + kCongestionRejectionFactor); + double filtered_max_frame_size_bytes = + config_.MaxFrameSizePercentileEnabled() + ? max_frame_size_bytes_percentile_.GetFilteredValue() + : max_frame_size_bytes_; + bool is_not_congested = + delta_frame_bytes > + congestion_rejection_factor * filtered_max_frame_size_bytes; + + if (is_not_congested || config_.estimate_noise_when_congested) { + // Update the variance of the deviation from the line given by the Kalman + // filter. + EstimateRandomJitter(delay_deviation_ms); + } + if (is_not_congested) { + // Neither a delay outlier nor a congested frame, so we can safely update + // the Kalman filter with the sample. + kalman_filter_.PredictAndUpdate(frame_delay.ms(), delta_frame_bytes, + filtered_max_frame_size_bytes, + var_noise_ms2_); + } + } else { + // Delay outliers affect the noise estimate through a value equal to the + // outlier rejection threshold. + double num_stddev = (delay_deviation_ms >= 0) ? num_stddev_delay_outlier + : -num_stddev_delay_outlier; + EstimateRandomJitter(num_stddev * sqrt(var_noise_ms2_)); + } + // Post process the total estimated jitter + if (startup_count_ >= kFrameProcessingStartupCount) { + PostProcessEstimate(); + } else { + startup_count_++; + } +} + +// Updates the nack/packet ratio. +void JitterEstimator::FrameNacked() { + if (nack_count_ < kNackLimit) { + nack_count_++; + } + latest_nack_ = clock_->CurrentTime(); +} + +void JitterEstimator::UpdateRtt(TimeDelta rtt) { + rtt_filter_.Update(rtt); +} + +JitterEstimator::Config JitterEstimator::GetConfigForTest() const { + return config_; +} + +// Estimates the random jitter by calculating the variance of the sample +// distance from the line given by the Kalman filter. +void JitterEstimator::EstimateRandomJitter(double d_dT) { + Timestamp now = clock_->CurrentTime(); + if (last_update_time_.has_value()) { + fps_counter_.AddSample((now - *last_update_time_).us()); + } + last_update_time_ = now; + + if (alpha_count_ == 0) { + RTC_DCHECK_NOTREACHED(); + return; + } + double alpha = + static_cast<double>(alpha_count_ - 1) / static_cast<double>(alpha_count_); + alpha_count_++; + if (alpha_count_ > kAlphaCountMax) + alpha_count_ = kAlphaCountMax; + + // In order to avoid a low frame rate stream to react slower to changes, + // scale the alpha weight relative a 30 fps stream. + Frequency fps = GetFrameRate(); + if (fps > Frequency::Zero()) { + constexpr Frequency k30Fps = Frequency::Hertz(30); + double rate_scale = k30Fps / fps; + // At startup, there can be a lot of noise in the fps estimate. + // Interpolate rate_scale linearly, from 1.0 at sample #1, to 30.0 / fps + // at sample #kFrameProcessingStartupCount. + if (alpha_count_ < kFrameProcessingStartupCount) { + rate_scale = (alpha_count_ * rate_scale + + (kFrameProcessingStartupCount - alpha_count_)) / + kFrameProcessingStartupCount; + } + alpha = pow(alpha, rate_scale); + } + + double avg_noise_ms = alpha * avg_noise_ms_ + (1 - alpha) * d_dT; + double var_noise_ms2 = alpha * var_noise_ms2_ + (1 - alpha) * + (d_dT - avg_noise_ms_) * + (d_dT - avg_noise_ms_); + avg_noise_ms_ = avg_noise_ms; + var_noise_ms2_ = var_noise_ms2; + if (var_noise_ms2_ < 1.0) { + // The variance should never be zero, since we might get stuck and consider + // all samples as outliers. + var_noise_ms2_ = 1.0; + } +} + +double JitterEstimator::NoiseThreshold() const { + double noise_threshold_ms = + kNoiseStdDevs * sqrt(var_noise_ms2_) - kNoiseStdDevOffset; + if (noise_threshold_ms < 1.0) { + noise_threshold_ms = 1.0; + } + return noise_threshold_ms; +} + +// Calculates the current jitter estimate from the filtered estimates. +TimeDelta JitterEstimator::CalculateEstimate() { + // Using median- and percentile-filtered versions of the frame sizes may be + // more robust than using sample mean-style estimates. + double filtered_avg_frame_size_bytes = + config_.avg_frame_size_median + ? avg_frame_size_median_bytes_.GetFilteredValue() + : avg_frame_size_bytes_; + double filtered_max_frame_size_bytes = + config_.MaxFrameSizePercentileEnabled() + ? max_frame_size_bytes_percentile_.GetFilteredValue() + : max_frame_size_bytes_; + double worst_case_frame_size_deviation_bytes = + filtered_max_frame_size_bytes - filtered_avg_frame_size_bytes; + double ret_ms = kalman_filter_.GetFrameDelayVariationEstimateSizeBased( + worst_case_frame_size_deviation_bytes) + + NoiseThreshold(); + TimeDelta ret = TimeDelta::Millis(ret_ms); + + // A very low estimate (or negative) is neglected. + if (ret < kMinJitterEstimate) { + ret = prev_estimate_.value_or(kMinJitterEstimate); + // Sanity check to make sure that no other method has set `prev_estimate_` + // to a value lower than `kMinJitterEstimate`. + RTC_DCHECK_GE(ret, kMinJitterEstimate); + } else if (ret > kMaxJitterEstimate) { // Sanity + ret = kMaxJitterEstimate; + } + prev_estimate_ = ret; + return ret; +} + +void JitterEstimator::PostProcessEstimate() { + filter_jitter_estimate_ = CalculateEstimate(); +} + +// Returns the current filtered estimate if available, +// otherwise tries to calculate an estimate. +TimeDelta JitterEstimator::GetJitterEstimate( + double rtt_multiplier, + absl::optional<TimeDelta> rtt_mult_add_cap) { + TimeDelta jitter = CalculateEstimate() + OPERATING_SYSTEM_JITTER; + Timestamp now = clock_->CurrentTime(); + + if (now - latest_nack_ > kNackCountTimeout) + nack_count_ = 0; + + if (filter_jitter_estimate_ > jitter) + jitter = filter_jitter_estimate_; + if (nack_count_ >= kNackLimit) { + if (rtt_mult_add_cap.has_value()) { + jitter += std::min(rtt_filter_.Rtt() * rtt_multiplier, + rtt_mult_add_cap.value()); + } else { + jitter += rtt_filter_.Rtt() * rtt_multiplier; + } + } + + static const Frequency kJitterScaleLowThreshold = Frequency::Hertz(5); + static const Frequency kJitterScaleHighThreshold = Frequency::Hertz(10); + Frequency fps = GetFrameRate(); + // Ignore jitter for very low fps streams. + if (fps < kJitterScaleLowThreshold) { + if (fps.IsZero()) { + return std::max(TimeDelta::Zero(), jitter); + } + return TimeDelta::Zero(); + } + + // Semi-low frame rate; scale by factor linearly interpolated from 0.0 at + // kJitterScaleLowThreshold to 1.0 at kJitterScaleHighThreshold. + if (fps < kJitterScaleHighThreshold) { + jitter = (1.0 / (kJitterScaleHighThreshold - kJitterScaleLowThreshold)) * + (fps - kJitterScaleLowThreshold) * jitter; + } + + return std::max(TimeDelta::Zero(), jitter); +} + +Frequency JitterEstimator::GetFrameRate() const { + TimeDelta mean_frame_period = TimeDelta::Micros(fps_counter_.ComputeMean()); + if (mean_frame_period <= TimeDelta::Zero()) + return Frequency::Zero(); + + Frequency fps = 1 / mean_frame_period; + // Sanity check. + RTC_DCHECK_GE(fps, Frequency::Zero()); + return std::min(fps, kMaxFramerateEstimate); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h new file mode 100644 index 0000000000..a89a4bf1fd --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_ +#define MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_ + +#include <algorithm> +#include <memory> +#include <queue> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "modules/video_coding/timing/frame_delay_variation_kalman_filter.h" +#include "modules/video_coding/timing/rtt_filter.h" +#include "rtc_base/experiments/struct_parameters_parser.h" +#include "rtc_base/numerics/moving_percentile_filter.h" +#include "rtc_base/rolling_accumulator.h" + +namespace webrtc { + +class Clock; + +class JitterEstimator { + public: + // Configuration struct for statically overriding some constants and + // behaviour, configurable through field trials. + struct Config { + static constexpr char kFieldTrialsKey[] = "WebRTC-JitterEstimatorConfig"; + + // Parses a field trial string and validates the values. + static Config ParseAndValidate(absl::string_view field_trial); + + std::unique_ptr<StructParametersParser> Parser() { + // clang-format off + return StructParametersParser::Create( + "avg_frame_size_median", &avg_frame_size_median, + "max_frame_size_percentile", &max_frame_size_percentile, + "frame_size_window", &frame_size_window, + "num_stddev_delay_clamp", &num_stddev_delay_clamp, + "num_stddev_delay_outlier", &num_stddev_delay_outlier, + "num_stddev_size_outlier", &num_stddev_size_outlier, + "congestion_rejection_factor", &congestion_rejection_factor, + "estimate_noise_when_congested", &estimate_noise_when_congested); + // clang-format on + } + + bool MaxFrameSizePercentileEnabled() const { + return max_frame_size_percentile.has_value(); + } + + // If true, the "avg" frame size is calculated as the median over a window + // of recent frame sizes. + bool avg_frame_size_median = false; + + // If set, the "max" frame size is calculated as this percentile over a + // window of recent frame sizes. + absl::optional<double> max_frame_size_percentile = absl::nullopt; + + // The length of the percentile filters' window, in number of frames. + absl::optional<int> frame_size_window = absl::nullopt; + + // The incoming frame delay variation samples are clamped to be at most + // this number of standard deviations away from zero. + // + // Increasing this value clamps fewer samples. + absl::optional<double> num_stddev_delay_clamp = absl::nullopt; + + // A (relative) frame delay variation sample is an outlier if its absolute + // deviation from the Kalman filter model falls outside this number of + // sample standard deviations. + // + // Increasing this value rejects fewer samples. + absl::optional<double> num_stddev_delay_outlier = absl::nullopt; + + // An (absolute) frame size sample is an outlier if its positive deviation + // from the estimated average frame size falls outside this number of sample + // standard deviations. + // + // Increasing this value rejects fewer samples. + absl::optional<double> num_stddev_size_outlier = absl::nullopt; + + // A (relative) frame size variation sample is deemed "congested", and is + // thus rejected, if its value is less than this factor times the estimated + // max frame size. + // + // Decreasing this value rejects fewer samples. + absl::optional<double> congestion_rejection_factor = absl::nullopt; + + // If true, the noise estimate will be updated for congestion rejected + // frames. This is currently enabled by default, but that may not be optimal + // since congested frames typically are not spread around the line with + // Gaussian noise. (This is the whole reason for the congestion rejection!) + bool estimate_noise_when_congested = true; + }; + + JitterEstimator(Clock* clock, const FieldTrialsView& field_trials); + JitterEstimator(const JitterEstimator&) = delete; + JitterEstimator& operator=(const JitterEstimator&) = delete; + ~JitterEstimator(); + + // Resets the estimate to the initial state. + void Reset(); + + // Updates the jitter estimate with the new data. + // + // Input: + // - frame_delay : Delay-delta calculated by UTILDelayEstimate. + // - frame_size : Frame size of the current frame. + void UpdateEstimate(TimeDelta frame_delay, DataSize frame_size); + + // Returns the current jitter estimate and adds an RTT dependent term in cases + // of retransmission. + // Input: + // - rtt_multiplier : RTT param multiplier (when applicable). + // - rtt_mult_add_cap : Multiplier cap from the RTTMultExperiment. + // + // Return value : Jitter estimate. + TimeDelta GetJitterEstimate(double rtt_multiplier, + absl::optional<TimeDelta> rtt_mult_add_cap); + + // Updates the nack counter. + void FrameNacked(); + + // Updates the RTT filter. + // + // Input: + // - rtt : Round trip time. + void UpdateRtt(TimeDelta rtt); + + // Returns the configuration. Only to be used by unit tests. + Config GetConfigForTest() const; + + private: + // Updates the random jitter estimate, i.e. the variance of the time + // deviations from the line given by the Kalman filter. + // + // Input: + // - d_dT : The deviation from the kalman estimate. + void EstimateRandomJitter(double d_dT); + + double NoiseThreshold() const; + + // Calculates the current jitter estimate. + // + // Return value : The current jitter estimate. + TimeDelta CalculateEstimate(); + + // Post process the calculated estimate. + void PostProcessEstimate(); + + // Returns the estimated incoming frame rate. + Frequency GetFrameRate() const; + + // Configuration that may override some internals. + const Config config_; + + // Filters the {frame_delay_delta, frame_size_delta} measurements through + // a linear Kalman filter. + FrameDelayVariationKalmanFilter kalman_filter_; + + // TODO(bugs.webrtc.org/14381): Update `avg_frame_size_bytes_` to DataSize + // when api/units have sufficient precision. + double avg_frame_size_bytes_; // Average frame size + double var_frame_size_bytes2_; // Frame size variance. Unit is bytes^2. + // Largest frame size received (descending with a factor kPsi). + // Used by default. + // TODO(bugs.webrtc.org/14381): Update `max_frame_size_bytes_` to DataSize + // when api/units have sufficient precision. + double max_frame_size_bytes_; + // Percentile frame sized received (over a window). Only used if configured. + MovingMedianFilter<int64_t> avg_frame_size_median_bytes_; + MovingPercentileFilter<int64_t> max_frame_size_bytes_percentile_; + // TODO(bugs.webrtc.org/14381): Update `startup_frame_size_sum_bytes_` to + // DataSize when api/units have sufficient precision. + double startup_frame_size_sum_bytes_; + size_t startup_frame_size_count_; + + absl::optional<Timestamp> last_update_time_; + // The previously returned jitter estimate + absl::optional<TimeDelta> prev_estimate_; + // Frame size of the previous frame + absl::optional<DataSize> prev_frame_size_; + // Average of the random jitter. Unit is milliseconds. + double avg_noise_ms_; + // Variance of the time-deviation from the line. Unit is milliseconds^2. + double var_noise_ms2_; + size_t alpha_count_; + // The filtered sum of jitter estimates + TimeDelta filter_jitter_estimate_ = TimeDelta::Zero(); + + size_t startup_count_; + // Time when the latest nack was seen + Timestamp latest_nack_ = Timestamp::Zero(); + // Keeps track of the number of nacks received, but never goes above + // kNackLimit. + size_t nack_count_; + RttFilter rtt_filter_; + + // Tracks frame rates in microseconds. + rtc::RollingAccumulator<uint64_t> fps_counter_; + Clock* clock_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build new file mode 100644 index 0000000000..a0d6154d78 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("jitter_estimator_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc new file mode 100644 index 0000000000..8e0c01587f --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc @@ -0,0 +1,305 @@ +/* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/jitter_estimator.h" + +#include <stdint.h> + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/field_trials.h" +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "rtc_base/numerics/histogram_percentile_counter.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +// Generates some simple test data in the form of a sawtooth wave. +class ValueGenerator { + public: + explicit ValueGenerator(int32_t amplitude) + : amplitude_(amplitude), counter_(0) {} + + virtual ~ValueGenerator() = default; + + TimeDelta Delay() const { + return TimeDelta::Millis((counter_ % 11) - 5) * amplitude_; + } + + DataSize FrameSize() const { + return DataSize::Bytes(1000 + Delay().ms() / 5); + } + + void Advance() { ++counter_; } + + private: + const int32_t amplitude_; + int64_t counter_; +}; + +class JitterEstimatorTest : public ::testing::Test { + protected: + explicit JitterEstimatorTest(const std::string& field_trials) + : fake_clock_(0), + field_trials_(FieldTrials::CreateNoGlobal(field_trials)), + estimator_(&fake_clock_, *field_trials_) {} + JitterEstimatorTest() : JitterEstimatorTest("") {} + virtual ~JitterEstimatorTest() {} + + void Run(int duration_s, int framerate_fps, ValueGenerator& gen) { + TimeDelta tick = 1 / Frequency::Hertz(framerate_fps); + for (int i = 0; i < duration_s * framerate_fps; ++i) { + estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + fake_clock_.AdvanceTime(tick); + gen.Advance(); + } + } + + SimulatedClock fake_clock_; + std::unique_ptr<FieldTrials> field_trials_; + JitterEstimator estimator_; +}; + +TEST_F(JitterEstimatorTest, SteadyStateConvergence) { + ValueGenerator gen(10); + Run(/*duration_s=*/60, /*framerate_fps=*/30, gen); + EXPECT_EQ(estimator_.GetJitterEstimate(0, absl::nullopt).ms(), 54); +} + +TEST_F(JitterEstimatorTest, + SizeOutlierIsNotRejectedAndIncreasesJitterEstimate) { + ValueGenerator gen(10); + + // Steady state. + Run(/*duration_s=*/60, /*framerate_fps=*/30, gen); + TimeDelta steady_state_jitter = + estimator_.GetJitterEstimate(0, absl::nullopt); + + // A single outlier frame size... + estimator_.UpdateEstimate(gen.Delay(), 10 * gen.FrameSize()); + TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt); + + // ...changes the estimate. + EXPECT_GT(outlier_jitter.ms(), 1.25 * steady_state_jitter.ms()); +} + +TEST_F(JitterEstimatorTest, LowFramerateDisablesJitterEstimator) { + ValueGenerator gen(10); + // At 5 fps, we disable jitter delay altogether. + TimeDelta time_delta = 1 / Frequency::Hertz(5); + for (int i = 0; i < 60; ++i) { + estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + fake_clock_.AdvanceTime(time_delta); + if (i > 2) + EXPECT_EQ(estimator_.GetJitterEstimate(0, absl::nullopt), + TimeDelta::Zero()); + gen.Advance(); + } +} + +TEST_F(JitterEstimatorTest, RttMultAddCap) { + std::vector<std::pair<TimeDelta, rtc::HistogramPercentileCounter>> + jitter_by_rtt_mult_cap; + jitter_by_rtt_mult_cap.emplace_back( + /*rtt_mult_add_cap=*/TimeDelta::Millis(10), /*long_tail_boundary=*/1000); + jitter_by_rtt_mult_cap.emplace_back( + /*rtt_mult_add_cap=*/TimeDelta::Millis(200), /*long_tail_boundary=*/1000); + + for (auto& [rtt_mult_add_cap, jitter] : jitter_by_rtt_mult_cap) { + estimator_.Reset(); + + ValueGenerator gen(50); + TimeDelta time_delta = 1 / Frequency::Hertz(30); + constexpr TimeDelta kRtt = TimeDelta::Millis(250); + for (int i = 0; i < 100; ++i) { + estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + fake_clock_.AdvanceTime(time_delta); + estimator_.FrameNacked(); + estimator_.UpdateRtt(kRtt); + jitter.Add( + estimator_.GetJitterEstimate(/*rtt_mult=*/1.0, rtt_mult_add_cap) + .ms()); + gen.Advance(); + } + } + + // 200ms cap should result in at least 25% higher max compared to 10ms. + EXPECT_GT(*jitter_by_rtt_mult_cap[1].second.GetPercentile(1.0), + *jitter_by_rtt_mult_cap[0].second.GetPercentile(1.0) * 1.25); +} + +// By default, the `JitterEstimator` is not robust against single large frames. +TEST_F(JitterEstimatorTest, Single2xFrameSizeImpactsJitterEstimate) { + ValueGenerator gen(10); + + // Steady state. + Run(/*duration_s=*/60, /*framerate_fps=*/30, gen); + TimeDelta steady_state_jitter = + estimator_.GetJitterEstimate(0, absl::nullopt); + + // A single outlier frame size... + estimator_.UpdateEstimate(gen.Delay(), 2 * gen.FrameSize()); + TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt); + + // ...impacts the estimate. + EXPECT_GT(outlier_jitter.ms(), steady_state_jitter.ms()); +} + +// Under the default config, congested frames are used when calculating the +// noise variance, meaning that they will impact the final jitter estimate. +TEST_F(JitterEstimatorTest, CongestedFrameImpactsJitterEstimate) { + ValueGenerator gen(10); + + // Steady state. + Run(/*duration_s=*/10, /*framerate_fps=*/30, gen); + TimeDelta steady_state_jitter = + estimator_.GetJitterEstimate(0, absl::nullopt); + + // Congested frame... + estimator_.UpdateEstimate(-10 * gen.Delay(), 0.1 * gen.FrameSize()); + TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt); + + // ...impacts the estimate. + EXPECT_GT(outlier_jitter.ms(), steady_state_jitter.ms()); +} + +TEST_F(JitterEstimatorTest, EmptyFieldTrialsParsesToUnsetConfig) { + JitterEstimator::Config config = estimator_.GetConfigForTest(); + EXPECT_FALSE(config.avg_frame_size_median); + EXPECT_FALSE(config.max_frame_size_percentile.has_value()); + EXPECT_FALSE(config.frame_size_window.has_value()); + EXPECT_FALSE(config.num_stddev_delay_clamp.has_value()); + EXPECT_FALSE(config.num_stddev_delay_outlier.has_value()); + EXPECT_FALSE(config.num_stddev_size_outlier.has_value()); + EXPECT_FALSE(config.congestion_rejection_factor.has_value()); + EXPECT_TRUE(config.estimate_noise_when_congested); +} + +class FieldTrialsOverriddenJitterEstimatorTest : public JitterEstimatorTest { + protected: + FieldTrialsOverriddenJitterEstimatorTest() + : JitterEstimatorTest( + "WebRTC-JitterEstimatorConfig/" + "avg_frame_size_median:true," + "max_frame_size_percentile:0.9," + "frame_size_window:30," + "num_stddev_delay_clamp:1.1," + "num_stddev_delay_outlier:2," + "num_stddev_size_outlier:3.1," + "congestion_rejection_factor:-1.55," + "estimate_noise_when_congested:false/") {} + ~FieldTrialsOverriddenJitterEstimatorTest() {} +}; + +TEST_F(FieldTrialsOverriddenJitterEstimatorTest, FieldTrialsParsesCorrectly) { + JitterEstimator::Config config = estimator_.GetConfigForTest(); + EXPECT_TRUE(config.avg_frame_size_median); + EXPECT_EQ(*config.max_frame_size_percentile, 0.9); + EXPECT_EQ(*config.frame_size_window, 30); + EXPECT_EQ(*config.num_stddev_delay_clamp, 1.1); + EXPECT_EQ(*config.num_stddev_delay_outlier, 2.0); + EXPECT_EQ(*config.num_stddev_size_outlier, 3.1); + EXPECT_EQ(*config.congestion_rejection_factor, -1.55); + EXPECT_FALSE(config.estimate_noise_when_congested); +} + +TEST_F(FieldTrialsOverriddenJitterEstimatorTest, + DelayOutlierIsRejectedAndMaintainsJitterEstimate) { + ValueGenerator gen(10); + + // Steady state. + Run(/*duration_s=*/60, /*framerate_fps=*/30, gen); + TimeDelta steady_state_jitter = + estimator_.GetJitterEstimate(0, absl::nullopt); + + // A single outlier frame size... + estimator_.UpdateEstimate(10 * gen.Delay(), gen.FrameSize()); + TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt); + + // ...does not change the estimate. + EXPECT_EQ(outlier_jitter.ms(), steady_state_jitter.ms()); +} + +// The field trial is configured to be robust against the `(1 - 0.9) = 10%` +// largest frames over a window of length `30`. +TEST_F(FieldTrialsOverriddenJitterEstimatorTest, + Four2xFrameSizesImpactJitterEstimate) { + ValueGenerator gen(10); + + // Steady state. + Run(/*duration_s=*/60, /*framerate_fps=*/30, gen); + TimeDelta steady_state_jitter = + estimator_.GetJitterEstimate(0, absl::nullopt); + + // Three outlier frames do not impact the jitter estimate. + for (int i = 0; i < 3; ++i) { + estimator_.UpdateEstimate(gen.Delay(), 2 * gen.FrameSize()); + } + TimeDelta outlier_jitter_3x = estimator_.GetJitterEstimate(0, absl::nullopt); + EXPECT_EQ(outlier_jitter_3x.ms(), steady_state_jitter.ms()); + + // Four outlier frames do impact the jitter estimate. + estimator_.UpdateEstimate(gen.Delay(), 2 * gen.FrameSize()); + TimeDelta outlier_jitter_4x = estimator_.GetJitterEstimate(0, absl::nullopt); + EXPECT_GT(outlier_jitter_4x.ms(), outlier_jitter_3x.ms()); +} + +// When so configured, congested frames are NOT used when calculating the +// noise variance, meaning that they will NOT impact the final jitter estimate. +TEST_F(FieldTrialsOverriddenJitterEstimatorTest, + CongestedFrameDoesNotImpactJitterEstimate) { + ValueGenerator gen(10); + + // Steady state. + Run(/*duration_s=*/10, /*framerate_fps=*/30, gen); + TimeDelta steady_state_jitter = + estimator_.GetJitterEstimate(0, absl::nullopt); + + // Congested frame... + estimator_.UpdateEstimate(-10 * gen.Delay(), 0.1 * gen.FrameSize()); + TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt); + + // ...does not impact the estimate. + EXPECT_EQ(outlier_jitter.ms(), steady_state_jitter.ms()); +} + +class MisconfiguredFieldTrialsJitterEstimatorTest : public JitterEstimatorTest { + protected: + MisconfiguredFieldTrialsJitterEstimatorTest() + : JitterEstimatorTest( + "WebRTC-JitterEstimatorConfig/" + "max_frame_size_percentile:-0.9," + "frame_size_window:-1," + "num_stddev_delay_clamp:-1.9," + "num_stddev_delay_outlier:-2," + "num_stddev_size_outlier:-23.1/") {} + ~MisconfiguredFieldTrialsJitterEstimatorTest() {} +}; + +TEST_F(MisconfiguredFieldTrialsJitterEstimatorTest, FieldTrialsAreValidated) { + JitterEstimator::Config config = estimator_.GetConfigForTest(); + EXPECT_EQ(*config.max_frame_size_percentile, 0.0); + EXPECT_EQ(*config.frame_size_window, 1); + EXPECT_EQ(*config.num_stddev_delay_clamp, 0.0); + EXPECT_EQ(*config.num_stddev_delay_outlier, 0.0); + EXPECT_EQ(*config.num_stddev_size_outlier, 0.0); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc new file mode 100644 index 0000000000..6962224d61 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/rtt_filter.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +#include <algorithm> + +#include "absl/algorithm/container.h" +#include "absl/container/inlined_vector.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +namespace { + +constexpr TimeDelta kMaxRtt = TimeDelta::Seconds(3); +constexpr uint32_t kFilterFactorMax = 35; +constexpr double kJumpStddev = 2.5; +constexpr double kDriftStdDev = 3.5; + +} // namespace + +RttFilter::RttFilter() + : avg_rtt_(TimeDelta::Zero()), + var_rtt_(0), + max_rtt_(TimeDelta::Zero()), + jump_buf_(kMaxDriftJumpCount, TimeDelta::Zero()), + drift_buf_(kMaxDriftJumpCount, TimeDelta::Zero()) { + Reset(); +} + +void RttFilter::Reset() { + got_non_zero_update_ = false; + avg_rtt_ = TimeDelta::Zero(); + var_rtt_ = 0; + max_rtt_ = TimeDelta::Zero(); + filt_fact_count_ = 1; + absl::c_fill(jump_buf_, TimeDelta::Zero()); + absl::c_fill(drift_buf_, TimeDelta::Zero()); +} + +void RttFilter::Update(TimeDelta rtt) { + if (!got_non_zero_update_) { + if (rtt.IsZero()) { + return; + } + got_non_zero_update_ = true; + } + + // Sanity check + if (rtt > kMaxRtt) { + rtt = kMaxRtt; + } + + double filt_factor = 0; + if (filt_fact_count_ > 1) { + filt_factor = static_cast<double>(filt_fact_count_ - 1) / filt_fact_count_; + } + filt_fact_count_++; + if (filt_fact_count_ > kFilterFactorMax) { + // This prevents filt_factor from going above + // (_filt_fact_max - 1) / filt_fact_max_, + // e.g., filt_fact_max_ = 50 => filt_factor = 49/50 = 0.98 + filt_fact_count_ = kFilterFactorMax; + } + TimeDelta old_avg = avg_rtt_; + int64_t old_var = var_rtt_; + avg_rtt_ = filt_factor * avg_rtt_ + (1 - filt_factor) * rtt; + int64_t delta_ms = (rtt - avg_rtt_).ms(); + var_rtt_ = filt_factor * var_rtt_ + (1 - filt_factor) * (delta_ms * delta_ms); + max_rtt_ = std::max(rtt, max_rtt_); + if (!JumpDetection(rtt) || !DriftDetection(rtt)) { + // In some cases we don't want to update the statistics + avg_rtt_ = old_avg; + var_rtt_ = old_var; + } +} + +bool RttFilter::JumpDetection(TimeDelta rtt) { + TimeDelta diff_from_avg = avg_rtt_ - rtt; + // Unit of var_rtt_ is ms^2. + TimeDelta jump_threshold = TimeDelta::Millis(kJumpStddev * sqrt(var_rtt_)); + if (diff_from_avg.Abs() > jump_threshold) { + bool positive_diff = diff_from_avg >= TimeDelta::Zero(); + if (!jump_buf_.empty() && positive_diff != last_jump_positive_) { + // Since the signs differ the samples currently + // in the buffer is useless as they represent a + // jump in a different direction. + jump_buf_.clear(); + } + if (jump_buf_.size() < kMaxDriftJumpCount) { + // Update the buffer used for the short time statistics. + // The sign of the diff is used for updating the counter since + // we want to use the same buffer for keeping track of when + // the RTT jumps down and up. + jump_buf_.push_back(rtt); + last_jump_positive_ = positive_diff; + } + if (jump_buf_.size() >= kMaxDriftJumpCount) { + // Detected an RTT jump + ShortRttFilter(jump_buf_); + filt_fact_count_ = kMaxDriftJumpCount + 1; + jump_buf_.clear(); + } else { + return false; + } + } else { + jump_buf_.clear(); + } + return true; +} + +bool RttFilter::DriftDetection(TimeDelta rtt) { + // Unit of sqrt of var_rtt_ is ms. + TimeDelta drift_threshold = TimeDelta::Millis(kDriftStdDev * sqrt(var_rtt_)); + if (max_rtt_ - avg_rtt_ > drift_threshold) { + if (drift_buf_.size() < kMaxDriftJumpCount) { + // Update the buffer used for the short time statistics. + drift_buf_.push_back(rtt); + } + if (drift_buf_.size() >= kMaxDriftJumpCount) { + // Detected an RTT drift + ShortRttFilter(drift_buf_); + filt_fact_count_ = kMaxDriftJumpCount + 1; + drift_buf_.clear(); + } + } else { + drift_buf_.clear(); + } + return true; +} + +void RttFilter::ShortRttFilter(const BufferList& buf) { + RTC_DCHECK_EQ(buf.size(), kMaxDriftJumpCount); + max_rtt_ = TimeDelta::Zero(); + avg_rtt_ = TimeDelta::Zero(); + for (const TimeDelta& rtt : buf) { + if (rtt > max_rtt_) { + max_rtt_ = rtt; + } + avg_rtt_ += rtt; + } + avg_rtt_ = avg_rtt_ / static_cast<double>(buf.size()); +} + +TimeDelta RttFilter::Rtt() const { + return max_rtt_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h new file mode 100644 index 0000000000..b8700b23ee --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_ +#define MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_ + +#include <stdint.h> + +#include "absl/container/inlined_vector.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +class RttFilter { + public: + RttFilter(); + RttFilter(const RttFilter&) = delete; + RttFilter& operator=(const RttFilter&) = delete; + + // Resets the filter. + void Reset(); + // Updates the filter with a new sample. + void Update(TimeDelta rtt); + // A getter function for the current RTT level. + TimeDelta Rtt() const; + + private: + // The size of the drift and jump memory buffers + // and thus also the detection threshold for these + // detectors in number of samples. + static constexpr int kMaxDriftJumpCount = 5; + using BufferList = absl::InlinedVector<TimeDelta, kMaxDriftJumpCount>; + + // Detects RTT jumps by comparing the difference between + // samples and average to the standard deviation. + // Returns true if the long time statistics should be updated + // and false otherwise + bool JumpDetection(TimeDelta rtt); + + // Detects RTT drifts by comparing the difference between + // max and average to the standard deviation. + // Returns true if the long time statistics should be updated + // and false otherwise + bool DriftDetection(TimeDelta rtt); + + // Computes the short time average and maximum of the vector buf. + void ShortRttFilter(const BufferList& buf); + + bool got_non_zero_update_; + TimeDelta avg_rtt_; + // Variance units are TimeDelta^2. Store as ms^2. + int64_t var_rtt_; + TimeDelta max_rtt_; + uint32_t filt_fact_count_; + bool last_jump_positive_ = false; + BufferList jump_buf_; + BufferList drift_buf_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build new file mode 100644 index 0000000000..488a49d9c2 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build @@ -0,0 +1,221 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("rtt_filter_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc new file mode 100644 index 0000000000..05502e6f5b --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/rtt_filter.h" + +#include "api/units/time_delta.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(RttFilterTest, RttIsCapped) { + RttFilter rtt_filter; + rtt_filter.Update(TimeDelta::Seconds(500)); + + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Seconds(3)); +} + +// If the difference between samples is more than away 2.5 stddev from the mean +// then this is considered a jump. After more than 5 data points at the new +// level, the RTT is reset to the new level. +TEST(RttFilterTest, PositiveJumpDetection) { + RttFilter rtt_filter; + + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + + // Trigger 5 jumps. + rtt_filter.Update(TimeDelta::Millis(1400)); + rtt_filter.Update(TimeDelta::Millis(1500)); + rtt_filter.Update(TimeDelta::Millis(1600)); + rtt_filter.Update(TimeDelta::Millis(1600)); + + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(1600)); + + rtt_filter.Update(TimeDelta::Millis(1600)); + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(1600)); +} + +TEST(RttFilterTest, NegativeJumpDetection) { + RttFilter rtt_filter; + + for (int i = 0; i < 10; ++i) + rtt_filter.Update(TimeDelta::Millis(1500)); + + // Trigger 5 negative data points that jump rtt down. + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + // Before 5 data points at the new level, max RTT is still 1500. + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(1500)); + + rtt_filter.Update(TimeDelta::Millis(300)); + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(300)); +} + +TEST(RttFilterTest, JumpsResetByDirectionShift) { + RttFilter rtt_filter; + for (int i = 0; i < 10; ++i) + rtt_filter.Update(TimeDelta::Millis(1500)); + + // Trigger 4 negative jumps, then a positive one. This resets the jump + // detection. + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(2000)); + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(2000)); + + rtt_filter.Update(TimeDelta::Millis(300)); + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(2000)); +} + +// If the difference between the max and average is more than 3.5 stddevs away +// then a drift is detected, and a short filter is applied to find a new max +// rtt. +TEST(RttFilterTest, DriftDetection) { + RttFilter rtt_filter; + + // Descend RTT by 30ms and settle at 700ms RTT. A drift is detected after rtt + // of 700ms is reported around 50 times for these targets. + constexpr TimeDelta kStartRtt = TimeDelta::Millis(1000); + constexpr TimeDelta kDriftTarget = TimeDelta::Millis(700); + constexpr TimeDelta kDelta = TimeDelta::Millis(30); + for (TimeDelta rtt = kStartRtt; rtt >= kDriftTarget; rtt -= kDelta) + rtt_filter.Update(rtt); + + EXPECT_EQ(rtt_filter.Rtt(), kStartRtt); + + for (int i = 0; i < 50; ++i) + rtt_filter.Update(kDriftTarget); + EXPECT_EQ(rtt_filter.Rtt(), kDriftTarget); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.cc b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.cc new file mode 100644 index 0000000000..dc62ac674a --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.cc @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/timestamp_extrapolator.h" + +#include <algorithm> + +#include "absl/types/optional.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" + +namespace webrtc { + +namespace { + +constexpr double kLambda = 1; +constexpr uint32_t kStartUpFilterDelayInPackets = 2; +constexpr double kAlarmThreshold = 60e3; +// in timestamp ticks, i.e. 15 ms +constexpr double kAccDrift = 6600; +constexpr double kAccMaxError = 7000; +constexpr double kP11 = 1e10; + +} // namespace + +TimestampExtrapolator::TimestampExtrapolator(Timestamp start) + : start_(Timestamp::Zero()), + prev_(Timestamp::Zero()), + packet_count_(0), + detector_accumulator_pos_(0), + detector_accumulator_neg_(0) { + Reset(start); +} + +void TimestampExtrapolator::Reset(Timestamp start) { + start_ = start; + prev_ = start_; + first_unwrapped_timestamp_ = absl::nullopt; + w_[0] = 90.0; + w_[1] = 0; + p_[0][0] = 1; + p_[1][1] = kP11; + p_[0][1] = p_[1][0] = 0; + unwrapper_ = RtpTimestampUnwrapper(); + packet_count_ = 0; + detector_accumulator_pos_ = 0; + detector_accumulator_neg_ = 0; +} + +void TimestampExtrapolator::Update(Timestamp now, uint32_t ts90khz) { + if (now - prev_ > TimeDelta::Seconds(10)) { + // Ten seconds without a complete frame. + // Reset the extrapolator + Reset(now); + } else { + prev_ = now; + } + + // Remove offset to prevent badly scaled matrices + const TimeDelta offset = now - start_; + double t_ms = offset.ms(); + + int64_t unwrapped_ts90khz = unwrapper_.Unwrap(ts90khz); + + if (!first_unwrapped_timestamp_) { + // Make an initial guess of the offset, + // should be almost correct since t_ms - start + // should about zero at this time. + w_[1] = -w_[0] * t_ms; + first_unwrapped_timestamp_ = unwrapped_ts90khz; + } + + double residual = + (static_cast<double>(unwrapped_ts90khz) - *first_unwrapped_timestamp_) - + t_ms * w_[0] - w_[1]; + if (DelayChangeDetection(residual) && + packet_count_ >= kStartUpFilterDelayInPackets) { + // A sudden change of average network delay has been detected. + // Force the filter to adjust its offset parameter by changing + // the offset uncertainty. Don't do this during startup. + p_[1][1] = kP11; + } + + if (prev_unwrapped_timestamp_ && + unwrapped_ts90khz < prev_unwrapped_timestamp_) { + // Drop reordered frames. + return; + } + + // T = [t(k) 1]'; + // that = T'*w; + // K = P*T/(lambda + T'*P*T); + double K[2]; + K[0] = p_[0][0] * t_ms + p_[0][1]; + K[1] = p_[1][0] * t_ms + p_[1][1]; + double TPT = kLambda + t_ms * K[0] + K[1]; + K[0] /= TPT; + K[1] /= TPT; + // w = w + K*(ts(k) - that); + w_[0] = w_[0] + K[0] * residual; + w_[1] = w_[1] + K[1] * residual; + // P = 1/lambda*(P - K*T'*P); + double p00 = + 1 / kLambda * (p_[0][0] - (K[0] * t_ms * p_[0][0] + K[0] * p_[1][0])); + double p01 = + 1 / kLambda * (p_[0][1] - (K[0] * t_ms * p_[0][1] + K[0] * p_[1][1])); + p_[1][0] = + 1 / kLambda * (p_[1][0] - (K[1] * t_ms * p_[0][0] + K[1] * p_[1][0])); + p_[1][1] = + 1 / kLambda * (p_[1][1] - (K[1] * t_ms * p_[0][1] + K[1] * p_[1][1])); + p_[0][0] = p00; + p_[0][1] = p01; + prev_unwrapped_timestamp_ = unwrapped_ts90khz; + if (packet_count_ < kStartUpFilterDelayInPackets) { + packet_count_++; + } +} + +absl::optional<Timestamp> TimestampExtrapolator::ExtrapolateLocalTime( + uint32_t timestamp90khz) const { + int64_t unwrapped_ts90khz = unwrapper_.PeekUnwrap(timestamp90khz); + RTC_DCHECK_GE(unwrapped_ts90khz, 0); + + if (!first_unwrapped_timestamp_) { + return absl::nullopt; + } else if (packet_count_ < kStartUpFilterDelayInPackets) { + constexpr double kRtpTicksPerMs = 90; + TimeDelta diff = TimeDelta::Millis( + (unwrapped_ts90khz - *prev_unwrapped_timestamp_) / kRtpTicksPerMs); + if (diff.ms() < 0) { + RTC_DCHECK_GE(prev_.ms(), -diff.ms()); + } + return prev_ + diff; + } else if (w_[0] < 1e-3) { + return start_; + } else { + double timestampDiff = unwrapped_ts90khz - *first_unwrapped_timestamp_; + auto diff_ms = static_cast<int64_t>((timestampDiff - w_[1]) / w_[0] + 0.5); + if (diff_ms < 0) { + RTC_DCHECK_GE(start_.ms(), -diff_ms); + } + return start_ + TimeDelta::Millis(diff_ms); + } +} + +bool TimestampExtrapolator::DelayChangeDetection(double error) { + // CUSUM detection of sudden delay changes + error = (error > 0) ? std::min(error, kAccMaxError) + : std::max(error, -kAccMaxError); + detector_accumulator_pos_ = + std::max(detector_accumulator_pos_ + error - kAccDrift, double{0}); + detector_accumulator_neg_ = + std::min(detector_accumulator_neg_ + error + kAccDrift, double{0}); + if (detector_accumulator_pos_ > kAlarmThreshold || + detector_accumulator_neg_ < -kAlarmThreshold) { + // Alarm + detector_accumulator_pos_ = detector_accumulator_neg_ = 0; + return true; + } + return false; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.h b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.h new file mode 100644 index 0000000000..6a9763943e --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_ +#define MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_ + +#include <stdint.h> + +#include "absl/types/optional.h" +#include "api/units/timestamp.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" + +namespace webrtc { + +// Not thread safe. +class TimestampExtrapolator { + public: + explicit TimestampExtrapolator(Timestamp start); + void Update(Timestamp now, uint32_t ts90khz); + absl::optional<Timestamp> ExtrapolateLocalTime(uint32_t timestamp90khz) const; + void Reset(Timestamp start); + + private: + void CheckForWrapArounds(uint32_t ts90khz); + bool DelayChangeDetection(double error); + + double w_[2]; + double p_[2][2]; + Timestamp start_; + Timestamp prev_; + absl::optional<int64_t> first_unwrapped_timestamp_; + RtpTimestampUnwrapper unwrapper_; + absl::optional<int64_t> prev_unwrapped_timestamp_; + uint32_t packet_count_; + double detector_accumulator_pos_; + double detector_accumulator_neg_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_gn/moz.build new file mode 100644 index 0000000000..4cd085dd09 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_gn/moz.build @@ -0,0 +1,221 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("timestamp_extrapolator_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_unittest.cc new file mode 100644 index 0000000000..0b5fd74a8e --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_unittest.cc @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/timestamp_extrapolator.h" + +#include <stdint.h> + +#include <limits> + +#include "absl/types/optional.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::Eq; +using ::testing::Optional; + +namespace { + +constexpr Frequency kRtpHz = Frequency::KiloHertz(90); +constexpr Frequency k25Fps = Frequency::Hertz(25); +constexpr TimeDelta k25FpsDelay = 1 / k25Fps; + +} // namespace + +TEST(TimestampExtrapolatorTest, ExtrapolationOccursAfter2Packets) { + SimulatedClock clock(Timestamp::Millis(1337)); + TimestampExtrapolator ts_extrapolator(clock.CurrentTime()); + + // No packets so no timestamp. + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(90000), Eq(absl::nullopt)); + + uint32_t rtp = 90000; + clock.AdvanceTime(k25FpsDelay); + // First result is a bit confusing since it is based off the "start" time, + // which is arbitrary. + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); + + rtp += kRtpHz / k25Fps; + clock.AdvanceTime(k25FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp + 90000), + Optional(clock.CurrentTime() + TimeDelta::Seconds(1))); +} + +TEST(TimestampExtrapolatorTest, ResetsAfter10SecondPause) { + SimulatedClock clock(Timestamp::Millis(1337)); + TimestampExtrapolator ts_extrapolator(clock.CurrentTime()); + + uint32_t rtp = 90000; + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); + + rtp += kRtpHz / k25Fps; + clock.AdvanceTime(k25FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); + + rtp += 10 * kRtpHz.hertz(); + clock.AdvanceTime(TimeDelta::Seconds(10) + TimeDelta::Micros(1)); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); +} + +TEST(TimestampExtrapolatorTest, TimestampExtrapolatesMultipleRtpWrapArounds) { + SimulatedClock clock(Timestamp::Millis(1337)); + TimestampExtrapolator ts_extrapolator(clock.CurrentTime()); + + uint32_t rtp = std::numeric_limits<uint32_t>::max(); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); + + // One overflow. Static cast to avoid undefined behaviour with +=. + rtp += static_cast<uint32_t>(kRtpHz / k25Fps); + clock.AdvanceTime(k25FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); + + // Assert that extrapolation works across the boundary as expected. + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp + 90000), + Optional(clock.CurrentTime() + TimeDelta::Seconds(1))); + // This is not quite 1s since the math always rounds up. + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp - 90000), + Optional(clock.CurrentTime() - TimeDelta::Millis(999))); + + // In order to avoid a wrap arounds reset, add a packet every 10s until we + // overflow twice. + constexpr TimeDelta kRtpOverflowDelay = + std::numeric_limits<uint32_t>::max() / kRtpHz; + const Timestamp overflow_time = clock.CurrentTime() + kRtpOverflowDelay * 2; + + while (clock.CurrentTime() < overflow_time) { + clock.AdvanceTime(TimeDelta::Seconds(10)); + // Static-cast before += to avoid undefined behaviour of overflow. + rtp += static_cast<uint32_t>(kRtpHz * TimeDelta::Seconds(10)); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); + } +} + +TEST(TimestampExtrapolatorTest, NegativeRtpTimestampWrapAround) { + SimulatedClock clock(Timestamp::Millis(1337)); + TimestampExtrapolator ts_extrapolator(clock.CurrentTime()); + uint32_t rtp = 0; + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); + // Go backwards! + rtp -= kRtpHz.hertz(); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime() - TimeDelta::Seconds(1))); +} + +TEST(TimestampExtrapolatorTest, Slow90KHzClock) { + // This simulates a slow camera, which produces frames at 24Hz instead of + // 25Hz. The extrapolator should be able to resolve this with enough data. + SimulatedClock clock(Timestamp::Millis(1337)); + TimestampExtrapolator ts_extrapolator(clock.CurrentTime()); + + constexpr TimeDelta k24FpsDelay = 1 / Frequency::Hertz(24); + uint32_t rtp = 90000; + ts_extrapolator.Update(clock.CurrentTime(), rtp); + + // Slow camera will increment RTP at 25 FPS rate even though its producing at + // 24 FPS. After 25 frames the extrapolator should settle at this rate. + for (int i = 0; i < 25; ++i) { + rtp += kRtpHz / k25Fps; + clock.AdvanceTime(k24FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + } + + // The camera would normally produce 25 frames in 90K ticks, but is slow + // so takes 1s + k24FpsDelay for 90K ticks. + constexpr Frequency kSlowRtpHz = 90000 / (25 * k24FpsDelay); + // The extrapolator will be predicting that time at millisecond precision. + auto ts = ts_extrapolator.ExtrapolateLocalTime(rtp + kSlowRtpHz.hertz()); + ASSERT_TRUE(ts.has_value()); + EXPECT_EQ(ts->ms(), clock.TimeInMilliseconds() + 1000); +} + +TEST(TimestampExtrapolatorTest, Fast90KHzClock) { + // This simulates a fast camera, which produces frames at 26Hz instead of + // 25Hz. The extrapolator should be able to resolve this with enough data. + SimulatedClock clock(Timestamp::Millis(1337)); + TimestampExtrapolator ts_extrapolator(clock.CurrentTime()); + + constexpr TimeDelta k26FpsDelay = 1 / Frequency::Hertz(26); + uint32_t rtp = 90000; + ts_extrapolator.Update(clock.CurrentTime(), rtp); + + // Fast camera will increment RTP at 25 FPS rate even though its producing at + // 26 FPS. After 25 frames the extrapolator should settle at this rate. + for (int i = 0; i < 25; ++i) { + rtp += kRtpHz / k25Fps; + clock.AdvanceTime(k26FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + } + + // The camera would normally produce 25 frames in 90K ticks, but is slow + // so takes 1s + k24FpsDelay for 90K ticks. + constexpr Frequency kSlowRtpHz = 90000 / (25 * k26FpsDelay); + // The extrapolator will be predicting that time at millisecond precision. + auto ts = ts_extrapolator.ExtrapolateLocalTime(rtp + kSlowRtpHz.hertz()); + ASSERT_TRUE(ts.has_value()); + EXPECT_EQ(ts->ms(), clock.TimeInMilliseconds() + 1000); +} + +TEST(TimestampExtrapolatorTest, TimestampJump) { + // This simulates a jump in RTP timestamp, which could occur if a camera was + // swapped for example. + SimulatedClock clock(Timestamp::Millis(1337)); + TimestampExtrapolator ts_extrapolator(clock.CurrentTime()); + + uint32_t rtp = 90000; + clock.AdvanceTime(k25FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + rtp += kRtpHz / k25Fps; + clock.AdvanceTime(k25FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + rtp += kRtpHz / k25Fps; + clock.AdvanceTime(k25FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp), + Optional(clock.CurrentTime())); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp + 90000), + Optional(clock.CurrentTime() + TimeDelta::Seconds(1))); + + // Jump RTP. + uint32_t new_rtp = 1337 * 90000; + clock.AdvanceTime(k25FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), new_rtp); + new_rtp += kRtpHz / k25Fps; + clock.AdvanceTime(k25FpsDelay); + ts_extrapolator.Update(clock.CurrentTime(), new_rtp); + EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(new_rtp), + Optional(clock.CurrentTime())); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing.cc b/third_party/libwebrtc/modules/video_coding/timing/timing.cc new file mode 100644 index 0000000000..0b61d5a35e --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timing.cc @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/timing.h" + +#include <algorithm> + +#include "api/units/time_delta.h" +#include "modules/video_coding/timing/timestamp_extrapolator.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace { + +// Default pacing that is used for the low-latency renderer path. +constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8); +constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold = + TimeDelta::Millis(500); + +void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) { + if (min_delay > max_delay) { + RTC_LOG(LS_ERROR) + << "Playout delays set incorrectly: min playout delay (" << min_delay + << ") > max playout delay (" << max_delay + << "). This is undefined behaviour. Application writers should " + "ensure that the min delay is always less than or equals max " + "delay. If trying to use the playout delay header extensions " + "described in " + "https://webrtc.googlesource.com/src/+/refs/heads/main/docs/" + "native-code/rtp-hdrext/playout-delay/, be careful that a playout " + "delay hint or A/V sync settings may have caused this conflict."; + } +} + +} // namespace + +VCMTiming::VCMTiming(Clock* clock, const FieldTrialsView& field_trials) + : clock_(clock), + ts_extrapolator_( + std::make_unique<TimestampExtrapolator>(clock_->CurrentTime())), + codec_timer_(std::make_unique<CodecTimer>()), + render_delay_(kDefaultRenderDelay), + min_playout_delay_(TimeDelta::Zero()), + max_playout_delay_(TimeDelta::Seconds(10)), + jitter_delay_(TimeDelta::Zero()), + current_delay_(TimeDelta::Zero()), + prev_frame_timestamp_(0), + num_decoded_frames_(0), + zero_playout_delay_min_pacing_("min_pacing", + kZeroPlayoutDelayDefaultMinPacing), + last_decode_scheduled_(Timestamp::Zero()) { + ParseFieldTrial({&zero_playout_delay_min_pacing_}, + field_trials.Lookup("WebRTC-ZeroPlayoutDelay")); +} + +void VCMTiming::Reset() { + MutexLock lock(&mutex_); + ts_extrapolator_->Reset(clock_->CurrentTime()); + codec_timer_ = std::make_unique<CodecTimer>(); + render_delay_ = kDefaultRenderDelay; + min_playout_delay_ = TimeDelta::Zero(); + jitter_delay_ = TimeDelta::Zero(); + current_delay_ = TimeDelta::Zero(); + prev_frame_timestamp_ = 0; +} + +void VCMTiming::set_render_delay(TimeDelta render_delay) { + MutexLock lock(&mutex_); + render_delay_ = render_delay; +} + +TimeDelta VCMTiming::min_playout_delay() const { + MutexLock lock(&mutex_); + return min_playout_delay_; +} + +void VCMTiming::set_min_playout_delay(TimeDelta min_playout_delay) { + MutexLock lock(&mutex_); + if (min_playout_delay_ != min_playout_delay) { + CheckDelaysValid(min_playout_delay, max_playout_delay_); + min_playout_delay_ = min_playout_delay; + } +} + +void VCMTiming::set_max_playout_delay(TimeDelta max_playout_delay) { + MutexLock lock(&mutex_); + if (max_playout_delay_ != max_playout_delay) { + CheckDelaysValid(min_playout_delay_, max_playout_delay); + max_playout_delay_ = max_playout_delay; + } +} + +void VCMTiming::SetJitterDelay(TimeDelta jitter_delay) { + MutexLock lock(&mutex_); + if (jitter_delay != jitter_delay_) { + jitter_delay_ = jitter_delay; + // When in initial state, set current delay to minimum delay. + if (current_delay_.IsZero()) { + current_delay_ = jitter_delay_; + } + } +} + +void VCMTiming::UpdateCurrentDelay(uint32_t frame_timestamp) { + MutexLock lock(&mutex_); + TimeDelta target_delay = TargetDelayInternal(); + + if (current_delay_.IsZero()) { + // Not initialized, set current delay to target. + current_delay_ = target_delay; + } else if (target_delay != current_delay_) { + TimeDelta delay_diff = target_delay - current_delay_; + // Never change the delay with more than 100 ms every second. If we're + // changing the delay in too large steps we will get noticeable freezes. By + // limiting the change we can increase the delay in smaller steps, which + // will be experienced as the video is played in slow motion. When lowering + // the delay the video will be played at a faster pace. + TimeDelta max_change = TimeDelta::Zero(); + if (frame_timestamp < 0x0000ffff && prev_frame_timestamp_ > 0xffff0000) { + // wrap + max_change = + TimeDelta::Millis(kDelayMaxChangeMsPerS * + (frame_timestamp + (static_cast<int64_t>(1) << 32) - + prev_frame_timestamp_) / + 90000); + } else { + max_change = + TimeDelta::Millis(kDelayMaxChangeMsPerS * + (frame_timestamp - prev_frame_timestamp_) / 90000); + } + + if (max_change <= TimeDelta::Zero()) { + // Any changes less than 1 ms are truncated and will be postponed. + // Negative change will be due to reordering and should be ignored. + return; + } + delay_diff = std::max(delay_diff, -max_change); + delay_diff = std::min(delay_diff, max_change); + + current_delay_ = current_delay_ + delay_diff; + } + prev_frame_timestamp_ = frame_timestamp; +} + +void VCMTiming::UpdateCurrentDelay(Timestamp render_time, + Timestamp actual_decode_time) { + MutexLock lock(&mutex_); + TimeDelta target_delay = TargetDelayInternal(); + TimeDelta delayed = + (actual_decode_time - render_time) + RequiredDecodeTime() + render_delay_; + + // Only consider `delayed` as negative by more than a few microseconds. + if (delayed.ms() < 0) { + return; + } + if (current_delay_ + delayed <= target_delay) { + current_delay_ += delayed; + } else { + current_delay_ = target_delay; + } +} + +void VCMTiming::StopDecodeTimer(TimeDelta decode_time, Timestamp now) { + MutexLock lock(&mutex_); + codec_timer_->AddTiming(decode_time.ms(), now.ms()); + RTC_DCHECK_GE(decode_time, TimeDelta::Zero()); + ++num_decoded_frames_; +} + +void VCMTiming::IncomingTimestamp(uint32_t rtp_timestamp, Timestamp now) { + MutexLock lock(&mutex_); + ts_extrapolator_->Update(now, rtp_timestamp); +} + +Timestamp VCMTiming::RenderTime(uint32_t frame_timestamp, Timestamp now) const { + MutexLock lock(&mutex_); + return RenderTimeInternal(frame_timestamp, now); +} + +void VCMTiming::SetLastDecodeScheduledTimestamp( + Timestamp last_decode_scheduled) { + MutexLock lock(&mutex_); + last_decode_scheduled_ = last_decode_scheduled; +} + +Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp, + Timestamp now) const { + if (UseLowLatencyRendering()) { + // Render as soon as possible or with low-latency renderer algorithm. + return Timestamp::Zero(); + } + // Note that TimestampExtrapolator::ExtrapolateLocalTime is not a const + // method; it mutates the object's wraparound state. + Timestamp estimated_complete_time = + ts_extrapolator_->ExtrapolateLocalTime(frame_timestamp).value_or(now); + + // Make sure the actual delay stays in the range of `min_playout_delay_` + // and `max_playout_delay_`. + TimeDelta actual_delay = + current_delay_.Clamped(min_playout_delay_, max_playout_delay_); + return estimated_complete_time + actual_delay; +} + +TimeDelta VCMTiming::RequiredDecodeTime() const { + const int decode_time_ms = codec_timer_->RequiredDecodeTimeMs(); + RTC_DCHECK_GE(decode_time_ms, 0); + return TimeDelta::Millis(decode_time_ms); +} + +TimeDelta VCMTiming::MaxWaitingTime(Timestamp render_time, + Timestamp now, + bool too_many_frames_queued) const { + MutexLock lock(&mutex_); + + if (render_time.IsZero() && zero_playout_delay_min_pacing_->us() > 0 && + min_playout_delay_.IsZero() && max_playout_delay_ > TimeDelta::Zero()) { + // `render_time` == 0 indicates that the frame should be decoded and + // rendered as soon as possible. However, the decoder can be choked if too + // many frames are sent at once. Therefore, limit the interframe delay to + // |zero_playout_delay_min_pacing_| unless too many frames are queued in + // which case the frames are sent to the decoder at once. + if (too_many_frames_queued) { + return TimeDelta::Zero(); + } + Timestamp earliest_next_decode_start_time = + last_decode_scheduled_ + zero_playout_delay_min_pacing_; + TimeDelta max_wait_time = now >= earliest_next_decode_start_time + ? TimeDelta::Zero() + : earliest_next_decode_start_time - now; + return max_wait_time; + } + return render_time - now - RequiredDecodeTime() - render_delay_; +} + +TimeDelta VCMTiming::TargetVideoDelay() const { + MutexLock lock(&mutex_); + return TargetDelayInternal(); +} + +TimeDelta VCMTiming::TargetDelayInternal() const { + return std::max(min_playout_delay_, + jitter_delay_ + RequiredDecodeTime() + render_delay_); +} + +VideoFrame::RenderParameters VCMTiming::RenderParameters() const { + MutexLock lock(&mutex_); + return {.use_low_latency_rendering = UseLowLatencyRendering(), + .max_composition_delay_in_frames = max_composition_delay_in_frames_}; +} + +bool VCMTiming::UseLowLatencyRendering() const { + // min_playout_delay_==0, + // max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates + // that the low-latency path should be used, which means that frames should be + // decoded and rendered as soon as possible. + return min_playout_delay_.IsZero() && + max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold; +} + +VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const { + MutexLock lock(&mutex_); + return VideoDelayTimings{.max_decode_duration = RequiredDecodeTime(), + .current_delay = current_delay_, + .target_delay = TargetDelayInternal(), + .jitter_buffer_delay = jitter_delay_, + .min_playout_delay = min_playout_delay_, + .max_playout_delay = max_playout_delay_, + .render_delay = render_delay_, + .num_decoded_frames = num_decoded_frames_}; +} + +void VCMTiming::SetTimingFrameInfo(const TimingFrameInfo& info) { + MutexLock lock(&mutex_); + timing_frame_info_.emplace(info); +} + +absl::optional<TimingFrameInfo> VCMTiming::GetTimingFrameInfo() { + MutexLock lock(&mutex_); + return timing_frame_info_; +} + +void VCMTiming::SetMaxCompositionDelayInFrames( + absl::optional<int> max_composition_delay_in_frames) { + MutexLock lock(&mutex_); + max_composition_delay_in_frames_ = max_composition_delay_in_frames; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing.h b/third_party/libwebrtc/modules/video_coding/timing/timing.h new file mode 100644 index 0000000000..727527f009 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timing.h @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_TIMING_H_ +#define MODULES_VIDEO_CODING_TIMING_TIMING_H_ + +#include <memory> + +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/units/time_delta.h" +#include "api/video/video_frame.h" +#include "api/video/video_timing.h" +#include "modules/video_coding/timing/codec_timer.h" +#include "modules/video_coding/timing/timestamp_extrapolator.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +class VCMTiming { + public: + static constexpr auto kDefaultRenderDelay = TimeDelta::Millis(10); + static constexpr auto kDelayMaxChangeMsPerS = 100; + + VCMTiming(Clock* clock, const FieldTrialsView& field_trials); + virtual ~VCMTiming() = default; + + // Resets the timing to the initial state. + void Reset(); + + // Set the amount of time needed to render an image. Defaults to 10 ms. + void set_render_delay(TimeDelta render_delay); + + // Set the minimum time the video must be delayed on the receiver to + // get the desired jitter buffer level. + void SetJitterDelay(TimeDelta required_delay); + + // Set/get the minimum playout delay from capture to render. + TimeDelta min_playout_delay() const; + void set_min_playout_delay(TimeDelta min_playout_delay); + + // Set/get the maximum playout delay from capture to render in ms. + void set_max_playout_delay(TimeDelta max_playout_delay); + + // Increases or decreases the current delay to get closer to the target delay. + // Calculates how long it has been since the previous call to this function, + // and increases/decreases the delay in proportion to the time difference. + void UpdateCurrentDelay(uint32_t frame_timestamp); + + // Increases or decreases the current delay to get closer to the target delay. + // Given the actual decode time in ms and the render time in ms for a frame, + // this function calculates how late the frame is and increases the delay + // accordingly. + void UpdateCurrentDelay(Timestamp render_time, Timestamp actual_decode_time); + + // Stops the decoder timer, should be called when the decoder returns a frame + // or when the decoded frame callback is called. + void StopDecodeTimer(TimeDelta decode_time, Timestamp now); + + // Used to report that a frame is passed to decoding. Updates the timestamp + // filter which is used to map between timestamps and receiver system time. + virtual void IncomingTimestamp(uint32_t rtp_timestamp, + Timestamp last_packet_time); + + // Returns the receiver system time when the frame with timestamp + // `frame_timestamp` should be rendered, assuming that the system time + // currently is `now`. + virtual Timestamp RenderTime(uint32_t frame_timestamp, Timestamp now) const; + + // Returns the maximum time in ms that we can wait for a frame to become + // complete before we must pass it to the decoder. render_time==0 indicates + // that the frames should be processed as quickly as possible, with possibly + // only a small delay added to make sure that the decoder is not overloaded. + // In this case, the parameter too_many_frames_queued is used to signal that + // the decode queue is full and that the frame should be decoded as soon as + // possible. + virtual TimeDelta MaxWaitingTime(Timestamp render_time, + Timestamp now, + bool too_many_frames_queued) const; + + // Returns the current target delay which is required delay + decode time + + // render delay. + TimeDelta TargetVideoDelay() const; + + // Return current timing information. Returns true if the first frame has been + // decoded, false otherwise. + struct VideoDelayTimings { + TimeDelta max_decode_duration; + TimeDelta current_delay; + TimeDelta target_delay; + TimeDelta jitter_buffer_delay; + TimeDelta min_playout_delay; + TimeDelta max_playout_delay; + TimeDelta render_delay; + size_t num_decoded_frames; + }; + VideoDelayTimings GetTimings() const; + + void SetTimingFrameInfo(const TimingFrameInfo& info); + absl::optional<TimingFrameInfo> GetTimingFrameInfo(); + + void SetMaxCompositionDelayInFrames( + absl::optional<int> max_composition_delay_in_frames); + + VideoFrame::RenderParameters RenderParameters() const; + + // Updates the last time a frame was scheduled for decoding. + void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled); + + protected: + TimeDelta RequiredDecodeTime() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + private: + mutable Mutex mutex_; + Clock* const clock_; + const std::unique_ptr<TimestampExtrapolator> ts_extrapolator_ + RTC_PT_GUARDED_BY(mutex_); + std::unique_ptr<CodecTimer> codec_timer_ RTC_GUARDED_BY(mutex_) + RTC_PT_GUARDED_BY(mutex_); + TimeDelta render_delay_ RTC_GUARDED_BY(mutex_); + // Best-effort playout delay range for frames from capture to render. + // The receiver tries to keep the delay between `min_playout_delay_ms_` + // and `max_playout_delay_ms_` taking the network jitter into account. + // A special case is where min_playout_delay_ms_ = max_playout_delay_ms_ = 0, + // in which case the receiver tries to play the frames as they arrive. + TimeDelta min_playout_delay_ RTC_GUARDED_BY(mutex_); + TimeDelta max_playout_delay_ RTC_GUARDED_BY(mutex_); + TimeDelta jitter_delay_ RTC_GUARDED_BY(mutex_); + TimeDelta current_delay_ RTC_GUARDED_BY(mutex_); + uint32_t prev_frame_timestamp_ RTC_GUARDED_BY(mutex_); + absl::optional<TimingFrameInfo> timing_frame_info_ RTC_GUARDED_BY(mutex_); + size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_); + absl::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_); + // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing + // determines the minimum delay between frames scheduled for decoding that is + // used when min playout delay=0 and max playout delay>=0. + FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_ + RTC_GUARDED_BY(mutex_); + // Timestamp at which the last frame was scheduled to be sent to the decoder. + // Used only when the RTP header extension playout delay is set to min=0 ms + // which is indicated by a render time set to 0. + Timestamp last_decode_scheduled_ RTC_GUARDED_BY(mutex_); +}; +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_TIMING_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build new file mode 100644 index 0000000000..8ab43fb748 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/timing.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("timing_module_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc new file mode 100644 index 0000000000..8633c0de39 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc @@ -0,0 +1,339 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/timing.h" + +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" +#include "test/scoped_key_value_config.h" + +namespace webrtc { +namespace { + +constexpr Frequency k25Fps = Frequency::Hertz(25); +constexpr Frequency k90kHz = Frequency::KiloHertz(90); + +} // namespace + +TEST(ReceiverTimingTest, JitterDelay) { + test::ScopedKeyValueConfig field_trials; + SimulatedClock clock(0); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + + uint32_t timestamp = 0; + timing.UpdateCurrentDelay(timestamp); + + timing.Reset(); + + timing.IncomingTimestamp(timestamp, clock.CurrentTime()); + TimeDelta jitter_delay = TimeDelta::Millis(20); + timing.SetJitterDelay(jitter_delay); + timing.UpdateCurrentDelay(timestamp); + timing.set_render_delay(TimeDelta::Zero()); + auto wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + // First update initializes the render time. Since we have no decode delay + // we get wait_time = renderTime - now - renderDelay = jitter. + EXPECT_EQ(jitter_delay, wait_time); + + jitter_delay += TimeDelta::Millis(VCMTiming::kDelayMaxChangeMsPerS + 10); + timestamp += 90000; + clock.AdvanceTimeMilliseconds(1000); + timing.SetJitterDelay(jitter_delay); + timing.UpdateCurrentDelay(timestamp); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + // Since we gradually increase the delay we only get 100 ms every second. + EXPECT_EQ(jitter_delay - TimeDelta::Millis(10), wait_time); + + timestamp += 90000; + clock.AdvanceTimeMilliseconds(1000); + timing.UpdateCurrentDelay(timestamp); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + EXPECT_EQ(jitter_delay, wait_time); + + // Insert frames without jitter, verify that this gives the exact wait time. + const int kNumFrames = 300; + for (int i = 0; i < kNumFrames; i++) { + clock.AdvanceTime(1 / k25Fps); + timestamp += k90kHz / k25Fps; + timing.IncomingTimestamp(timestamp, clock.CurrentTime()); + } + timing.UpdateCurrentDelay(timestamp); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + EXPECT_EQ(jitter_delay, wait_time); + + // Add decode time estimates for 1 second. + const TimeDelta kDecodeTime = TimeDelta::Millis(10); + for (int i = 0; i < k25Fps.hertz(); i++) { + clock.AdvanceTime(kDecodeTime); + timing.StopDecodeTimer(kDecodeTime, clock.CurrentTime()); + timestamp += k90kHz / k25Fps; + clock.AdvanceTime(1 / k25Fps - kDecodeTime); + timing.IncomingTimestamp(timestamp, clock.CurrentTime()); + } + timing.UpdateCurrentDelay(timestamp); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + EXPECT_EQ(jitter_delay, wait_time); + + const TimeDelta kMinTotalDelay = TimeDelta::Millis(200); + timing.set_min_playout_delay(kMinTotalDelay); + clock.AdvanceTimeMilliseconds(5000); + timestamp += 5 * 90000; + timing.UpdateCurrentDelay(timestamp); + const TimeDelta kRenderDelay = TimeDelta::Millis(10); + timing.set_render_delay(kRenderDelay); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + // We should at least have kMinTotalDelayMs - decodeTime (10) - renderTime + // (10) to wait. + EXPECT_EQ(kMinTotalDelay - kDecodeTime - kRenderDelay, wait_time); + // The total video delay should be equal to the min total delay. + EXPECT_EQ(kMinTotalDelay, timing.TargetVideoDelay()); + + // Reset playout delay. + timing.set_min_playout_delay(TimeDelta::Zero()); + clock.AdvanceTimeMilliseconds(5000); + timestamp += 5 * 90000; + timing.UpdateCurrentDelay(timestamp); +} + +TEST(ReceiverTimingTest, TimestampWrapAround) { + constexpr auto kStartTime = Timestamp::Millis(1337); + test::ScopedKeyValueConfig field_trials; + SimulatedClock clock(kStartTime); + VCMTiming timing(&clock, field_trials); + + // Provoke a wrap-around. The fifth frame will have wrapped at 25 fps. + constexpr uint32_t kRtpTicksPerFrame = k90kHz / k25Fps; + uint32_t timestamp = 0xFFFFFFFFu - 3 * kRtpTicksPerFrame; + for (int i = 0; i < 5; ++i) { + timing.IncomingTimestamp(timestamp, clock.CurrentTime()); + clock.AdvanceTime(1 / k25Fps); + timestamp += kRtpTicksPerFrame; + EXPECT_EQ(kStartTime + 3 / k25Fps, + timing.RenderTime(0xFFFFFFFFu, clock.CurrentTime())); + // One ms later in 90 kHz. + EXPECT_EQ(kStartTime + 3 / k25Fps + TimeDelta::Millis(1), + timing.RenderTime(89u, clock.CurrentTime())); + } +} + +TEST(ReceiverTimingTest, UseLowLatencyRenderer) { + test::ScopedKeyValueConfig field_trials; + SimulatedClock clock(0); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + // Default is false. + EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); + // False if min playout delay > 0. + timing.set_min_playout_delay(TimeDelta::Millis(10)); + timing.set_max_playout_delay(TimeDelta::Millis(20)); + EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); + // True if min==0, max > 0. + timing.set_min_playout_delay(TimeDelta::Zero()); + EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering); + // True if min==max==0. + timing.set_max_playout_delay(TimeDelta::Zero()); + EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering); + // True also for max playout delay==500 ms. + timing.set_max_playout_delay(TimeDelta::Millis(500)); + EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering); + // False if max playout delay > 500 ms. + timing.set_max_playout_delay(TimeDelta::Millis(501)); + EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); +} + +TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) { + // This is the default path when the RTP playout delay header extension is set + // to min==0 and max==0. + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60); + constexpr Timestamp kZeroRenderTime = Timestamp::Zero(); + SimulatedClock clock(kStartTimeUs); + test::ScopedKeyValueConfig field_trials; + VCMTiming timing(&clock, field_trials); + timing.Reset(); + timing.set_max_playout_delay(TimeDelta::Zero()); + for (int i = 0; i < 10; ++i) { + clock.AdvanceTime(kTimeDelta); + Timestamp now = clock.CurrentTime(); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + } + // Another frame submitted at the same time also returns a negative max + // waiting time. + Timestamp now = clock.CurrentTime(); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + // MaxWaitingTime should be less than zero even if there's a burst of frames. + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); +} + +TEST(ReceiverTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) { + // The minimum pacing is enabled by a field trial and active if the RTP + // playout delay header extension is set to min==0. + constexpr TimeDelta kMinPacing = TimeDelta::Millis(3); + test::ScopedKeyValueConfig field_trials( + "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60); + constexpr auto kZeroRenderTime = Timestamp::Zero(); + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + // MaxWaitingTime() returns zero for evenly spaced video frames. + for (int i = 0; i < 10; ++i) { + clock.AdvanceTime(kTimeDelta); + Timestamp now = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + timing.SetLastDecodeScheduledTimestamp(now); + } + // Another frame submitted at the same time is paced according to the field + // trial setting. + auto now = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); + // If there's a burst of frames, the wait time is calculated based on next + // decode time. + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); + // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime. + constexpr TimeDelta kTwoMs = TimeDelta::Millis(2); + clock.AdvanceTime(kTwoMs); + now = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing - kTwoMs); + // A frame is decoded at the current time, the wait time should be restored to + // pacing delay. + timing.SetLastDecodeScheduledTimestamp(now); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); +} + +TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) { + // The minimum pacing is enabled by a field trial but should not have any + // effect if render_time_ms is greater than 0; + test::ScopedKeyValueConfig field_trials( + "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0); + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + clock.AdvanceTime(kTimeDelta); + auto now = clock.CurrentTime(); + Timestamp render_time = now + TimeDelta::Millis(30); + // Estimate the internal processing delay from the first frame. + TimeDelta estimated_processing_delay = + (render_time - now) - + timing.MaxWaitingTime(render_time, now, + /*too_many_frames_queued=*/false); + EXPECT_GT(estimated_processing_delay, TimeDelta::Zero()); + + // Any other frame submitted at the same time should be scheduled according to + // its render time. + for (int i = 0; i < 5; ++i) { + render_time += kTimeDelta; + EXPECT_EQ(timing.MaxWaitingTime(render_time, now, + /*too_many_frames_queued=*/false), + render_time - now - estimated_processing_delay); + } +} + +TEST(ReceiverTimingTest, MaxWaitingTimeReturnsZeroIfTooManyFramesQueuedIsTrue) { + // The minimum pacing is enabled by a field trial and active if the RTP + // playout delay header extension is set to min==0. + constexpr TimeDelta kMinPacing = TimeDelta::Millis(3); + test::ScopedKeyValueConfig field_trials( + "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0); + constexpr auto kZeroRenderTime = Timestamp::Zero(); + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + // MaxWaitingTime() returns zero for evenly spaced video frames. + for (int i = 0; i < 10; ++i) { + clock.AdvanceTime(kTimeDelta); + auto now = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + timing.SetLastDecodeScheduledTimestamp(now); + } + // Another frame submitted at the same time is paced according to the field + // trial setting. + auto now_ms = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms, + /*too_many_frames_queued=*/false), + kMinPacing); + // MaxWaitingTime returns 0 even if there's a burst of frames if + // too_many_frames_queued is set to true. + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms, + /*too_many_frames_queued=*/true), + TimeDelta::Zero()); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms, + /*too_many_frames_queued=*/true), + TimeDelta::Zero()); +} + +TEST(ReceiverTimingTest, UpdateCurrentDelayCapsWhenOffByMicroseconds) { + test::ScopedKeyValueConfig field_trials; + SimulatedClock clock(0); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + + // Set larger initial current delay. + timing.set_min_playout_delay(TimeDelta::Millis(200)); + timing.UpdateCurrentDelay(Timestamp::Millis(900), Timestamp::Millis(1000)); + + // Add a few microseconds to ensure that the delta of decode time is 0 after + // rounding, and should reset to the target delay. + timing.set_min_playout_delay(TimeDelta::Millis(50)); + Timestamp decode_time = Timestamp::Millis(1337); + Timestamp render_time = + decode_time + TimeDelta::Millis(10) + TimeDelta::Micros(37); + timing.UpdateCurrentDelay(render_time, decode_time); + EXPECT_EQ(timing.GetTimings().current_delay, timing.TargetVideoDelay()); +} + +} // namespace webrtc |