summaryrefslogtreecommitdiffstats
path: root/widget/TouchResampler.cpp
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /widget/TouchResampler.cpp
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'widget/TouchResampler.cpp')
-rw-r--r--widget/TouchResampler.cpp377
1 files changed, 377 insertions, 0 deletions
diff --git a/widget/TouchResampler.cpp b/widget/TouchResampler.cpp
new file mode 100644
index 0000000000..eeed30fe0e
--- /dev/null
+++ b/widget/TouchResampler.cpp
@@ -0,0 +1,377 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "TouchResampler.h"
+
+#include "nsAlgorithm.h"
+
+/**
+ * TouchResampler implementation
+ */
+
+namespace mozilla {
+namespace widget {
+
+// The values below have been tested and found to be acceptable on a device
+// with a display refresh rate of 60Hz and touch sampling rate of 100Hz.
+// While their "ideal" values are dependent on the exact rates of each device,
+// the values we've picked below should be somewhat robust across a variation of
+// different rates. They mostly aim to avoid making predictions that are too far
+// away (in terms of distance) from the finger, and to detect pauses in the
+// finger motion without too much delay.
+
+// Maximum time between two consecutive data points to consider resampling
+// between them.
+// Values between 1x and 5x of the touch sampling interval are reasonable.
+static const double kTouchResampleWindowSize = 40.0;
+
+// These next two values constrain the sampling timestamp.
+// Our caller will usually adjust frame timestamps to be slightly in the past,
+// for example by 5ms. This means that, during normal operation, we will
+// maximally need to predict by [touch sampling rate] minus 5ms.
+// So we would like kTouchResampleMaxPredictMs to satisfy the following:
+// kTouchResampleMaxPredictMs + [frame time adjust] > [touch sampling rate]
+static const double kTouchResampleMaxPredictMs = 8.0;
+// This one is a protection against very outdated frame timestamps.
+// Values larger than the touch sampling interval and less than 3x of the vsync
+// interval are reasonable.
+static const double kTouchResampleMaxBacksampleMs = 20.0;
+
+// The maximum age of the most recent data point to consider resampling.
+// Should be between 1x and 3x of the touch sampling interval.
+static const double kTouchResampleOldTouchThresholdMs = 17.0;
+
+uint64_t TouchResampler::ProcessEvent(MultiTouchInput&& aInput) {
+ mCurrentTouches.UpdateFromEvent(aInput);
+
+ uint64_t eventId = mNextEventId;
+ mNextEventId++;
+
+ if (aInput.mType == MultiTouchInput::MULTITOUCH_MOVE) {
+ // Touch move events are deferred until NotifyFrame.
+ mDeferredTouchMoveEvents.push({std::move(aInput), eventId});
+ } else {
+ // Non-move events are transferred to the outgoing queue unmodified.
+ // If there are pending touch move events, flush those out first, so that
+ // events are emitted in the right order.
+ FlushDeferredTouchMoveEventsUnresampled();
+ if (mInResampledState) {
+ // Return to a non-resampled state before emitting a non-move event.
+ ReturnToNonResampledState();
+ }
+ EmitEvent(std::move(aInput), eventId);
+ }
+
+ return eventId;
+}
+
+void TouchResampler::NotifyFrame(const TimeStamp& aTimeStamp) {
+ TimeStamp lastTouchTime = mCurrentTouches.LatestDataPointTime();
+ if (mDeferredTouchMoveEvents.empty() ||
+ (lastTouchTime &&
+ lastTouchTime < aTimeStamp - TimeDuration::FromMilliseconds(
+ kTouchResampleOldTouchThresholdMs))) {
+ // We haven't received a touch move event in a while, so the fingers must
+ // have stopped moving. Flush any old touch move events.
+ FlushDeferredTouchMoveEventsUnresampled();
+
+ if (mInResampledState) {
+ // Make sure we pause at the resting position that we actually observed,
+ // and not at a resampled position.
+ ReturnToNonResampledState();
+ }
+
+ // Clear touch location history so that we don't resample across a pause.
+ mCurrentTouches.ClearDataPoints();
+ return;
+ }
+
+ MOZ_RELEASE_ASSERT(lastTouchTime);
+ TimeStamp lowerBound = lastTouchTime - TimeDuration::FromMilliseconds(
+ kTouchResampleMaxBacksampleMs);
+ TimeStamp upperBound = lastTouchTime + TimeDuration::FromMilliseconds(
+ kTouchResampleMaxPredictMs);
+ TimeStamp sampleTime = clamped(aTimeStamp, lowerBound, upperBound);
+
+ if (mLastEmittedEventTime && sampleTime < mLastEmittedEventTime) {
+ // Keep emitted timestamps in order.
+ sampleTime = mLastEmittedEventTime;
+ }
+
+ // We have at least one pending touch move event. Pick one of the events from
+ // mDeferredTouchMoveEvents as the base event for the resampling adjustment.
+ // We want to produce an event stream whose timestamps are in the right order.
+ // As the base event, use the first event that's at or after sampleTime,
+ // unless there is no such event, in that case use the last one we have. We
+ // will set the timestamp on the resampled event to sampleTime later.
+ // Flush out any older events so that everything remains in the right order.
+ MultiTouchInput input;
+ uint64_t eventId;
+ while (true) {
+ MOZ_RELEASE_ASSERT(!mDeferredTouchMoveEvents.empty());
+ std::tie(input, eventId) = std::move(mDeferredTouchMoveEvents.front());
+ mDeferredTouchMoveEvents.pop();
+ if (mDeferredTouchMoveEvents.empty() || input.mTimeStamp >= sampleTime) {
+ break;
+ }
+ // Flush this event to the outgoing queue without resampling. What ends up
+ // on the screen will still be smooth because we will proceed to emit a
+ // resampled event before the paint for this frame starts.
+ PrependLeftoverHistoricalData(&input);
+ MOZ_RELEASE_ASSERT(input.mTimeStamp < sampleTime);
+ EmitEvent(std::move(input), eventId);
+ }
+
+ mOriginalOfResampledTouchMove = Nothing();
+
+ // Compute the resampled touch positions.
+ nsTArray<ScreenIntPoint> resampledPositions;
+ bool anyPositionDifferentFromOriginal = false;
+ for (const auto& touch : input.mTouches) {
+ ScreenIntPoint resampledPosition =
+ mCurrentTouches.ResampleTouchPositionAtTime(
+ touch.mIdentifier, touch.mScreenPoint, sampleTime);
+ if (resampledPosition != touch.mScreenPoint) {
+ anyPositionDifferentFromOriginal = true;
+ }
+ resampledPositions.AppendElement(resampledPosition);
+ }
+
+ if (anyPositionDifferentFromOriginal) {
+ // Store a copy of the original event, so that we can return to an
+ // non-resampled position later, if necessary.
+ mOriginalOfResampledTouchMove = Some(input);
+
+ // Add the original observed position to the historical data, as well as any
+ // leftover historical positions from the previous touch move event, and
+ // store the resampled values in the "final" position of the event.
+ PrependLeftoverHistoricalData(&input);
+ for (size_t i = 0; i < input.mTouches.Length(); i++) {
+ auto& touch = input.mTouches[i];
+ touch.mHistoricalData.AppendElement(SingleTouchData::HistoricalTouchData{
+ input.mTimeStamp,
+ touch.mScreenPoint,
+ touch.mLocalScreenPoint,
+ touch.mRadius,
+ touch.mRotationAngle,
+ touch.mForce,
+ });
+
+ // Remove any historical touch data that's in the future, compared to
+ // sampleTime. This data will be included by upcoming touch move
+ // events. This only happens if the frame timestamp can be older than the
+ // event timestamp, i.e. if interpolation occurs (rather than
+ // extrapolation).
+ auto futureDataStart = std::find_if(
+ touch.mHistoricalData.begin(), touch.mHistoricalData.end(),
+ [sampleTime](
+ const SingleTouchData::HistoricalTouchData& aHistoricalData) {
+ return aHistoricalData.mTimeStamp > sampleTime;
+ });
+ if (futureDataStart != touch.mHistoricalData.end()) {
+ nsTArray<SingleTouchData::HistoricalTouchData> futureData(
+ Span<SingleTouchData::HistoricalTouchData>(touch.mHistoricalData)
+ .From(futureDataStart.GetIndex()));
+ touch.mHistoricalData.TruncateLength(futureDataStart.GetIndex());
+ mRemainingTouchData.insert({touch.mIdentifier, std::move(futureData)});
+ }
+
+ touch.mScreenPoint = resampledPositions[i];
+ }
+ input.mTimeStamp = sampleTime;
+ }
+
+ EmitEvent(std::move(input), eventId);
+ mInResampledState = anyPositionDifferentFromOriginal;
+}
+
+void TouchResampler::PrependLeftoverHistoricalData(MultiTouchInput* aInput) {
+ for (auto& touch : aInput->mTouches) {
+ auto leftoverData = mRemainingTouchData.find(touch.mIdentifier);
+ if (leftoverData != mRemainingTouchData.end()) {
+ nsTArray<SingleTouchData::HistoricalTouchData> data =
+ std::move(leftoverData->second);
+ mRemainingTouchData.erase(leftoverData);
+ touch.mHistoricalData.InsertElementsAt(0, data);
+ }
+
+ if (TimeStamp cutoffTime = mLastEmittedEventTime) {
+ // If we received historical touch data that was further in the past than
+ // the last resampled event, discard that data so that the touch data
+ // points are emitted in order.
+ touch.mHistoricalData.RemoveElementsBy(
+ [cutoffTime](const SingleTouchData::HistoricalTouchData& aTouchData) {
+ return aTouchData.mTimeStamp < cutoffTime;
+ });
+ }
+ }
+ mRemainingTouchData.clear();
+}
+
+void TouchResampler::FlushDeferredTouchMoveEventsUnresampled() {
+ while (!mDeferredTouchMoveEvents.empty()) {
+ auto [input, eventId] = std::move(mDeferredTouchMoveEvents.front());
+ mDeferredTouchMoveEvents.pop();
+ PrependLeftoverHistoricalData(&input);
+ EmitEvent(std::move(input), eventId);
+ mInResampledState = false;
+ mOriginalOfResampledTouchMove = Nothing();
+ }
+}
+
+void TouchResampler::ReturnToNonResampledState() {
+ MOZ_RELEASE_ASSERT(mInResampledState);
+ MOZ_RELEASE_ASSERT(mDeferredTouchMoveEvents.empty(),
+ "Don't call this if there is a deferred touch move event. "
+ "We can return to the non-resampled state by sending that "
+ "event, rather than a copy of a previous event.");
+
+ // The last outgoing event was a resampled touch move event.
+ // Return to the non-resampled state, by sending a touch move event to
+ // "overwrite" any resampled positions with the original observed positions.
+ MultiTouchInput input = std::move(*mOriginalOfResampledTouchMove);
+ mOriginalOfResampledTouchMove = Nothing();
+
+ // For the event's timestamp, we want to backdate the correction as far as we
+ // can, while still preserving timestamp ordering. But we also don't want to
+ // backdate it to be older than it was originally.
+ if (mLastEmittedEventTime > input.mTimeStamp) {
+ input.mTimeStamp = mLastEmittedEventTime;
+ }
+
+ // Assemble the correct historical touch data for this event.
+ // We don't want to include data points that we've already sent out with the
+ // resampled event. And from the leftover data points, we only want those that
+ // don't duplicate the final time + position of this event.
+ for (auto& touch : input.mTouches) {
+ touch.mHistoricalData.Clear();
+ }
+ PrependLeftoverHistoricalData(&input);
+ for (auto& touch : input.mTouches) {
+ touch.mHistoricalData.RemoveElementsBy([&](const auto& histData) {
+ return histData.mTimeStamp >= input.mTimeStamp;
+ });
+ }
+
+ EmitExtraEvent(std::move(input));
+ mInResampledState = false;
+}
+
+void TouchResampler::TouchInfo::Update(const SingleTouchData& aTouch,
+ const TimeStamp& aEventTime) {
+ for (const auto& historicalData : aTouch.mHistoricalData) {
+ mBaseDataPoint = mLatestDataPoint;
+ mLatestDataPoint =
+ Some(DataPoint{historicalData.mTimeStamp, historicalData.mScreenPoint});
+ }
+ mBaseDataPoint = mLatestDataPoint;
+ mLatestDataPoint = Some(DataPoint{aEventTime, aTouch.mScreenPoint});
+}
+
+ScreenIntPoint TouchResampler::TouchInfo::ResampleAtTime(
+ const ScreenIntPoint& aLastObservedPosition, const TimeStamp& aTimeStamp) {
+ TimeStamp cutoff =
+ aTimeStamp - TimeDuration::FromMilliseconds(kTouchResampleWindowSize);
+ if (!mBaseDataPoint || !mLatestDataPoint ||
+ !(mBaseDataPoint->mTimeStamp < mLatestDataPoint->mTimeStamp) ||
+ mBaseDataPoint->mTimeStamp < cutoff) {
+ return aLastObservedPosition;
+ }
+
+ // For the actual resampling, connect the last two data points with a line and
+ // sample along that line.
+ TimeStamp t1 = mBaseDataPoint->mTimeStamp;
+ TimeStamp t2 = mLatestDataPoint->mTimeStamp;
+ double t = (aTimeStamp - t1) / (t2 - t1);
+
+ double x1 = mBaseDataPoint->mPosition.x;
+ double x2 = mLatestDataPoint->mPosition.x;
+ double y1 = mBaseDataPoint->mPosition.y;
+ double y2 = mLatestDataPoint->mPosition.y;
+
+ int32_t resampledX = round(x1 + t * (x2 - x1));
+ int32_t resampledY = round(y1 + t * (y2 - y1));
+ return ScreenIntPoint(resampledX, resampledY);
+}
+
+void TouchResampler::CurrentTouches::UpdateFromEvent(
+ const MultiTouchInput& aInput) {
+ switch (aInput.mType) {
+ case MultiTouchInput::MULTITOUCH_START: {
+ // A new touch has been added; make sure mTouches reflects the current
+ // touches in the event.
+ nsTArray<TouchInfo> newTouches;
+ for (const auto& touch : aInput.mTouches) {
+ const auto touchInfo = TouchByIdentifier(touch.mIdentifier);
+ if (touchInfo != mTouches.end()) {
+ // This is one of the existing touches.
+ newTouches.AppendElement(std::move(*touchInfo));
+ mTouches.RemoveElementAt(touchInfo);
+ } else {
+ // This is the new touch.
+ newTouches.AppendElement(TouchInfo{
+ touch.mIdentifier, Nothing(),
+ Some(DataPoint{aInput.mTimeStamp, touch.mScreenPoint})});
+ }
+ }
+ MOZ_ASSERT(mTouches.IsEmpty(), "Missing touch end before touch start?");
+ mTouches = std::move(newTouches);
+ break;
+ }
+
+ case MultiTouchInput::MULTITOUCH_MOVE: {
+ // The touches have moved.
+ // Add position information to the history data points.
+ for (const auto& touch : aInput.mTouches) {
+ const auto touchInfo = TouchByIdentifier(touch.mIdentifier);
+ MOZ_ASSERT(touchInfo != mTouches.end());
+ if (touchInfo != mTouches.end()) {
+ touchInfo->Update(touch, aInput.mTimeStamp);
+ }
+ }
+ mLatestDataPointTime = aInput.mTimeStamp;
+ break;
+ }
+
+ case MultiTouchInput::MULTITOUCH_END: {
+ // A touch has been removed.
+ MOZ_RELEASE_ASSERT(aInput.mTouches.Length() == 1);
+ const auto touchInfo = TouchByIdentifier(aInput.mTouches[0].mIdentifier);
+ MOZ_ASSERT(touchInfo != mTouches.end());
+ if (touchInfo != mTouches.end()) {
+ mTouches.RemoveElementAt(touchInfo);
+ }
+ break;
+ }
+
+ case MultiTouchInput::MULTITOUCH_CANCEL:
+ // All touches are canceled.
+ mTouches.Clear();
+ break;
+ }
+}
+
+nsTArray<TouchResampler::TouchInfo>::iterator
+TouchResampler::CurrentTouches::TouchByIdentifier(int32_t aIdentifier) {
+ return std::find_if(mTouches.begin(), mTouches.end(),
+ [aIdentifier](const TouchInfo& info) {
+ return info.mIdentifier == aIdentifier;
+ });
+}
+
+ScreenIntPoint TouchResampler::CurrentTouches::ResampleTouchPositionAtTime(
+ int32_t aIdentifier, const ScreenIntPoint& aLastObservedPosition,
+ const TimeStamp& aTimeStamp) {
+ const auto touchInfo = TouchByIdentifier(aIdentifier);
+ MOZ_ASSERT(touchInfo != mTouches.end());
+ if (touchInfo != mTouches.end()) {
+ return touchInfo->ResampleAtTime(aLastObservedPosition, aTimeStamp);
+ }
+ return aLastObservedPosition;
+}
+
+} // namespace widget
+} // namespace mozilla