diff options
Diffstat (limited to 'widget/TouchResampler.cpp')
| -rw-r--r-- | widget/TouchResampler.cpp | 377 |
1 files changed, 377 insertions, 0 deletions
diff --git a/widget/TouchResampler.cpp b/widget/TouchResampler.cpp new file mode 100644 index 0000000000..eeed30fe0e --- /dev/null +++ b/widget/TouchResampler.cpp @@ -0,0 +1,377 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "TouchResampler.h" + +#include "nsAlgorithm.h" + +/** + * TouchResampler implementation + */ + +namespace mozilla { +namespace widget { + +// The values below have been tested and found to be acceptable on a device +// with a display refresh rate of 60Hz and touch sampling rate of 100Hz. +// While their "ideal" values are dependent on the exact rates of each device, +// the values we've picked below should be somewhat robust across a variation of +// different rates. They mostly aim to avoid making predictions that are too far +// away (in terms of distance) from the finger, and to detect pauses in the +// finger motion without too much delay. + +// Maximum time between two consecutive data points to consider resampling +// between them. +// Values between 1x and 5x of the touch sampling interval are reasonable. +static const double kTouchResampleWindowSize = 40.0; + +// These next two values constrain the sampling timestamp. +// Our caller will usually adjust frame timestamps to be slightly in the past, +// for example by 5ms. This means that, during normal operation, we will +// maximally need to predict by [touch sampling rate] minus 5ms. +// So we would like kTouchResampleMaxPredictMs to satisfy the following: +// kTouchResampleMaxPredictMs + [frame time adjust] > [touch sampling rate] +static const double kTouchResampleMaxPredictMs = 8.0; +// This one is a protection against very outdated frame timestamps. +// Values larger than the touch sampling interval and less than 3x of the vsync +// interval are reasonable. +static const double kTouchResampleMaxBacksampleMs = 20.0; + +// The maximum age of the most recent data point to consider resampling. +// Should be between 1x and 3x of the touch sampling interval. +static const double kTouchResampleOldTouchThresholdMs = 17.0; + +uint64_t TouchResampler::ProcessEvent(MultiTouchInput&& aInput) { + mCurrentTouches.UpdateFromEvent(aInput); + + uint64_t eventId = mNextEventId; + mNextEventId++; + + if (aInput.mType == MultiTouchInput::MULTITOUCH_MOVE) { + // Touch move events are deferred until NotifyFrame. + mDeferredTouchMoveEvents.push({std::move(aInput), eventId}); + } else { + // Non-move events are transferred to the outgoing queue unmodified. + // If there are pending touch move events, flush those out first, so that + // events are emitted in the right order. + FlushDeferredTouchMoveEventsUnresampled(); + if (mInResampledState) { + // Return to a non-resampled state before emitting a non-move event. + ReturnToNonResampledState(); + } + EmitEvent(std::move(aInput), eventId); + } + + return eventId; +} + +void TouchResampler::NotifyFrame(const TimeStamp& aTimeStamp) { + TimeStamp lastTouchTime = mCurrentTouches.LatestDataPointTime(); + if (mDeferredTouchMoveEvents.empty() || + (lastTouchTime && + lastTouchTime < aTimeStamp - TimeDuration::FromMilliseconds( + kTouchResampleOldTouchThresholdMs))) { + // We haven't received a touch move event in a while, so the fingers must + // have stopped moving. Flush any old touch move events. + FlushDeferredTouchMoveEventsUnresampled(); + + if (mInResampledState) { + // Make sure we pause at the resting position that we actually observed, + // and not at a resampled position. + ReturnToNonResampledState(); + } + + // Clear touch location history so that we don't resample across a pause. + mCurrentTouches.ClearDataPoints(); + return; + } + + MOZ_RELEASE_ASSERT(lastTouchTime); + TimeStamp lowerBound = lastTouchTime - TimeDuration::FromMilliseconds( + kTouchResampleMaxBacksampleMs); + TimeStamp upperBound = lastTouchTime + TimeDuration::FromMilliseconds( + kTouchResampleMaxPredictMs); + TimeStamp sampleTime = clamped(aTimeStamp, lowerBound, upperBound); + + if (mLastEmittedEventTime && sampleTime < mLastEmittedEventTime) { + // Keep emitted timestamps in order. + sampleTime = mLastEmittedEventTime; + } + + // We have at least one pending touch move event. Pick one of the events from + // mDeferredTouchMoveEvents as the base event for the resampling adjustment. + // We want to produce an event stream whose timestamps are in the right order. + // As the base event, use the first event that's at or after sampleTime, + // unless there is no such event, in that case use the last one we have. We + // will set the timestamp on the resampled event to sampleTime later. + // Flush out any older events so that everything remains in the right order. + MultiTouchInput input; + uint64_t eventId; + while (true) { + MOZ_RELEASE_ASSERT(!mDeferredTouchMoveEvents.empty()); + std::tie(input, eventId) = std::move(mDeferredTouchMoveEvents.front()); + mDeferredTouchMoveEvents.pop(); + if (mDeferredTouchMoveEvents.empty() || input.mTimeStamp >= sampleTime) { + break; + } + // Flush this event to the outgoing queue without resampling. What ends up + // on the screen will still be smooth because we will proceed to emit a + // resampled event before the paint for this frame starts. + PrependLeftoverHistoricalData(&input); + MOZ_RELEASE_ASSERT(input.mTimeStamp < sampleTime); + EmitEvent(std::move(input), eventId); + } + + mOriginalOfResampledTouchMove = Nothing(); + + // Compute the resampled touch positions. + nsTArray<ScreenIntPoint> resampledPositions; + bool anyPositionDifferentFromOriginal = false; + for (const auto& touch : input.mTouches) { + ScreenIntPoint resampledPosition = + mCurrentTouches.ResampleTouchPositionAtTime( + touch.mIdentifier, touch.mScreenPoint, sampleTime); + if (resampledPosition != touch.mScreenPoint) { + anyPositionDifferentFromOriginal = true; + } + resampledPositions.AppendElement(resampledPosition); + } + + if (anyPositionDifferentFromOriginal) { + // Store a copy of the original event, so that we can return to an + // non-resampled position later, if necessary. + mOriginalOfResampledTouchMove = Some(input); + + // Add the original observed position to the historical data, as well as any + // leftover historical positions from the previous touch move event, and + // store the resampled values in the "final" position of the event. + PrependLeftoverHistoricalData(&input); + for (size_t i = 0; i < input.mTouches.Length(); i++) { + auto& touch = input.mTouches[i]; + touch.mHistoricalData.AppendElement(SingleTouchData::HistoricalTouchData{ + input.mTimeStamp, + touch.mScreenPoint, + touch.mLocalScreenPoint, + touch.mRadius, + touch.mRotationAngle, + touch.mForce, + }); + + // Remove any historical touch data that's in the future, compared to + // sampleTime. This data will be included by upcoming touch move + // events. This only happens if the frame timestamp can be older than the + // event timestamp, i.e. if interpolation occurs (rather than + // extrapolation). + auto futureDataStart = std::find_if( + touch.mHistoricalData.begin(), touch.mHistoricalData.end(), + [sampleTime]( + const SingleTouchData::HistoricalTouchData& aHistoricalData) { + return aHistoricalData.mTimeStamp > sampleTime; + }); + if (futureDataStart != touch.mHistoricalData.end()) { + nsTArray<SingleTouchData::HistoricalTouchData> futureData( + Span<SingleTouchData::HistoricalTouchData>(touch.mHistoricalData) + .From(futureDataStart.GetIndex())); + touch.mHistoricalData.TruncateLength(futureDataStart.GetIndex()); + mRemainingTouchData.insert({touch.mIdentifier, std::move(futureData)}); + } + + touch.mScreenPoint = resampledPositions[i]; + } + input.mTimeStamp = sampleTime; + } + + EmitEvent(std::move(input), eventId); + mInResampledState = anyPositionDifferentFromOriginal; +} + +void TouchResampler::PrependLeftoverHistoricalData(MultiTouchInput* aInput) { + for (auto& touch : aInput->mTouches) { + auto leftoverData = mRemainingTouchData.find(touch.mIdentifier); + if (leftoverData != mRemainingTouchData.end()) { + nsTArray<SingleTouchData::HistoricalTouchData> data = + std::move(leftoverData->second); + mRemainingTouchData.erase(leftoverData); + touch.mHistoricalData.InsertElementsAt(0, data); + } + + if (TimeStamp cutoffTime = mLastEmittedEventTime) { + // If we received historical touch data that was further in the past than + // the last resampled event, discard that data so that the touch data + // points are emitted in order. + touch.mHistoricalData.RemoveElementsBy( + [cutoffTime](const SingleTouchData::HistoricalTouchData& aTouchData) { + return aTouchData.mTimeStamp < cutoffTime; + }); + } + } + mRemainingTouchData.clear(); +} + +void TouchResampler::FlushDeferredTouchMoveEventsUnresampled() { + while (!mDeferredTouchMoveEvents.empty()) { + auto [input, eventId] = std::move(mDeferredTouchMoveEvents.front()); + mDeferredTouchMoveEvents.pop(); + PrependLeftoverHistoricalData(&input); + EmitEvent(std::move(input), eventId); + mInResampledState = false; + mOriginalOfResampledTouchMove = Nothing(); + } +} + +void TouchResampler::ReturnToNonResampledState() { + MOZ_RELEASE_ASSERT(mInResampledState); + MOZ_RELEASE_ASSERT(mDeferredTouchMoveEvents.empty(), + "Don't call this if there is a deferred touch move event. " + "We can return to the non-resampled state by sending that " + "event, rather than a copy of a previous event."); + + // The last outgoing event was a resampled touch move event. + // Return to the non-resampled state, by sending a touch move event to + // "overwrite" any resampled positions with the original observed positions. + MultiTouchInput input = std::move(*mOriginalOfResampledTouchMove); + mOriginalOfResampledTouchMove = Nothing(); + + // For the event's timestamp, we want to backdate the correction as far as we + // can, while still preserving timestamp ordering. But we also don't want to + // backdate it to be older than it was originally. + if (mLastEmittedEventTime > input.mTimeStamp) { + input.mTimeStamp = mLastEmittedEventTime; + } + + // Assemble the correct historical touch data for this event. + // We don't want to include data points that we've already sent out with the + // resampled event. And from the leftover data points, we only want those that + // don't duplicate the final time + position of this event. + for (auto& touch : input.mTouches) { + touch.mHistoricalData.Clear(); + } + PrependLeftoverHistoricalData(&input); + for (auto& touch : input.mTouches) { + touch.mHistoricalData.RemoveElementsBy([&](const auto& histData) { + return histData.mTimeStamp >= input.mTimeStamp; + }); + } + + EmitExtraEvent(std::move(input)); + mInResampledState = false; +} + +void TouchResampler::TouchInfo::Update(const SingleTouchData& aTouch, + const TimeStamp& aEventTime) { + for (const auto& historicalData : aTouch.mHistoricalData) { + mBaseDataPoint = mLatestDataPoint; + mLatestDataPoint = + Some(DataPoint{historicalData.mTimeStamp, historicalData.mScreenPoint}); + } + mBaseDataPoint = mLatestDataPoint; + mLatestDataPoint = Some(DataPoint{aEventTime, aTouch.mScreenPoint}); +} + +ScreenIntPoint TouchResampler::TouchInfo::ResampleAtTime( + const ScreenIntPoint& aLastObservedPosition, const TimeStamp& aTimeStamp) { + TimeStamp cutoff = + aTimeStamp - TimeDuration::FromMilliseconds(kTouchResampleWindowSize); + if (!mBaseDataPoint || !mLatestDataPoint || + !(mBaseDataPoint->mTimeStamp < mLatestDataPoint->mTimeStamp) || + mBaseDataPoint->mTimeStamp < cutoff) { + return aLastObservedPosition; + } + + // For the actual resampling, connect the last two data points with a line and + // sample along that line. + TimeStamp t1 = mBaseDataPoint->mTimeStamp; + TimeStamp t2 = mLatestDataPoint->mTimeStamp; + double t = (aTimeStamp - t1) / (t2 - t1); + + double x1 = mBaseDataPoint->mPosition.x; + double x2 = mLatestDataPoint->mPosition.x; + double y1 = mBaseDataPoint->mPosition.y; + double y2 = mLatestDataPoint->mPosition.y; + + int32_t resampledX = round(x1 + t * (x2 - x1)); + int32_t resampledY = round(y1 + t * (y2 - y1)); + return ScreenIntPoint(resampledX, resampledY); +} + +void TouchResampler::CurrentTouches::UpdateFromEvent( + const MultiTouchInput& aInput) { + switch (aInput.mType) { + case MultiTouchInput::MULTITOUCH_START: { + // A new touch has been added; make sure mTouches reflects the current + // touches in the event. + nsTArray<TouchInfo> newTouches; + for (const auto& touch : aInput.mTouches) { + const auto touchInfo = TouchByIdentifier(touch.mIdentifier); + if (touchInfo != mTouches.end()) { + // This is one of the existing touches. + newTouches.AppendElement(std::move(*touchInfo)); + mTouches.RemoveElementAt(touchInfo); + } else { + // This is the new touch. + newTouches.AppendElement(TouchInfo{ + touch.mIdentifier, Nothing(), + Some(DataPoint{aInput.mTimeStamp, touch.mScreenPoint})}); + } + } + MOZ_ASSERT(mTouches.IsEmpty(), "Missing touch end before touch start?"); + mTouches = std::move(newTouches); + break; + } + + case MultiTouchInput::MULTITOUCH_MOVE: { + // The touches have moved. + // Add position information to the history data points. + for (const auto& touch : aInput.mTouches) { + const auto touchInfo = TouchByIdentifier(touch.mIdentifier); + MOZ_ASSERT(touchInfo != mTouches.end()); + if (touchInfo != mTouches.end()) { + touchInfo->Update(touch, aInput.mTimeStamp); + } + } + mLatestDataPointTime = aInput.mTimeStamp; + break; + } + + case MultiTouchInput::MULTITOUCH_END: { + // A touch has been removed. + MOZ_RELEASE_ASSERT(aInput.mTouches.Length() == 1); + const auto touchInfo = TouchByIdentifier(aInput.mTouches[0].mIdentifier); + MOZ_ASSERT(touchInfo != mTouches.end()); + if (touchInfo != mTouches.end()) { + mTouches.RemoveElementAt(touchInfo); + } + break; + } + + case MultiTouchInput::MULTITOUCH_CANCEL: + // All touches are canceled. + mTouches.Clear(); + break; + } +} + +nsTArray<TouchResampler::TouchInfo>::iterator +TouchResampler::CurrentTouches::TouchByIdentifier(int32_t aIdentifier) { + return std::find_if(mTouches.begin(), mTouches.end(), + [aIdentifier](const TouchInfo& info) { + return info.mIdentifier == aIdentifier; + }); +} + +ScreenIntPoint TouchResampler::CurrentTouches::ResampleTouchPositionAtTime( + int32_t aIdentifier, const ScreenIntPoint& aLastObservedPosition, + const TimeStamp& aTimeStamp) { + const auto touchInfo = TouchByIdentifier(aIdentifier); + MOZ_ASSERT(touchInfo != mTouches.end()); + if (touchInfo != mTouches.end()) { + return touchInfo->ResampleAtTime(aLastObservedPosition, aTimeStamp); + } + return aLastObservedPosition; +} + +} // namespace widget +} // namespace mozilla |