From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- dom/media/DriftCompensation.h | 137 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 dom/media/DriftCompensation.h (limited to 'dom/media/DriftCompensation.h') diff --git a/dom/media/DriftCompensation.h b/dom/media/DriftCompensation.h new file mode 100644 index 0000000000..ef22f7106f --- /dev/null +++ b/dom/media/DriftCompensation.h @@ -0,0 +1,137 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef DriftCompensation_h_ +#define DriftCompensation_h_ + +#include "MediaSegment.h" +#include "VideoUtils.h" +#include "mozilla/Atomics.h" +#include "mozilla/Unused.h" + +namespace mozilla { + +static LazyLogModule gDriftCompensatorLog("DriftCompensator"); +#define LOG(type, ...) MOZ_LOG(gDriftCompensatorLog, type, (__VA_ARGS__)) + +/** + * DriftCompensator can be used to handle drift between audio and video tracks + * from the MediaTrackGraph. + * + * Drift can occur because audio is driven by a MediaTrackGraph running off an + * audio callback, thus it's progressed by the clock of one the audio output + * devices on the user's machine. Video on the other hand is always expressed in + * wall-clock TimeStamps, i.e., it's progressed by the system clock. These + * clocks will, over time, drift apart. + * + * Do not use the DriftCompensator across multiple audio tracks, as it will + * automatically record the start time of the first audio samples, and all + * samples for the same audio track on the same audio clock will have to be + * processed to retain accuracy. + * + * DriftCompensator is designed to be used from two threads: + * - The audio thread for notifications of audio samples. + * - The video thread for compensating drift of video frames to match the audio + * clock. + */ +class DriftCompensator { + const RefPtr mVideoThread; + const TrackRate mAudioRate; + + // Number of audio samples produced. Any thread. + Atomic mAudioSamples{0}; + + // Time the first audio samples were added. mVideoThread only. + TimeStamp mAudioStartTime; + + void SetAudioStartTime(TimeStamp aTime) { + MOZ_ASSERT(mVideoThread->IsOnCurrentThread()); + MOZ_ASSERT(mAudioStartTime.IsNull()); + mAudioStartTime = aTime; + } + + protected: + virtual ~DriftCompensator() = default; + + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DriftCompensator) + + DriftCompensator(RefPtr aVideoThread, TrackRate aAudioRate) + : mVideoThread(std::move(aVideoThread)), mAudioRate(aAudioRate) { + MOZ_ASSERT(mAudioRate > 0); + } + + void NotifyAudioStart(TimeStamp aStart) { + MOZ_ASSERT(mAudioSamples == 0); + LOG(LogLevel::Info, "DriftCompensator %p at rate %d started", this, + mAudioRate); + nsresult rv = mVideoThread->Dispatch(NewRunnableMethod( + "DriftCompensator::SetAudioStartTime", this, + &DriftCompensator::SetAudioStartTime, aStart)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + } + + /** + * aSamples is the number of samples fed by an AudioStream. + */ + void NotifyAudio(TrackTime aSamples) { + MOZ_ASSERT(aSamples > 0); + mAudioSamples += aSamples; + + LOG(LogLevel::Verbose, + "DriftCompensator %p Processed another %" PRId64 + " samples; now %.3fs audio", + this, aSamples, static_cast(mAudioSamples) / mAudioRate); + } + + /** + * Drift compensates a video TimeStamp based on historical audio data. + */ + virtual TimeStamp GetVideoTime(TimeStamp aNow, TimeStamp aTime) { + MOZ_ASSERT(mVideoThread->IsOnCurrentThread()); + TrackTime samples = mAudioSamples; + + if (samples / mAudioRate < 10) { + // We don't apply compensation for the first 10 seconds because of the + // higher inaccuracy during this time. + LOG(LogLevel::Debug, "DriftCompensator %p %" PRId64 "ms so far; ignoring", + this, samples * 1000 / mAudioRate); + return aTime; + } + + if (aNow == mAudioStartTime) { + LOG(LogLevel::Warning, + "DriftCompensator %p video scale 0, assuming no drift", this); + return aTime; + } + + double videoScaleUs = (aNow - mAudioStartTime).ToMicroseconds(); + double audioScaleUs = FramesToUsecs(samples, mAudioRate).value(); + double videoDurationUs = (aTime - mAudioStartTime).ToMicroseconds(); + + TimeStamp reclocked = + mAudioStartTime + TimeDuration::FromMicroseconds( + videoDurationUs * audioScaleUs / videoScaleUs); + + LOG(LogLevel::Debug, + "DriftCompensator %p GetVideoTime, v-now: %.3fs, a-now: %.3fs; %.3fs " + "-> %.3fs (d %.3fms)", + this, (aNow - mAudioStartTime).ToSeconds(), + TimeDuration::FromMicroseconds(audioScaleUs).ToSeconds(), + (aTime - mAudioStartTime).ToSeconds(), + (reclocked - mAudioStartTime).ToSeconds(), + (reclocked - aTime).ToMilliseconds()); + + return reclocked; + } +}; + +#undef LOG + +} // namespace mozilla + +#endif /* DriftCompensation_h_ */ -- cgit v1.2.3