1 files changed, 137 insertions, 0 deletions
diff --git a/dom/media/DriftCompensation.h b/dom/media/DriftCompensation.h
new file mode 100644
index 0000000000..ef22f7106f
--- /dev/null
+++ b/dom/media/DriftCompensation.h
@@ -0,0 +1,137 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#ifndef DriftCompensation_h_
+#define DriftCompensation_h_
+
+#include "MediaSegment.h"
+#include "VideoUtils.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/Unused.h"
+
+namespace mozilla {
+
+static LazyLogModule gDriftCompensatorLog("DriftCompensator");
+#define LOG(type, ...) MOZ_LOG(gDriftCompensatorLog, type, (__VA_ARGS__))
+
+/**
+ * DriftCompensator can be used to handle drift between audio and video tracks
+ * from the MediaTrackGraph.
+ *
+ * Drift can occur because audio is driven by a MediaTrackGraph running off an
+ * audio callback, thus it's progressed by the clock of one the audio output
+ * devices on the user's machine. Video on the other hand is always expressed in
+ * wall-clock TimeStamps, i.e., it's progressed by the system clock. These
+ * clocks will, over time, drift apart.
+ *
+ * Do not use the DriftCompensator across multiple audio tracks, as it will
+ * automatically record the start time of the first audio samples, and all
+ * samples for the same audio track on the same audio clock will have to be
+ * processed to retain accuracy.
+ *
+ * DriftCompensator is designed to be used from two threads:
+ * - The audio thread for notifications of audio samples.
+ * - The video thread for compensating drift of video frames to match the audio
+ *   clock.
+ */
+class DriftCompensator {
+  const RefPtr<nsIEventTarget> mVideoThread;
+  const TrackRate mAudioRate;
+
+  // Number of audio samples produced. Any thread.
+  Atomic<TrackTime> mAudioSamples{0};
+
+  // Time the first audio samples were added. mVideoThread only.
+  TimeStamp mAudioStartTime;
+
+  void SetAudioStartTime(TimeStamp aTime) {
+    MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
+    MOZ_ASSERT(mAudioStartTime.IsNull());
+    mAudioStartTime = aTime;
+  }
+
+ protected:
+  virtual ~DriftCompensator() = default;
+
+ public:
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DriftCompensator)
+
+  DriftCompensator(RefPtr<nsIEventTarget> aVideoThread, TrackRate aAudioRate)
+      : mVideoThread(std::move(aVideoThread)), mAudioRate(aAudioRate) {
+    MOZ_ASSERT(mAudioRate > 0);
+  }
+
+  void NotifyAudioStart(TimeStamp aStart) {
+    MOZ_ASSERT(mAudioSamples == 0);
+    LOG(LogLevel::Info, "DriftCompensator %p at rate %d started", this,
+        mAudioRate);
+    nsresult rv = mVideoThread->Dispatch(NewRunnableMethod<TimeStamp>(
+        "DriftCompensator::SetAudioStartTime", this,
+        &DriftCompensator::SetAudioStartTime, aStart));
+    MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+    Unused << rv;
+  }
+
+  /**
+   * aSamples is the number of samples fed by an AudioStream.
+   */
+  void NotifyAudio(TrackTime aSamples) {
+    MOZ_ASSERT(aSamples > 0);
+    mAudioSamples += aSamples;
+
+    LOG(LogLevel::Verbose,
+        "DriftCompensator %p Processed another %" PRId64
+        " samples; now %.3fs audio",
+        this, aSamples, static_cast<double>(mAudioSamples) / mAudioRate);
+  }
+
+  /**
+   * Drift compensates a video TimeStamp based on historical audio data.
+   */
+  virtual TimeStamp GetVideoTime(TimeStamp aNow, TimeStamp aTime) {
+    MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
+    TrackTime samples = mAudioSamples;
+
+    if (samples / mAudioRate < 10) {
+      // We don't apply compensation for the first 10 seconds because of the
+      // higher inaccuracy during this time.
+      LOG(LogLevel::Debug, "DriftCompensator %p %" PRId64 "ms so far; ignoring",
+          this, samples * 1000 / mAudioRate);
+      return aTime;
+    }
+
+    if (aNow == mAudioStartTime) {
+      LOG(LogLevel::Warning,
+          "DriftCompensator %p video scale 0, assuming no drift", this);
+      return aTime;
+    }
+
+    double videoScaleUs = (aNow - mAudioStartTime).ToMicroseconds();
+    double audioScaleUs = FramesToUsecs(samples, mAudioRate).value();
+    double videoDurationUs = (aTime - mAudioStartTime).ToMicroseconds();
+
+    TimeStamp reclocked =
+        mAudioStartTime + TimeDuration::FromMicroseconds(
+                              videoDurationUs * audioScaleUs / videoScaleUs);
+
+    LOG(LogLevel::Debug,
+        "DriftCompensator %p GetVideoTime, v-now: %.3fs, a-now: %.3fs; %.3fs "
+        "-> %.3fs (d %.3fms)",
+        this, (aNow - mAudioStartTime).ToSeconds(),
+        TimeDuration::FromMicroseconds(audioScaleUs).ToSeconds(),
+        (aTime - mAudioStartTime).ToSeconds(),
+        (reclocked - mAudioStartTime).ToSeconds(),
+        (reclocked - aTime).ToMilliseconds());
+
+    return reclocked;
+  }
+};
+
+#undef LOG
+
+}  // namespace mozilla
+
+#endif /* DriftCompensation_h_ */