1 files changed, 579 insertions, 0 deletions
diff --git a/dom/media/mediasink/AudioSinkWrapper.cpp b/dom/media/mediasink/AudioSinkWrapper.cpp
new file mode 100644
index 0000000000..909de78fd9
--- /dev/null
+++ b/dom/media/mediasink/AudioSinkWrapper.cpp
@@ -0,0 +1,579 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AudioSinkWrapper.h"
+#include "AudioDeviceInfo.h"
+#include "AudioSink.h"
+#include "VideoUtils.h"
+#include "mozilla/Logging.h"
+#include "mozilla/Result.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "nsPrintfCString.h"
+#include "nsThreadManager.h"
+
+mozilla::LazyLogModule gAudioSinkWrapperLog("AudioSinkWrapper");
+#define LOG(...) \
+  MOZ_LOG(gAudioSinkWrapperLog, mozilla::LogLevel::Debug, (__VA_ARGS__));
+#define LOGV(...) \
+  MOZ_LOG(gAudioSinkWrapperLog, mozilla::LogLevel::Verbose, (__VA_ARGS__));
+
+namespace mozilla {
+
+using media::TimeUnit;
+
+AudioSinkWrapper::~AudioSinkWrapper() = default;
+
+void AudioSinkWrapper::Shutdown() {
+  AssertOwnerThread();
+  MOZ_ASSERT(!mIsStarted, "Must be called after playback stopped.");
+  mSinkCreator = nullptr;
+}
+
+/* static */
+already_AddRefed<TaskQueue> AudioSinkWrapper::CreateAsyncInitTaskQueue() {
+  return nsThreadManager::get().CreateBackgroundTaskQueue("AsyncAudioSinkInit");
+}
+
+RefPtr<MediaSink::EndedPromise> AudioSinkWrapper::OnEnded(TrackType aType) {
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+  if (aType == TrackInfo::kAudioTrack) {
+    return mEndedPromise;
+  }
+  return nullptr;
+}
+
+TimeUnit AudioSinkWrapper::GetEndTime(TrackType aType) const {
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+  if (aType != TrackInfo::kAudioTrack) {
+    return TimeUnit::Zero();
+  }
+
+  if (mAudioSink && mAudioSink->AudioStreamCallbackStarted()) {
+    auto time = mAudioSink->GetEndTime();
+    LOGV("%p: GetEndTime return %lf from sink", this, time.ToSeconds());
+    return time;
+  }
+
+  RefPtr<const AudioData> audio = mAudioQueue.PeekBack();
+  if (audio) {
+    LOGV("%p: GetEndTime return %lf from queue", this,
+         audio->GetEndTime().ToSeconds());
+    return audio->GetEndTime();
+  }
+
+  LOGV("%p: GetEndTime return %lf from last packet", this,
+       mLastPacketEndTime.ToSeconds());
+  return mLastPacketEndTime;
+}
+
+TimeUnit AudioSinkWrapper::GetSystemClockPosition(TimeStamp aNow) const {
+  AssertOwnerThread();
+  MOZ_ASSERT(!mClockStartTime.IsNull());
+  // Time elapsed since we started playing.
+  double delta = (aNow - mClockStartTime).ToSeconds();
+  // Take playback rate into account.
+  return mPositionAtClockStart +
+         TimeUnit::FromSeconds(delta * mParams.mPlaybackRate);
+}
+
+bool AudioSinkWrapper::IsMuted() const {
+  AssertOwnerThread();
+  return mParams.mVolume == 0.0;
+}
+
+TimeUnit AudioSinkWrapper::GetPosition(TimeStamp* aTimeStamp) {
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+
+  TimeUnit pos;
+  TimeStamp t = TimeStamp::Now();
+
+  if (mAudioSink) {
+    if (mLastClockSource == ClockSource::SystemClock) {
+      TimeUnit switchTime = GetSystemClockPosition(t);
+      // Update the _actual_ start time of the audio stream now that it has
+      // started, preventing any clock discontinuity.
+      mAudioSink->UpdateStartTime(switchTime);
+      LOGV("%p: switching to audio clock at media time %lf", this,
+           switchTime.ToSeconds());
+    }
+    // Rely on the audio sink to report playback position when it is not ended.
+    pos = mAudioSink->GetPosition();
+    LOGV("%p: Getting position from the Audio Sink %lf", this, pos.ToSeconds());
+    mLastClockSource = ClockSource::AudioStream;
+  } else if (!mClockStartTime.IsNull()) {
+    // Calculate playback position using system clock if we are still playing,
+    // but not rendering the audio, because this audio sink is muted.
+    pos = GetSystemClockPosition(t);
+    LOGV("%p: Getting position from the system clock %lf", this,
+         pos.ToSeconds());
+    if (mAudioQueue.GetSize() > 0) {
+      // Audio track, but it won't be dequeued.  Discard packets
+      // that are behind the current media time, to keep the queue size under
+      // control.
+      DropAudioPacketsIfNeeded(pos);
+    }
+    // Without an AudioSink, it's necessary to manually check if the audio has
+    // "ended", meaning that all the audio packets have been consumed,
+    // to resolve the ended promise.
+    if (CheckIfEnded()) {
+      MOZ_ASSERT(!mAudioSink);
+      mEndedPromiseHolder.ResolveIfExists(true, __func__);
+    }
+    mLastClockSource = ClockSource::SystemClock;
+
+    if (!mAudioSink && mAsyncCreateCount == 0 && NeedAudioSink() &&
+        t > mRetrySinkTime) {
+      MaybeAsyncCreateAudioSink(mAudioDevice);
+    }
+  } else {
+    // Return how long we've played if we are not playing.
+    pos = mPositionAtClockStart;
+    LOGV("%p: Getting static position, not playing %lf", this, pos.ToSeconds());
+    mLastClockSource = ClockSource::Paused;
+  }
+
+  if (aTimeStamp) {
+    *aTimeStamp = t;
+  }
+
+  return pos;
+}
+
+bool AudioSinkWrapper::CheckIfEnded() const {
+  return mAudioQueue.IsFinished() && mAudioQueue.GetSize() == 0u;
+}
+
+bool AudioSinkWrapper::HasUnplayedFrames(TrackType aType) const {
+  AssertOwnerThread();
+  return mAudioSink ? mAudioSink->HasUnplayedFrames() : false;
+}
+
+media::TimeUnit AudioSinkWrapper::UnplayedDuration(TrackType aType) const {
+  AssertOwnerThread();
+  return mAudioSink ? mAudioSink->UnplayedDuration() : media::TimeUnit::Zero();
+}
+
+void AudioSinkWrapper::DropAudioPacketsIfNeeded(
+    const TimeUnit& aMediaPosition) {
+  RefPtr<AudioData> audio = mAudioQueue.PeekFront();
+  uint32_t dropped = 0;
+  while (audio && audio->GetEndTime() < aMediaPosition) {
+    // drop this packet, try the next one
+    audio = mAudioQueue.PopFront();
+    dropped++;
+    if (audio) {
+      mLastPacketEndTime = audio->GetEndTime();
+      LOGV(
+          "Dropping audio packets: media position: %lf, "
+          "packet dropped: [%lf, %lf] (%u so far).\n",
+          aMediaPosition.ToSeconds(), audio->mTime.ToSeconds(),
+          (audio->GetEndTime()).ToSeconds(), dropped);
+    }
+    audio = mAudioQueue.PeekFront();
+  }
+}
+
+void AudioSinkWrapper::OnMuted(bool aMuted) {
+  AssertOwnerThread();
+  LOG("%p: AudioSinkWrapper::OnMuted(%s)", this, aMuted ? "true" : "false");
+  // Nothing to do
+  if (mAudioEnded) {
+    LOG("%p: AudioSinkWrapper::OnMuted, but no audio track", this);
+    return;
+  }
+  if (aMuted) {
+    if (mAudioSink) {
+      LOG("AudioSinkWrapper muted, shutting down AudioStream.");
+      ShutDownAudioSink();
+    }
+  } else {
+    LOG("%p: AudioSinkWrapper unmuted, maybe re-creating an AudioStream.",
+        this);
+    MaybeAsyncCreateAudioSink(mAudioDevice);
+  }
+}
+
+void AudioSinkWrapper::SetVolume(double aVolume) {
+  AssertOwnerThread();
+
+  bool wasMuted = mParams.mVolume == 0;
+  bool nowMuted = aVolume == 0.;
+  mParams.mVolume = aVolume;
+
+  if (!wasMuted && nowMuted) {
+    OnMuted(true);
+  } else if (wasMuted && !nowMuted) {
+    OnMuted(false);
+  }
+
+  if (mAudioSink) {
+    mAudioSink->SetVolume(aVolume);
+  }
+}
+
+void AudioSinkWrapper::SetStreamName(const nsAString& aStreamName) {
+  AssertOwnerThread();
+  if (mAudioSink) {
+    mAudioSink->SetStreamName(aStreamName);
+  }
+}
+
+void AudioSinkWrapper::SetPlaybackRate(double aPlaybackRate) {
+  AssertOwnerThread();
+  if (mAudioSink) {
+    // Pass the playback rate to the audio sink. The underlying AudioStream
+    // will handle playback rate changes and report correct audio position.
+    mAudioSink->SetPlaybackRate(aPlaybackRate);
+  } else if (!mClockStartTime.IsNull()) {
+    // Adjust playback duration and start time when we are still playing.
+    TimeStamp now = TimeStamp::Now();
+    mPositionAtClockStart = GetSystemClockPosition(now);
+    mClockStartTime = now;
+  }
+  // mParams.mPlaybackRate affects GetSystemClockPosition(). It should be
+  // updated after the calls to GetSystemClockPosition();
+  mParams.mPlaybackRate = aPlaybackRate;
+
+  // Do nothing when not playing. Changes in playback rate will be taken into
+  // account by GetSystemClockPosition().
+}
+
+void AudioSinkWrapper::SetPreservesPitch(bool aPreservesPitch) {
+  AssertOwnerThread();
+  mParams.mPreservesPitch = aPreservesPitch;
+  if (mAudioSink) {
+    mAudioSink->SetPreservesPitch(aPreservesPitch);
+  }
+}
+
+void AudioSinkWrapper::SetPlaying(bool aPlaying) {
+  AssertOwnerThread();
+  LOG("%p: AudioSinkWrapper::SetPlaying %s", this, aPlaying ? "true" : "false");
+
+  // Resume/pause matters only when playback started.
+  if (!mIsStarted) {
+    return;
+  }
+
+  if (mAudioSink) {
+    mAudioSink->SetPlaying(aPlaying);
+  }
+
+  if (aPlaying) {
+    MOZ_ASSERT(mClockStartTime.IsNull());
+    TimeUnit switchTime = GetPosition();
+    mClockStartTime = TimeStamp::Now();
+    if (!mAudioSink && NeedAudioSink()) {
+      LOG("%p: AudioSinkWrapper::SetPlaying : starting an AudioSink", this);
+      DropAudioPacketsIfNeeded(switchTime);
+      SyncCreateAudioSink(switchTime);
+    }
+  } else {
+    // Remember how long we've played.
+    mPositionAtClockStart = GetPosition();
+    // mClockStartTime must be updated later since GetPosition()
+    // depends on the value of mClockStartTime.
+    mClockStartTime = TimeStamp();
+  }
+}
+
+RefPtr<GenericPromise> AudioSinkWrapper::SetAudioDevice(
+    RefPtr<AudioDeviceInfo> aDevice) {
+  return MaybeAsyncCreateAudioSink(std::move(aDevice));
+}
+
+double AudioSinkWrapper::PlaybackRate() const {
+  AssertOwnerThread();
+  return mParams.mPlaybackRate;
+}
+
+nsresult AudioSinkWrapper::Start(const TimeUnit& aStartTime,
+                                 const MediaInfo& aInfo) {
+  LOG("%p AudioSinkWrapper::Start", this);
+  AssertOwnerThread();
+  MOZ_ASSERT(!mIsStarted, "playback already started.");
+
+  mIsStarted = true;
+  mPositionAtClockStart = aStartTime;
+  mClockStartTime = TimeStamp::Now();
+  mAudioEnded = IsAudioSourceEnded(aInfo);
+  mLastPacketEndTime = TimeUnit::Zero();
+
+  if (mAudioEnded) {
+    // Resolve promise if we start playback at the end position of the audio.
+    mEndedPromise =
+        aInfo.HasAudio()
+            ? MediaSink::EndedPromise::CreateAndResolve(true, __func__)
+            : nullptr;
+    return NS_OK;
+  }
+
+  mEndedPromise = mEndedPromiseHolder.Ensure(__func__);
+  if (!NeedAudioSink()) {
+    return NS_OK;
+  }
+  return SyncCreateAudioSink(aStartTime);
+}
+
+bool AudioSinkWrapper::NeedAudioSink() {
+  // An AudioSink is needed if unmuted, playing, and not ended.  The not-ended
+  // check also avoids creating an AudioSink when there is no audio track.
+  return !IsMuted() && IsPlaying() && !mEndedPromiseHolder.IsEmpty();
+}
+
+void AudioSinkWrapper::StartAudioSink(UniquePtr<AudioSink> aAudioSink,
+                                      const TimeUnit& aStartTime) {
+  AssertOwnerThread();
+  MOZ_ASSERT(!mAudioSink);
+  mAudioSink = std::move(aAudioSink);
+  mAudioSink->Start(aStartTime)
+      ->Then(mOwnerThread.get(), __func__, this,
+             &AudioSinkWrapper::OnAudioEnded)
+      ->Track(mAudioSinkEndedRequest);
+}
+
+void AudioSinkWrapper::ShutDownAudioSink() {
+  AssertOwnerThread();
+  mAudioSinkEndedRequest.DisconnectIfExists();
+  if (IsPlaying()) {
+    mPositionAtClockStart = mAudioSink->GetPosition();
+    mClockStartTime = TimeStamp::Now();
+  }
+  mAudioSink->ShutDown();
+  mLastPacketEndTime = mAudioSink->GetEndTime();
+  mAudioSink = nullptr;
+}
+
+RefPtr<GenericPromise> AudioSinkWrapper::MaybeAsyncCreateAudioSink(
+    RefPtr<AudioDeviceInfo> aDevice) {
+  AssertOwnerThread();
+  UniquePtr<AudioSink> audioSink;
+  if (NeedAudioSink() && (!mAudioSink || aDevice != mAudioDevice)) {
+    LOG("%p: AudioSinkWrapper::MaybeAsyncCreateAudioSink: AudioSink needed",
+        this);
+    if (mAudioSink) {
+      ShutDownAudioSink();
+    }
+    audioSink = mSinkCreator();
+  } else {
+    LOG("%p: AudioSinkWrapper::MaybeAsyncCreateAudioSink: no AudioSink change",
+        this);
+    // Bounce off the background thread to keep promise resolution in order.
+  }
+  mAudioDevice = std::move(aDevice);
+  ++mAsyncCreateCount;
+  using Promise =
+      MozPromise<UniquePtr<AudioSink>, nsresult, /* IsExclusive = */ true>;
+  return InvokeAsync(mAsyncInitTaskQueue,
+                     "MaybeAsyncCreateAudioSink (Async part: initialization)",
+                     [self = RefPtr<AudioSinkWrapper>(this),
+                      audioSink{std::move(audioSink)},
+                      audioDevice = mAudioDevice, this]() mutable {
+                       if (!audioSink || !mAsyncInitTaskQueue->IsEmpty()) {
+                         // Either an AudioSink is not required or there's a
+                         // pending task to init an AudioSink with a possibly
+                         // different device.
+                         return Promise::CreateAndResolve(nullptr, __func__);
+                       }
+
+                       LOG("AudioSink initialization on background thread");
+                       // This can take about 200ms, e.g. on Windows, we don't
+                       // want to do it on the MDSM thread, because it would
+                       // make the clock not update for that amount of time, and
+                       // the video would therefore not update. The Start() call
+                       // is very cheap on the other hand, we can do it from the
+                       // MDSM thread.
+                       nsresult rv = audioSink->InitializeAudioStream(
+                           mParams, audioDevice,
+                           AudioSink::InitializationType::UNMUTING);
+                       if (NS_FAILED(rv)) {
+                         LOG("Async AudioSink initialization failed");
+                         return Promise::CreateAndReject(rv, __func__);
+                       }
+                       return Promise::CreateAndResolve(std::move(audioSink),
+                                                        __func__);
+                     })
+      ->Then(
+          mOwnerThread,
+          "MaybeAsyncCreateAudioSink (Async part: start from MDSM thread)",
+          [self = RefPtr<AudioSinkWrapper>(this), audioDevice = mAudioDevice,
+           this](Promise::ResolveOrRejectValue&& aValue) mutable {
+            LOG("AudioSink async init done, back on MDSM thread");
+            --mAsyncCreateCount;
+            UniquePtr<AudioSink> audioSink;
+            if (aValue.IsResolve()) {
+              audioSink = std::move(aValue.ResolveValue());
+            }
+            // It's possible that the newly created AudioSink isn't needed at
+            // this point, in some cases:
+            // 1. An AudioSink was created synchronously while this
+            // AudioSink was initialized asynchronously, bail out here. This
+            // happens when seeking (which does a synchronous initialization)
+            // right after unmuting.  mEndedPromiseHolder is managed by the
+            // other AudioSink, so don't touch it here.
+            // 2. The media element was muted while the async initialization
+            // was happening.
+            // 3. The AudioSinkWrapper was paused or stopped during
+            // asynchronous initialization.
+            // 4. The audio has ended during asynchronous initialization.
+            // 5. A change to a potentially different sink device is pending.
+            if (mAudioSink || !NeedAudioSink() || audioDevice != mAudioDevice) {
+              LOG("AudioSink async initialization isn't needed.");
+              if (audioSink) {
+                LOG("Shutting down unneeded AudioSink.");
+                audioSink->ShutDown();
+              }
+              return GenericPromise::CreateAndResolve(true, __func__);
+            }
+
+            if (aValue.IsReject()) {
+              if (audioDevice) {
+                // Device will be started when available again.
+                ScheduleRetrySink();
+              } else {
+                // Default device not available.  Report error.
+                MOZ_ASSERT(!mAudioSink);
+                mEndedPromiseHolder.RejectIfExists(aValue.RejectValue(),
+                                                   __func__);
+              }
+              return GenericPromise::CreateAndResolve(true, __func__);
+            }
+
+            if (!audioSink) {
+              // No-op because either an existing AudioSink was suitable or no
+              // AudioSink was needed when MaybeAsyncCreateAudioSink() set up
+              // this task.  We now need a new AudioSink, but that will be
+              // handled by another task, either already pending or a delayed
+              // retry task yet to be created by GetPosition().
+              return GenericPromise::CreateAndResolve(true, __func__);
+            }
+
+            MOZ_ASSERT(!mAudioSink);
+            // Avoiding the side effects of GetPosition() creating another
+            // sink another AudioSink and resolving mEndedPromiseHolder, which
+            // the new audioSink will now manage.
+            TimeUnit switchTime = GetSystemClockPosition(TimeStamp::Now());
+            DropAudioPacketsIfNeeded(switchTime);
+            mLastClockSource = ClockSource::SystemClock;
+
+            LOG("AudioSink async, start");
+            StartAudioSink(std::move(audioSink), switchTime);
+            return GenericPromise::CreateAndResolve(true, __func__);
+          });
+}
+
+nsresult AudioSinkWrapper::SyncCreateAudioSink(const TimeUnit& aStartTime) {
+  AssertOwnerThread();
+  MOZ_ASSERT(!mAudioSink);
+  MOZ_ASSERT(!mAudioSinkEndedRequest.Exists());
+
+  LOG("%p: AudioSinkWrapper::SyncCreateAudioSink(%lf)", this,
+      aStartTime.ToSeconds());
+
+  UniquePtr<AudioSink> audioSink = mSinkCreator();
+  nsresult rv = audioSink->InitializeAudioStream(
+      mParams, mAudioDevice, AudioSink::InitializationType::INITIAL);
+  if (NS_FAILED(rv)) {
+    LOG("Sync AudioSinkWrapper initialization failed");
+    // If a specific device has been specified through setSinkId()
+    // the sink is started after the device becomes available again.
+    if (mAudioDevice) {
+      ScheduleRetrySink();
+      return NS_OK;
+    }
+    // If a default output device is not available, the system may not support
+    // audio output.  Report an error so that playback can be aborted if there
+    // is no video.
+    mEndedPromiseHolder.RejectIfExists(rv, __func__);
+    return rv;
+  }
+  StartAudioSink(std::move(audioSink), aStartTime);
+
+  return NS_OK;
+}
+
+void AudioSinkWrapper::ScheduleRetrySink() {
+  mRetrySinkTime =
+      TimeStamp::Now() + TimeDuration::FromMilliseconds(
+                             StaticPrefs::media_audio_device_retry_ms());
+}
+
+bool AudioSinkWrapper::IsAudioSourceEnded(const MediaInfo& aInfo) const {
+  // no audio or empty audio queue which won't get data anymore is equivalent to
+  // audio ended
+  return !aInfo.HasAudio() ||
+         (mAudioQueue.IsFinished() && mAudioQueue.GetSize() == 0u);
+}
+
+void AudioSinkWrapper::Stop() {
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "playback not started.");
+
+  LOG("%p: AudioSinkWrapper::Stop", this);
+
+  mIsStarted = false;
+  mClockStartTime = TimeStamp();
+  mPositionAtClockStart = TimeUnit::Invalid();
+  mAudioEnded = true;
+  if (mAudioSink) {
+    ShutDownAudioSink();
+  }
+
+  mEndedPromiseHolder.ResolveIfExists(true, __func__);
+  mEndedPromise = nullptr;
+}
+
+bool AudioSinkWrapper::IsStarted() const {
+  AssertOwnerThread();
+  return mIsStarted;
+}
+
+bool AudioSinkWrapper::IsPlaying() const {
+  AssertOwnerThread();
+  MOZ_ASSERT(mClockStartTime.IsNull() || IsStarted());
+  return !mClockStartTime.IsNull();
+}
+
+void AudioSinkWrapper::OnAudioEnded(
+    const EndedPromise::ResolveOrRejectValue& aValue) {
+  AssertOwnerThread();
+  // This callback on mAudioSinkEndedRequest should have been disconnected if
+  // mEndedPromiseHolder has been settled.
+  MOZ_ASSERT(!mEndedPromiseHolder.IsEmpty());
+  LOG("%p: AudioSinkWrapper::OnAudioEnded %i", this, aValue.IsResolve());
+  mAudioSinkEndedRequest.Complete();
+  ShutDownAudioSink();
+  // System time is now used for the clock as video may not have ended.
+  if (aValue.IsResolve()) {
+    mAudioEnded = true;
+    mEndedPromiseHolder.Resolve(aValue.ResolveValue(), __func__);
+    return;
+  }
+  if (mAudioDevice) {
+    ScheduleRetrySink();  // Device will be restarted when available again.
+    return;
+  }
+  // Default device not available.  Report error.
+  mEndedPromiseHolder.Reject(aValue.RejectValue(), __func__);
+}
+
+void AudioSinkWrapper::GetDebugInfo(dom::MediaSinkDebugInfo& aInfo) {
+  AssertOwnerThread();
+  aInfo.mAudioSinkWrapper.mIsPlaying = IsPlaying();
+  aInfo.mAudioSinkWrapper.mIsStarted = IsStarted();
+  aInfo.mAudioSinkWrapper.mAudioEnded = mAudioEnded;
+  if (mAudioSink) {
+    mAudioSink->GetDebugInfo(aInfo);
+  }
+}
+
+}  // namespace mozilla
+
+#undef LOG
+#undef LOGV