summaryrefslogtreecommitdiffstats
path: root/dom/media/mediasink/AudioSinkWrapper.cpp
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--dom/media/mediasink/AudioSinkWrapper.cpp496
1 files changed, 496 insertions, 0 deletions
diff --git a/dom/media/mediasink/AudioSinkWrapper.cpp b/dom/media/mediasink/AudioSinkWrapper.cpp
new file mode 100644
index 0000000000..5a006479e1
--- /dev/null
+++ b/dom/media/mediasink/AudioSinkWrapper.cpp
@@ -0,0 +1,496 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AudioSinkWrapper.h"
+#include "AudioDeviceInfo.h"
+#include "AudioSink.h"
+#include "VideoUtils.h"
+#include "mozilla/Logging.h"
+#include "mozilla/Result.h"
+#include "nsPrintfCString.h"
+
+mozilla::LazyLogModule gAudioSinkWrapperLog("AudioSinkWrapper");
+#define LOG(...) \
+ MOZ_LOG(gAudioSinkWrapperLog, mozilla::LogLevel::Debug, (__VA_ARGS__));
+#define LOGV(...) \
+ MOZ_LOG(gAudioSinkWrapperLog, mozilla::LogLevel::Verbose, (__VA_ARGS__));
+
+namespace mozilla {
+
+using media::TimeUnit;
+
+AudioSinkWrapper::~AudioSinkWrapper() = default;
+
+void AudioSinkWrapper::Shutdown() {
+ AssertOwnerThread();
+ MOZ_ASSERT(!mIsStarted, "Must be called after playback stopped.");
+ mCreator = nullptr;
+ mEndedPromiseHolder.ResolveIfExists(true, __func__);
+}
+
+RefPtr<MediaSink::EndedPromise> AudioSinkWrapper::OnEnded(TrackType aType) {
+ AssertOwnerThread();
+ MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+ if (aType == TrackInfo::kAudioTrack) {
+ return mEndedPromise;
+ }
+ return nullptr;
+}
+
+TimeUnit AudioSinkWrapper::GetEndTime(TrackType aType) const {
+ AssertOwnerThread();
+ MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+ if (aType == TrackInfo::kAudioTrack && mAudioSink &&
+ mAudioSink->AudioStreamCallbackStarted()) {
+ return mAudioSink->GetEndTime();
+ }
+
+ if (aType == TrackInfo::kAudioTrack && !mAudioSink && IsMuted()) {
+ if (IsPlaying()) {
+ return GetSystemClockPosition(TimeStamp::Now());
+ }
+
+ return mPlayDuration;
+ }
+ return TimeUnit::Zero();
+}
+
+TimeUnit AudioSinkWrapper::GetSystemClockPosition(TimeStamp aNow) const {
+ AssertOwnerThread();
+ MOZ_ASSERT(!mPlayStartTime.IsNull());
+ // Time elapsed since we started playing.
+ double delta = (aNow - mPlayStartTime).ToSeconds();
+ // Take playback rate into account.
+ return mPlayDuration + TimeUnit::FromSeconds(delta * mParams.mPlaybackRate);
+}
+
+bool AudioSinkWrapper::IsMuted() const {
+ AssertOwnerThread();
+ return mParams.mVolume == 0.0;
+}
+
+TimeUnit AudioSinkWrapper::GetPosition(TimeStamp* aTimeStamp) {
+ AssertOwnerThread();
+ MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+
+ TimeUnit pos;
+ TimeStamp t = TimeStamp::Now();
+
+ if (!mAudioEnded && !IsMuted() && mAudioSink) {
+ if (mLastClockSource == ClockSource::SystemClock) {
+ TimeUnit switchTime = GetSystemClockPosition(t);
+ // Update the _actual_ start time of the audio stream now that it has
+ // started, preventing any clock discontinuity.
+ mAudioSink->UpdateStartTime(switchTime);
+ LOGV("%p: switching to audio clock at media time %lf", this,
+ switchTime.ToSeconds());
+ }
+ // Rely on the audio sink to report playback position when it is not ended.
+ pos = mAudioSink->GetPosition();
+ LOGV("%p: Getting position from the Audio Sink %lf", this, pos.ToSeconds());
+ mLastClockSource = ClockSource::AudioStream;
+ } else if (!mPlayStartTime.IsNull()) {
+ // Calculate playback position using system clock if we are still playing,
+ // but not rendering the audio, because this audio sink is muted.
+ pos = GetSystemClockPosition(t);
+ LOGV("%p: Getting position from the system clock %lf", this,
+ pos.ToSeconds());
+ if (IsMuted()) {
+ if (mAudioQueue.GetSize() > 0) {
+ // audio track, but it's muted and won't be dequeued, discard packets
+ // that are behind the current media time, to keep the queue size under
+ // control.
+ DropAudioPacketsIfNeeded(pos);
+ }
+ // If muted, it's necessary to manually check if the audio has "ended",
+ // meaning that all the audio packets have been consumed, to resolve the
+ // ended promise.
+ if (CheckIfEnded()) {
+ MOZ_ASSERT(!mAudioSink);
+ mEndedPromiseHolder.ResolveIfExists(true, __func__);
+ }
+ }
+ mLastClockSource = ClockSource::SystemClock;
+ } else {
+ // Return how long we've played if we are not playing.
+ pos = mPlayDuration;
+ LOGV("%p: Getting static position, not playing %lf", this, pos.ToSeconds());
+ mLastClockSource = ClockSource::Paused;
+ }
+
+ if (aTimeStamp) {
+ *aTimeStamp = t;
+ }
+
+ return pos;
+}
+
+bool AudioSinkWrapper::CheckIfEnded() const {
+ return mAudioQueue.IsFinished() && mAudioQueue.GetSize() == 0u;
+}
+
+bool AudioSinkWrapper::HasUnplayedFrames(TrackType aType) const {
+ AssertOwnerThread();
+ return mAudioSink ? mAudioSink->HasUnplayedFrames() : false;
+}
+
+media::TimeUnit AudioSinkWrapper::UnplayedDuration(TrackType aType) const {
+ AssertOwnerThread();
+ return mAudioSink ? mAudioSink->UnplayedDuration() : media::TimeUnit::Zero();
+}
+
+void AudioSinkWrapper::DropAudioPacketsIfNeeded(
+ const TimeUnit& aMediaPosition) {
+ RefPtr<AudioData> audio = mAudioQueue.PeekFront();
+ uint32_t dropped = 0;
+ while (audio && audio->mTime + audio->mDuration < aMediaPosition) {
+ // drop this packet, try the next one
+ audio = mAudioQueue.PopFront();
+ dropped++;
+ if (audio) {
+ LOGV(
+ "Dropping audio packets: media position: %lf, "
+ "packet dropped: [%lf, %lf] (%u so far).\n",
+ aMediaPosition.ToSeconds(), audio->mTime.ToSeconds(),
+ (audio->GetEndTime()).ToSeconds(), dropped);
+ }
+ audio = mAudioQueue.PeekFront();
+ }
+}
+
+void AudioSinkWrapper::OnMuted(bool aMuted) {
+ AssertOwnerThread();
+ LOG("%p: AudioSinkWrapper::OnMuted(%s)", this, aMuted ? "true" : "false");
+ // Nothing to do
+ if (mAudioEnded) {
+ LOG("%p: AudioSinkWrapper::OnMuted, but no audio track", this);
+ return;
+ }
+ if (aMuted) {
+ if (mAudioSink) {
+ LOG("AudioSinkWrapper muted, shutting down AudioStream.");
+ mAudioSinkEndedPromise.DisconnectIfExists();
+ if (IsPlaying()) {
+ mPlayDuration = mAudioSink->GetPosition();
+ mPlayStartTime = TimeStamp::Now();
+ }
+ Maybe<MozPromiseHolder<MediaSink::EndedPromise>> rv =
+ mAudioSink->Shutdown(ShutdownCause::Muting);
+ // There will generally be a promise here, except if the stream has
+ // errored out, or if it has just finished. In both cases, the promise has
+ // been handled appropriately, there is nothing to do.
+ if (rv.isSome()) {
+ mEndedPromiseHolder = std::move(rv.ref());
+ }
+ mAudioSink = nullptr;
+ }
+ } else {
+ if (!IsPlaying()) {
+ LOG("%p: AudioSinkWrapper::OnMuted: not playing, not re-creating an "
+ "AudioSink",
+ this);
+ return;
+ }
+ LOG("%p: AudioSinkWrapper unmuted, re-creating an AudioStream.", this);
+ TimeUnit mediaPosition = GetSystemClockPosition(TimeStamp::Now());
+ nsresult rv = StartAudioSink(mediaPosition, AudioSinkStartPolicy::ASYNC);
+ if (NS_FAILED(rv)) {
+ NS_WARNING(
+ "Could not start AudioSink from AudioSinkWrapper when unmuting");
+ }
+ }
+}
+
+void AudioSinkWrapper::SetVolume(double aVolume) {
+ AssertOwnerThread();
+
+ bool wasMuted = mParams.mVolume == 0;
+ bool nowMuted = aVolume == 0.;
+ mParams.mVolume = aVolume;
+
+ if (!wasMuted && nowMuted) {
+ OnMuted(true);
+ } else if (wasMuted && !nowMuted) {
+ OnMuted(false);
+ }
+
+ if (mAudioSink) {
+ mAudioSink->SetVolume(aVolume);
+ }
+}
+
+void AudioSinkWrapper::SetStreamName(const nsAString& aStreamName) {
+ AssertOwnerThread();
+ if (mAudioSink) {
+ mAudioSink->SetStreamName(aStreamName);
+ }
+}
+
+void AudioSinkWrapper::SetPlaybackRate(double aPlaybackRate) {
+ AssertOwnerThread();
+ if (!mAudioEnded && mAudioSink) {
+ // Pass the playback rate to the audio sink. The underlying AudioStream
+ // will handle playback rate changes and report correct audio position.
+ mAudioSink->SetPlaybackRate(aPlaybackRate);
+ } else if (!mPlayStartTime.IsNull()) {
+ // Adjust playback duration and start time when we are still playing.
+ TimeStamp now = TimeStamp::Now();
+ mPlayDuration = GetSystemClockPosition(now);
+ mPlayStartTime = now;
+ }
+ // mParams.mPlaybackRate affects GetSystemClockPosition(). It should be
+ // updated after the calls to GetSystemClockPosition();
+ mParams.mPlaybackRate = aPlaybackRate;
+
+ // Do nothing when not playing. Changes in playback rate will be taken into
+ // account by GetSystemClockPosition().
+}
+
+void AudioSinkWrapper::SetPreservesPitch(bool aPreservesPitch) {
+ AssertOwnerThread();
+ mParams.mPreservesPitch = aPreservesPitch;
+ if (mAudioSink) {
+ mAudioSink->SetPreservesPitch(aPreservesPitch);
+ }
+}
+
+void AudioSinkWrapper::SetPlaying(bool aPlaying) {
+ AssertOwnerThread();
+ LOG("%p: AudioSinkWrapper::SetPlaying %s", this, aPlaying ? "true" : "false");
+
+ // Resume/pause matters only when playback started.
+ if (!mIsStarted) {
+ return;
+ }
+
+ if (mAudioSink) {
+ mAudioSink->SetPlaying(aPlaying);
+ } else {
+ if (aPlaying) {
+ LOG("%p: AudioSinkWrapper::SetPlaying : starting an AudioSink", this);
+ TimeUnit switchTime = GetPosition();
+ DropAudioPacketsIfNeeded(switchTime);
+ StartAudioSink(switchTime, AudioSinkStartPolicy::SYNC);
+ }
+ }
+
+ if (aPlaying) {
+ MOZ_ASSERT(mPlayStartTime.IsNull());
+ mPlayStartTime = TimeStamp::Now();
+ } else {
+ // Remember how long we've played.
+ mPlayDuration = GetPosition();
+ // mPlayStartTime must be updated later since GetPosition()
+ // depends on the value of mPlayStartTime.
+ mPlayStartTime = TimeStamp();
+ }
+}
+
+double AudioSinkWrapper::PlaybackRate() const {
+ AssertOwnerThread();
+ return mParams.mPlaybackRate;
+}
+
+nsresult AudioSinkWrapper::Start(const TimeUnit& aStartTime,
+ const MediaInfo& aInfo) {
+ LOG("%p AudioSinkWrapper::Start", this);
+ AssertOwnerThread();
+ MOZ_ASSERT(!mIsStarted, "playback already started.");
+
+ mIsStarted = true;
+ mPlayDuration = aStartTime;
+ mPlayStartTime = TimeStamp::Now();
+ mAudioEnded = IsAudioSourceEnded(aInfo);
+
+ if (mAudioEnded) {
+ // Resolve promise if we start playback at the end position of the audio.
+ mEndedPromise =
+ aInfo.HasAudio()
+ ? MediaSink::EndedPromise::CreateAndResolve(true, __func__)
+ : nullptr;
+ return NS_OK;
+ }
+
+ return StartAudioSink(aStartTime, AudioSinkStartPolicy::SYNC);
+}
+
+nsresult AudioSinkWrapper::StartAudioSink(const TimeUnit& aStartTime,
+ AudioSinkStartPolicy aPolicy) {
+ MOZ_RELEASE_ASSERT(!mAudioSink);
+
+ nsresult rv = NS_OK;
+
+ mAudioSinkEndedPromise.DisconnectIfExists();
+ mEndedPromise = mEndedPromiseHolder.Ensure(__func__);
+ mEndedPromise
+ ->Then(mOwnerThread.get(), __func__, this,
+ &AudioSinkWrapper::OnAudioEnded, &AudioSinkWrapper::OnAudioEnded)
+ ->Track(mAudioSinkEndedPromise);
+
+ LOG("%p: AudioSinkWrapper::StartAudioSink (%s)", this,
+ aPolicy == AudioSinkStartPolicy::ASYNC ? "Async" : "Sync");
+
+ if (IsMuted()) {
+ LOG("%p: Muted: not starting an audio sink", this);
+ return NS_OK;
+ }
+ LOG("%p: Not muted: starting a new audio sink", this);
+ if (aPolicy == AudioSinkStartPolicy::ASYNC) {
+ UniquePtr<AudioSink> audioSink;
+ audioSink.reset(mCreator->Create());
+ NS_DispatchBackgroundTask(NS_NewRunnableFunction(
+ "StartAudioSink (Async part: initialization)",
+ [self = RefPtr<AudioSinkWrapper>(this), audioSink{std::move(audioSink)},
+ this]() mutable {
+ LOG("AudioSink initialization on background thread");
+ // This can take about 200ms, e.g. on Windows, we don't want to do
+ // it on the MDSM thread, because it would make the clock not update
+ // for that amount of time, and the video would therefore not
+ // update. The Start() call is very cheap on the other hand, we can
+ // do it from the MDSM thread.
+ nsresult rv = audioSink->InitializeAudioStream(
+ mParams, mAudioDevice, AudioSink::InitializationType::UNMUTING);
+ mOwnerThread->Dispatch(NS_NewRunnableFunction(
+ "StartAudioSink (Async part: start from MDSM thread)",
+ [self = RefPtr<AudioSinkWrapper>(this),
+ audioSink{std::move(audioSink)}, this, rv]() mutable {
+ LOG("AudioSink async init done, back on MDSM thread");
+ if (NS_FAILED(rv)) {
+ LOG("Async AudioSink initialization failed");
+ mEndedPromiseHolder.RejectIfExists(rv, __func__);
+ return;
+ }
+
+ // It's possible that the newly created isn't needed at this
+ // point, in some cases:
+ // 1. An AudioSink was created synchronously while this
+ // AudioSink was initialized asynchronously, bail out here. This
+ // happens when seeking (which does a synchronous
+ // initialization) right after unmuting.
+ // 2. The media element was muted while the async initialization
+ // was happening.
+ // 3. The AudioSinkWrapper was stopped during asynchronous
+ // creation.
+ // 4. The AudioSinkWrapper was paused during asynchronous
+ // creation.
+ if (mAudioSink || IsMuted() || !mIsStarted ||
+ mPlayStartTime.IsNull()) {
+ LOG("AudioSink initialized async isn't needed, shutting "
+ "it down.");
+ DebugOnly<Maybe<MozPromiseHolder<EndedPromise>>> rv =
+ audioSink->Shutdown();
+ MOZ_ASSERT(rv.inspect().isNothing());
+ return;
+ }
+
+ MOZ_ASSERT(!mAudioSink);
+ TimeUnit switchTime = GetPosition();
+ DropAudioPacketsIfNeeded(switchTime);
+ mAudioSink.swap(audioSink);
+ if (mTreatUnderrunAsSilence) {
+ mAudioSink->EnableTreatAudioUnderrunAsSilence(
+ mTreatUnderrunAsSilence);
+ }
+ LOG("AudioSink async, start");
+ nsresult rv2 =
+ mAudioSink->Start(switchTime, mEndedPromiseHolder);
+ if (NS_FAILED(rv2)) {
+ LOG("Async AudioSinkWrapper start failed");
+ mEndedPromiseHolder.RejectIfExists(rv2, __func__);
+ }
+ }));
+ }));
+ } else {
+ mAudioSink.reset(mCreator->Create());
+ nsresult rv = mAudioSink->InitializeAudioStream(
+ mParams, mAudioDevice, AudioSink::InitializationType::INITIAL);
+ if (NS_FAILED(rv)) {
+ mEndedPromiseHolder.RejectIfExists(rv, __func__);
+ LOG("Sync AudioSinkWrapper initialization failed");
+ return rv;
+ }
+ if (mTreatUnderrunAsSilence) {
+ mAudioSink->EnableTreatAudioUnderrunAsSilence(mTreatUnderrunAsSilence);
+ }
+ rv = mAudioSink->Start(aStartTime, mEndedPromiseHolder);
+ if (NS_FAILED(rv)) {
+ LOG("Sync AudioSinkWrapper start failed");
+ mEndedPromiseHolder.RejectIfExists(rv, __func__);
+ }
+ }
+
+ return rv;
+}
+
+bool AudioSinkWrapper::IsAudioSourceEnded(const MediaInfo& aInfo) const {
+ // no audio or empty audio queue which won't get data anymore is equivalent to
+ // audio ended
+ return !aInfo.HasAudio() ||
+ (mAudioQueue.IsFinished() && mAudioQueue.GetSize() == 0u);
+}
+
+void AudioSinkWrapper::Stop() {
+ AssertOwnerThread();
+ MOZ_ASSERT(mIsStarted, "playback not started.");
+
+ LOG("%p: AudioSinkWrapper::Stop", this);
+
+ mIsStarted = false;
+ mAudioEnded = true;
+
+ mAudioSinkEndedPromise.DisconnectIfExists();
+
+ if (mAudioSink) {
+ DebugOnly<Maybe<MozPromiseHolder<EndedPromise>>> rv =
+ mAudioSink->Shutdown();
+ MOZ_ASSERT(rv.inspect().isNothing());
+ mAudioSink = nullptr;
+ mEndedPromise = nullptr;
+ }
+}
+
+bool AudioSinkWrapper::IsStarted() const {
+ AssertOwnerThread();
+ return mIsStarted;
+}
+
+bool AudioSinkWrapper::IsPlaying() const {
+ AssertOwnerThread();
+ return IsStarted() && !mPlayStartTime.IsNull();
+}
+
+void AudioSinkWrapper::OnAudioEnded() {
+ AssertOwnerThread();
+ LOG("%p: AudioSinkWrapper::OnAudioEnded", this);
+ mAudioSinkEndedPromise.Complete();
+ mPlayDuration = GetPosition();
+ if (!mPlayStartTime.IsNull()) {
+ mPlayStartTime = TimeStamp::Now();
+ }
+ mAudioEnded = true;
+}
+
+void AudioSinkWrapper::GetDebugInfo(dom::MediaSinkDebugInfo& aInfo) {
+ AssertOwnerThread();
+ aInfo.mAudioSinkWrapper.mIsPlaying = IsPlaying();
+ aInfo.mAudioSinkWrapper.mIsStarted = IsStarted();
+ aInfo.mAudioSinkWrapper.mAudioEnded = mAudioEnded;
+ if (mAudioSink) {
+ mAudioSink->GetDebugInfo(aInfo);
+ }
+}
+
+void AudioSinkWrapper::EnableTreatAudioUnderrunAsSilence(bool aEnabled) {
+ mTreatUnderrunAsSilence = aEnabled;
+ if (mAudioSink) {
+ mAudioSink->EnableTreatAudioUnderrunAsSilence(aEnabled);
+ }
+}
+
+} // namespace mozilla
+
+#undef LOG
+#undef LOGV