summaryrefslogtreecommitdiffstats
path: root/dom/media/MediaTrackGraph.cpp
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /dom/media/MediaTrackGraph.cpp
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/MediaTrackGraph.cpp')
-rw-r--r--dom/media/MediaTrackGraph.cpp4318
1 files changed, 4318 insertions, 0 deletions
diff --git a/dom/media/MediaTrackGraph.cpp b/dom/media/MediaTrackGraph.cpp
new file mode 100644
index 0000000000..157ad403d2
--- /dev/null
+++ b/dom/media/MediaTrackGraph.cpp
@@ -0,0 +1,4318 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaTrackGraphImpl.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Unused.h"
+
+#include "AudioSegment.h"
+#include "CrossGraphPort.h"
+#include "VideoSegment.h"
+#include "nsContentUtils.h"
+#include "nsGlobalWindowInner.h"
+#include "nsPrintfCString.h"
+#include "nsServiceManagerUtils.h"
+#include "prerror.h"
+#include "mozilla/Logging.h"
+#include "mozilla/Attributes.h"
+#include "ForwardedInputTrack.h"
+#include "ImageContainer.h"
+#include "AudioCaptureTrack.h"
+#include "AudioDeviceInfo.h"
+#include "AudioNodeTrack.h"
+#include "AudioNodeExternalInputTrack.h"
+#if defined(MOZ_WEBRTC)
+# include "MediaEngineWebRTCAudio.h"
+#endif // MOZ_WEBRTC
+#include "MediaTrackListener.h"
+#include "mozilla/dom/BaseAudioContextBinding.h"
+#include "mozilla/dom/Document.h"
+#include "mozilla/dom/WorkletThread.h"
+#include "mozilla/media/MediaUtils.h"
+#include <algorithm>
+#include "GeckoProfiler.h"
+#include "VideoFrameContainer.h"
+#include "mozilla/AbstractThread.h"
+#include "mozilla/StaticPrefs_dom.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "transport/runnable_utils.h"
+#include "VideoUtils.h"
+#include "GraphRunner.h"
+#include "Tracing.h"
+#include "UnderrunHandler.h"
+#include "mozilla/CycleCollectedJSRuntime.h"
+#include "mozilla/Preferences.h"
+
+#include "webaudio/blink/DenormalDisabler.h"
+#include "webaudio/blink/HRTFDatabaseLoader.h"
+
+using namespace mozilla::layers;
+using namespace mozilla::dom;
+using namespace mozilla::gfx;
+using namespace mozilla::media;
+
+namespace mozilla {
+
+using AudioDeviceID = CubebUtils::AudioDeviceID;
+using IsInShutdown = MediaTrack::IsInShutdown;
+
+LazyLogModule gMediaTrackGraphLog("MediaTrackGraph");
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
+
+NativeInputTrack* DeviceInputTrackManager::GetNativeInputTrack() {
+ return mNativeInputTrack.get();
+}
+
+DeviceInputTrack* DeviceInputTrackManager::GetDeviceInputTrack(
+ CubebUtils::AudioDeviceID aID) {
+ if (mNativeInputTrack && mNativeInputTrack->mDeviceId == aID) {
+ return mNativeInputTrack.get();
+ }
+ for (const RefPtr<NonNativeInputTrack>& t : mNonNativeInputTracks) {
+ if (t->mDeviceId == aID) {
+ return t.get();
+ }
+ }
+ return nullptr;
+}
+
+NonNativeInputTrack* DeviceInputTrackManager::GetFirstNonNativeInputTrack() {
+ if (mNonNativeInputTracks.IsEmpty()) {
+ return nullptr;
+ }
+ return mNonNativeInputTracks[0].get();
+}
+
+void DeviceInputTrackManager::Add(DeviceInputTrack* aTrack) {
+ if (NativeInputTrack* native = aTrack->AsNativeInputTrack()) {
+ MOZ_ASSERT(!mNativeInputTrack);
+ mNativeInputTrack = native;
+ } else {
+ NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
+ MOZ_ASSERT(nonNative);
+ struct DeviceTrackComparator {
+ public:
+ bool Equals(const RefPtr<NonNativeInputTrack>& aTrack,
+ CubebUtils::AudioDeviceID aDeviceId) const {
+ return aTrack->mDeviceId == aDeviceId;
+ }
+ };
+ MOZ_ASSERT(!mNonNativeInputTracks.Contains(aTrack->mDeviceId,
+ DeviceTrackComparator()));
+ mNonNativeInputTracks.AppendElement(nonNative);
+ }
+}
+
+void DeviceInputTrackManager::Remove(DeviceInputTrack* aTrack) {
+ if (aTrack->AsNativeInputTrack()) {
+ MOZ_ASSERT(mNativeInputTrack);
+ MOZ_ASSERT(mNativeInputTrack.get() == aTrack->AsNativeInputTrack());
+ mNativeInputTrack = nullptr;
+ } else {
+ NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
+ MOZ_ASSERT(nonNative);
+ DebugOnly<bool> removed = mNonNativeInputTracks.RemoveElement(nonNative);
+ MOZ_ASSERT(removed);
+ }
+}
+
+/**
+ * A hash table containing the graph instances, one per Window ID,
+ * sample rate, and device ID combination.
+ */
+
+struct MediaTrackGraphImpl::Lookup final {
+ HashNumber Hash() const {
+ return HashGeneric(mWindowID, mSampleRate, mOutputDeviceID);
+ }
+ const uint64_t mWindowID;
+ const TrackRate mSampleRate;
+ const CubebUtils::AudioDeviceID mOutputDeviceID;
+};
+
+// Implicit to support GraphHashSet.lookup(*graph).
+MOZ_IMPLICIT MediaTrackGraphImpl::operator MediaTrackGraphImpl::Lookup() const {
+ return {mWindowID, mSampleRate, PrimaryOutputDeviceID()};
+}
+
+namespace {
+struct GraphHasher { // for HashSet
+ using Lookup = const MediaTrackGraphImpl::Lookup;
+
+ static HashNumber hash(const Lookup& aLookup) { return aLookup.Hash(); }
+
+ static bool match(const MediaTrackGraphImpl* aGraph, const Lookup& aLookup) {
+ return aGraph->mWindowID == aLookup.mWindowID &&
+ aGraph->GraphRate() == aLookup.mSampleRate &&
+ aGraph->PrimaryOutputDeviceID() == aLookup.mOutputDeviceID;
+ }
+};
+
+// The weak reference to the graph is removed when its last track is removed.
+using GraphHashSet =
+ HashSet<MediaTrackGraphImpl*, GraphHasher, InfallibleAllocPolicy>;
+GraphHashSet* Graphs() {
+ MOZ_ASSERT(NS_IsMainThread());
+ static GraphHashSet sGraphs(4); // 4 is minimum HashSet capacity
+ return &sGraphs;
+}
+} // anonymous namespace
+
+static void ApplyTrackDisabling(DisabledTrackMode aDisabledMode,
+ MediaSegment* aSegment,
+ MediaSegment* aRawSegment) {
+ if (aDisabledMode == DisabledTrackMode::ENABLED) {
+ return;
+ }
+ if (aDisabledMode == DisabledTrackMode::SILENCE_BLACK) {
+ aSegment->ReplaceWithDisabled();
+ if (aRawSegment) {
+ aRawSegment->ReplaceWithDisabled();
+ }
+ } else if (aDisabledMode == DisabledTrackMode::SILENCE_FREEZE) {
+ aSegment->ReplaceWithNull();
+ if (aRawSegment) {
+ aRawSegment->ReplaceWithNull();
+ }
+ } else {
+ MOZ_CRASH("Unsupported mode");
+ }
+}
+
+MediaTrackGraphImpl::~MediaTrackGraphImpl() {
+ MOZ_ASSERT(mTracks.IsEmpty() && mSuspendedTracks.IsEmpty(),
+ "All tracks should have been destroyed by messages from the main "
+ "thread");
+ LOG(LogLevel::Debug, ("MediaTrackGraph %p destroyed", this));
+ LOG(LogLevel::Debug, ("MediaTrackGraphImpl::~MediaTrackGraphImpl"));
+}
+
+void MediaTrackGraphImpl::AddTrackGraphThread(MediaTrack* aTrack) {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+ aTrack->mStartTime = mProcessedTime;
+
+ if (aTrack->IsSuspended()) {
+ mSuspendedTracks.AppendElement(aTrack);
+ LOG(LogLevel::Debug,
+ ("%p: Adding media track %p, in the suspended track array", this,
+ aTrack));
+ } else {
+ mTracks.AppendElement(aTrack);
+ LOG(LogLevel::Debug, ("%p: Adding media track %p, count %zu", this, aTrack,
+ mTracks.Length()));
+ }
+
+ SetTrackOrderDirty();
+}
+
+void MediaTrackGraphImpl::RemoveTrackGraphThread(MediaTrack* aTrack) {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+ // Remove references in mTrackUpdates before we allow aTrack to die.
+ // Pending updates are not needed (since the main thread has already given
+ // up the track) so we will just drop them.
+ {
+ MonitorAutoLock lock(mMonitor);
+ for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
+ if (mTrackUpdates[i].mTrack == aTrack) {
+ mTrackUpdates[i].mTrack = nullptr;
+ }
+ }
+ }
+
+ // Ensure that mFirstCycleBreaker is updated when necessary.
+ SetTrackOrderDirty();
+
+ UnregisterAllAudioOutputs(aTrack);
+
+ if (aTrack->IsSuspended()) {
+ mSuspendedTracks.RemoveElement(aTrack);
+ } else {
+ mTracks.RemoveElement(aTrack);
+ }
+
+ LOG(LogLevel::Debug, ("%p: Removed media track %p, count %zu", this, aTrack,
+ mTracks.Length()));
+
+ NS_RELEASE(aTrack); // probably destroying it
+}
+
+TrackTime MediaTrackGraphImpl::GraphTimeToTrackTimeWithBlocking(
+ const MediaTrack* aTrack, GraphTime aTime) const {
+ MOZ_ASSERT(
+ aTime <= mStateComputedTime,
+ "Don't ask about times where we haven't made blocking decisions yet");
+ return std::max<TrackTime>(
+ 0, std::min(aTime, aTrack->mStartBlocking) - aTrack->mStartTime);
+}
+
+GraphTime MediaTrackGraphImpl::IterationEnd() const {
+ MOZ_ASSERT(OnGraphThread());
+ return mIterationEndTime;
+}
+
+void MediaTrackGraphImpl::UpdateCurrentTimeForTracks(
+ GraphTime aPrevCurrentTime) {
+ MOZ_ASSERT(OnGraphThread());
+ for (MediaTrack* track : AllTracks()) {
+ // Shouldn't have already notified of ended *and* have output!
+ MOZ_ASSERT_IF(track->mStartBlocking > aPrevCurrentTime,
+ !track->mNotifiedEnded);
+
+ // Calculate blocked time and fire Blocked/Unblocked events
+ GraphTime blockedTime = mStateComputedTime - track->mStartBlocking;
+ NS_ASSERTION(blockedTime >= 0, "Error in blocking time");
+ track->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime,
+ blockedTime);
+ LOG(LogLevel::Verbose,
+ ("%p: MediaTrack %p bufferStartTime=%f blockedTime=%f", this, track,
+ MediaTimeToSeconds(track->mStartTime),
+ MediaTimeToSeconds(blockedTime)));
+ track->mStartBlocking = mStateComputedTime;
+
+ TrackTime trackCurrentTime =
+ track->GraphTimeToTrackTime(mStateComputedTime);
+ if (track->mEnded) {
+ MOZ_ASSERT(track->GetEnd() <= trackCurrentTime);
+ if (!track->mNotifiedEnded) {
+ // Playout of this track ended and listeners have not been notified.
+ track->mNotifiedEnded = true;
+ SetTrackOrderDirty();
+ for (const auto& listener : track->mTrackListeners) {
+ listener->NotifyOutput(this, track->GetEnd());
+ listener->NotifyEnded(this);
+ }
+ }
+ } else {
+ for (const auto& listener : track->mTrackListeners) {
+ listener->NotifyOutput(this, trackCurrentTime);
+ }
+ }
+ }
+}
+
+template <typename C, typename Chunk>
+void MediaTrackGraphImpl::ProcessChunkMetadataForInterval(MediaTrack* aTrack,
+ C& aSegment,
+ TrackTime aStart,
+ TrackTime aEnd) {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+ MOZ_ASSERT(aTrack);
+
+ TrackTime offset = 0;
+ for (typename C::ConstChunkIterator chunk(aSegment); !chunk.IsEnded();
+ chunk.Next()) {
+ if (offset >= aEnd) {
+ break;
+ }
+ offset += chunk->GetDuration();
+ if (chunk->IsNull() || offset < aStart) {
+ continue;
+ }
+ const PrincipalHandle& principalHandle = chunk->GetPrincipalHandle();
+ if (principalHandle != aSegment.GetLastPrincipalHandle()) {
+ aSegment.SetLastPrincipalHandle(principalHandle);
+ LOG(LogLevel::Debug,
+ ("%p: MediaTrack %p, principalHandle "
+ "changed in %sChunk with duration %lld",
+ this, aTrack,
+ aSegment.GetType() == MediaSegment::AUDIO ? "Audio" : "Video",
+ (long long)chunk->GetDuration()));
+ for (const auto& listener : aTrack->mTrackListeners) {
+ listener->NotifyPrincipalHandleChanged(this, principalHandle);
+ }
+ }
+ }
+}
+
+void MediaTrackGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime) {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+ for (MediaTrack* track : AllTracks()) {
+ TrackTime iterationStart = track->GraphTimeToTrackTime(aPrevCurrentTime);
+ TrackTime iterationEnd = track->GraphTimeToTrackTime(mProcessedTime);
+ if (!track->mSegment) {
+ continue;
+ }
+ if (track->mType == MediaSegment::AUDIO) {
+ ProcessChunkMetadataForInterval<AudioSegment, AudioChunk>(
+ track, *track->GetData<AudioSegment>(), iterationStart, iterationEnd);
+ } else if (track->mType == MediaSegment::VIDEO) {
+ ProcessChunkMetadataForInterval<VideoSegment, VideoChunk>(
+ track, *track->GetData<VideoSegment>(), iterationStart, iterationEnd);
+ } else {
+ MOZ_CRASH("Unknown track type");
+ }
+ }
+}
+
+GraphTime MediaTrackGraphImpl::WillUnderrun(MediaTrack* aTrack,
+ GraphTime aEndBlockingDecisions) {
+ // Ended tracks can't underrun. ProcessedMediaTracks also can't cause
+ // underrun currently, since we'll always be able to produce data for them
+ // unless they block on some other track.
+ if (aTrack->mEnded || aTrack->AsProcessedTrack()) {
+ return aEndBlockingDecisions;
+ }
+ // This track isn't ended or suspended. We don't need to call
+ // TrackTimeToGraphTime since an underrun is the only thing that can block
+ // it.
+ GraphTime bufferEnd = aTrack->GetEnd() + aTrack->mStartTime;
+#ifdef DEBUG
+ if (bufferEnd < mProcessedTime) {
+ LOG(LogLevel::Error, ("%p: MediaTrack %p underrun, "
+ "bufferEnd %f < mProcessedTime %f (%" PRId64
+ " < %" PRId64 "), TrackTime %" PRId64,
+ this, aTrack, MediaTimeToSeconds(bufferEnd),
+ MediaTimeToSeconds(mProcessedTime), bufferEnd,
+ mProcessedTime, aTrack->GetEnd()));
+ NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran");
+ }
+#endif
+ return std::min(bufferEnd, aEndBlockingDecisions);
+}
+
+namespace {
+// Value of mCycleMarker for unvisited tracks in cycle detection.
+const uint32_t NOT_VISITED = UINT32_MAX;
+// Value of mCycleMarker for ordered tracks in muted cycles.
+const uint32_t IN_MUTED_CYCLE = 1;
+} // namespace
+
+bool MediaTrackGraphImpl::AudioTrackPresent() {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+
+ bool audioTrackPresent = false;
+ for (MediaTrack* track : mTracks) {
+ if (track->AsAudioNodeTrack()) {
+ audioTrackPresent = true;
+ break;
+ }
+
+ if (track->mType == MediaSegment::AUDIO && !track->mNotifiedEnded) {
+ audioTrackPresent = true;
+ break;
+ }
+ }
+
+ // We may not have audio input device when we only have AudioNodeTracks. But
+ // if audioTrackPresent is false, we must have no input device.
+ MOZ_DIAGNOSTIC_ASSERT_IF(
+ !audioTrackPresent,
+ !mDeviceInputTrackManagerGraphThread.GetNativeInputTrack());
+
+ return audioTrackPresent;
+}
+
+void MediaTrackGraphImpl::CheckDriver() {
+ MOZ_ASSERT(OnGraphThread());
+ // An offline graph has only one driver.
+ // Otherwise, if a switch is already pending, let that happen.
+ if (!mRealtime || Switching()) {
+ return;
+ }
+
+ AudioCallbackDriver* audioCallbackDriver =
+ CurrentDriver()->AsAudioCallbackDriver();
+ if (audioCallbackDriver && !audioCallbackDriver->OnFallback()) {
+ for (PendingResumeOperation& op : mPendingResumeOperations) {
+ op.Apply(this);
+ }
+ mPendingResumeOperations.Clear();
+ }
+
+ // Note that this looks for any audio tracks, input or output, and switches
+ // to a SystemClockDriver if there are none active or no resume operations
+ // to make any active.
+ bool needAudioCallbackDriver =
+ !mPendingResumeOperations.IsEmpty() || AudioTrackPresent();
+ if (!needAudioCallbackDriver) {
+ if (audioCallbackDriver && audioCallbackDriver->IsStarted()) {
+ SwitchAtNextIteration(
+ new SystemClockDriver(this, CurrentDriver(), mSampleRate));
+ }
+ return;
+ }
+
+ NativeInputTrack* native =
+ mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
+ CubebUtils::AudioDeviceID inputDevice = native ? native->mDeviceId : nullptr;
+ uint32_t inputChannelCount =
+ native ? AudioInputChannelCount(native->mDeviceId) : 0;
+ AudioInputType inputPreference =
+ native ? AudioInputDevicePreference(native->mDeviceId)
+ : AudioInputType::Unknown;
+
+ uint32_t primaryOutputChannelCount = PrimaryOutputChannelCount();
+ if (!audioCallbackDriver) {
+ if (primaryOutputChannelCount > 0) {
+ AudioCallbackDriver* driver = new AudioCallbackDriver(
+ this, CurrentDriver(), mSampleRate, primaryOutputChannelCount,
+ inputChannelCount, PrimaryOutputDeviceID(), inputDevice,
+ inputPreference);
+ SwitchAtNextIteration(driver);
+ }
+ return;
+ }
+
+ // Check if this graph should switch to a different number of output channels.
+ // Generally, a driver switch is explicitly made by an event (e.g., setting
+ // the AudioDestinationNode channelCount), but if an HTMLMediaElement is
+ // directly playing back via another HTMLMediaElement, the number of channels
+ // of the media determines how many channels to output, and it can change
+ // dynamically.
+ if (primaryOutputChannelCount != audioCallbackDriver->OutputChannelCount()) {
+ AudioCallbackDriver* driver = new AudioCallbackDriver(
+ this, CurrentDriver(), mSampleRate, primaryOutputChannelCount,
+ inputChannelCount, PrimaryOutputDeviceID(), inputDevice,
+ inputPreference);
+ SwitchAtNextIteration(driver);
+ }
+}
+
+void MediaTrackGraphImpl::UpdateTrackOrder() {
+ if (!mTrackOrderDirty) {
+ return;
+ }
+
+ mTrackOrderDirty = false;
+
+ // The algorithm for finding cycles is based on Tim Leslie's iterative
+ // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly
+ // connected components (SCC) algorithm. There are variations (a) to
+ // distinguish whether tracks in SCCs of size 1 are in a cycle and (b) to
+ // re-run the algorithm over SCCs with breaks at DelayNodes.
+ //
+ // [1] http://www.timl.id.au/?p=327
+ // [2]
+ // https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582
+ // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707
+ //
+ // There are two stacks. One for the depth-first search (DFS),
+ mozilla::LinkedList<MediaTrack> dfsStack;
+ // and another for tracks popped from the DFS stack, but still being
+ // considered as part of SCCs involving tracks on the stack.
+ mozilla::LinkedList<MediaTrack> sccStack;
+
+ // An index into mTracks for the next track found with no unsatisfied
+ // upstream dependencies.
+ uint32_t orderedTrackCount = 0;
+
+ for (uint32_t i = 0; i < mTracks.Length(); ++i) {
+ MediaTrack* t = mTracks[i];
+ ProcessedMediaTrack* pt = t->AsProcessedTrack();
+ if (pt) {
+ // The dfsStack initially contains a list of all processed tracks in
+ // unchanged order.
+ dfsStack.insertBack(t);
+ pt->mCycleMarker = NOT_VISITED;
+ } else {
+ // SourceMediaTracks have no inputs and so can be ordered now.
+ mTracks[orderedTrackCount] = t;
+ ++orderedTrackCount;
+ }
+ }
+
+ // mNextStackMarker corresponds to "index" in Tarjan's algorithm. It is a
+ // counter to label mCycleMarker on the next visited track in the DFS
+ // uniquely in the set of visited tracks that are still being considered.
+ //
+ // In this implementation, the counter descends so that the values are
+ // strictly greater than the values that mCycleMarker takes when the track
+ // has been ordered (0 or IN_MUTED_CYCLE).
+ //
+ // Each new track labelled, as the DFS searches upstream, receives a value
+ // less than those used for all other tracks being considered.
+ uint32_t nextStackMarker = NOT_VISITED - 1;
+ // Reset list of DelayNodes in cycles stored at the tail of mTracks.
+ mFirstCycleBreaker = mTracks.Length();
+
+ // Rearrange dfsStack order as required to DFS upstream and pop tracks
+ // in processing order to place in mTracks.
+ while (auto pt = static_cast<ProcessedMediaTrack*>(dfsStack.getFirst())) {
+ const auto& inputs = pt->mInputs;
+ MOZ_ASSERT(pt->AsProcessedTrack());
+ if (pt->mCycleMarker == NOT_VISITED) {
+ // Record the position on the visited stack, so that any searches
+ // finding this track again know how much of the stack is in the cycle.
+ pt->mCycleMarker = nextStackMarker;
+ --nextStackMarker;
+ // Not-visited input tracks should be processed first.
+ // SourceMediaTracks have already been ordered.
+ for (uint32_t i = inputs.Length(); i--;) {
+ if (inputs[i]->GetSource()->IsSuspended()) {
+ continue;
+ }
+ auto input = inputs[i]->GetSource()->AsProcessedTrack();
+ if (input && input->mCycleMarker == NOT_VISITED) {
+ // It can be that this track has an input which is from a suspended
+ // AudioContext.
+ if (input->isInList()) {
+ input->remove();
+ dfsStack.insertFront(input);
+ }
+ }
+ }
+ continue;
+ }
+
+ // Returning from DFS. Pop from dfsStack.
+ pt->remove();
+
+ // cycleStackMarker keeps track of the highest marker value on any
+ // upstream track, if any, found receiving input, directly or indirectly,
+ // from the visited stack (and so from |ps|, making a cycle). In a
+ // variation from Tarjan's SCC algorithm, this does not include |ps|
+ // unless it is part of the cycle.
+ uint32_t cycleStackMarker = 0;
+ for (uint32_t i = inputs.Length(); i--;) {
+ if (inputs[i]->GetSource()->IsSuspended()) {
+ continue;
+ }
+ auto input = inputs[i]->GetSource()->AsProcessedTrack();
+ if (input) {
+ cycleStackMarker = std::max(cycleStackMarker, input->mCycleMarker);
+ }
+ }
+
+ if (cycleStackMarker <= IN_MUTED_CYCLE) {
+ // All inputs have been ordered and their stack markers have been removed.
+ // This track is not part of a cycle. It can be processed next.
+ pt->mCycleMarker = 0;
+ mTracks[orderedTrackCount] = pt;
+ ++orderedTrackCount;
+ continue;
+ }
+
+ // A cycle has been found. Record this track for ordering when all
+ // tracks in this SCC have been popped from the DFS stack.
+ sccStack.insertFront(pt);
+
+ if (cycleStackMarker > pt->mCycleMarker) {
+ // Cycles have been found that involve tracks that remain on the stack.
+ // Leave mCycleMarker indicating the most downstream (last) track on
+ // the stack known to be part of this SCC. In this way, any searches on
+ // other paths that find |ps| will know (without having to traverse from
+ // this track again) that they are part of this SCC (i.e. part of an
+ // intersecting cycle).
+ pt->mCycleMarker = cycleStackMarker;
+ continue;
+ }
+
+ // |pit| is the root of an SCC involving no other tracks on dfsStack, the
+ // complete SCC has been recorded, and tracks in this SCC are part of at
+ // least one cycle.
+ MOZ_ASSERT(cycleStackMarker == pt->mCycleMarker);
+ // If there are DelayNodes in this SCC, then they may break the cycles.
+ bool haveDelayNode = false;
+ auto next = sccStack.getFirst();
+ // Tracks in this SCC are identified by mCycleMarker <= cycleStackMarker.
+ // (There may be other tracks later in sccStack from other incompletely
+ // searched SCCs, involving tracks still on dfsStack.)
+ //
+ // DelayNodes in cycles must behave differently from those not in cycles,
+ // so all DelayNodes in the SCC must be identified.
+ while (next && static_cast<ProcessedMediaTrack*>(next)->mCycleMarker <=
+ cycleStackMarker) {
+ auto nt = next->AsAudioNodeTrack();
+ // Get next before perhaps removing from list below.
+ next = next->getNext();
+ if (nt && nt->Engine()->AsDelayNodeEngine()) {
+ haveDelayNode = true;
+ // DelayNodes break cycles by producing their output in a
+ // preprocessing phase; they do not need to be ordered before their
+ // consumers. Order them at the tail of mTracks so that they can be
+ // handled specially. Do so now, so that DFS ignores them.
+ nt->remove();
+ nt->mCycleMarker = 0;
+ --mFirstCycleBreaker;
+ mTracks[mFirstCycleBreaker] = nt;
+ }
+ }
+ auto after_scc = next;
+ while ((next = sccStack.getFirst()) != after_scc) {
+ next->remove();
+ auto removed = static_cast<ProcessedMediaTrack*>(next);
+ if (haveDelayNode) {
+ // Return tracks to the DFS stack again (to order and detect cycles
+ // without delayNodes). Any of these tracks that are still inputs
+ // for tracks on the visited stack must be returned to the front of
+ // the stack to be ordered before their dependents. We know that none
+ // of these tracks need input from tracks on the visited stack, so
+ // they can all be searched and ordered before the current stack head
+ // is popped.
+ removed->mCycleMarker = NOT_VISITED;
+ dfsStack.insertFront(removed);
+ } else {
+ // Tracks in cycles without any DelayNodes must be muted, and so do
+ // not need input and can be ordered now. They must be ordered before
+ // their consumers so that their muted output is available.
+ removed->mCycleMarker = IN_MUTED_CYCLE;
+ mTracks[orderedTrackCount] = removed;
+ ++orderedTrackCount;
+ }
+ }
+ }
+
+ MOZ_ASSERT(orderedTrackCount == mFirstCycleBreaker);
+}
+
+TrackTime MediaTrackGraphImpl::PlayAudio(const TrackAndVolume& aOutput,
+ GraphTime aPlayedTime,
+ uint32_t aOutputChannelCount) {
+ MOZ_ASSERT(OnGraphThread());
+ MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode");
+
+ TrackTime ticksWritten = 0;
+
+ ticksWritten = 0;
+ MediaTrack* track = aOutput.mTrack;
+ AudioSegment* audio = track->GetData<AudioSegment>();
+ AudioSegment output;
+
+ TrackTime offset = track->GraphTimeToTrackTime(aPlayedTime);
+
+ // We don't update Track->mTracksStartTime here to account for time spent
+ // blocked. Instead, we'll update it in UpdateCurrentTimeForTracks after
+ // the blocked period has completed. But we do need to make sure we play
+ // from the right offsets in the track buffer, even if we've already
+ // written silence for some amount of blocked time after the current time.
+ GraphTime t = aPlayedTime;
+ while (t < mStateComputedTime) {
+ bool blocked = t >= track->mStartBlocking;
+ GraphTime end = blocked ? mStateComputedTime : track->mStartBlocking;
+ NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!");
+
+ // Check how many ticks of sound we can provide if we are blocked some
+ // time in the middle of this cycle.
+ TrackTime toWrite = end - t;
+
+ if (blocked) {
+ output.InsertNullDataAtStart(toWrite);
+ ticksWritten += toWrite;
+ LOG(LogLevel::Verbose,
+ ("%p: MediaTrack %p writing %" PRId64 " blocking-silence samples for "
+ "%f to %f (%" PRId64 " to %" PRId64 ")",
+ this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end),
+ offset, offset + toWrite));
+ } else {
+ TrackTime endTicksNeeded = offset + toWrite;
+ TrackTime endTicksAvailable = audio->GetDuration();
+
+ if (endTicksNeeded <= endTicksAvailable) {
+ LOG(LogLevel::Verbose,
+ ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
+ "(samples %" PRId64 " to %" PRId64 ")",
+ this, track, toWrite, MediaTimeToSeconds(t),
+ MediaTimeToSeconds(end), offset, endTicksNeeded));
+ output.AppendSlice(*audio, offset, endTicksNeeded);
+ ticksWritten += toWrite;
+ offset = endTicksNeeded;
+ } else {
+ // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not
+ // ended."); If we are at the end of the track, maybe write the
+ // remaining samples, and pad with/output silence.
+ if (endTicksNeeded > endTicksAvailable && offset < endTicksAvailable) {
+ output.AppendSlice(*audio, offset, endTicksAvailable);
+
+ LOG(LogLevel::Verbose,
+ ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f "
+ "(samples %" PRId64 " to %" PRId64 ")",
+ this, track, toWrite, MediaTimeToSeconds(t),
+ MediaTimeToSeconds(end), offset, endTicksNeeded));
+ uint32_t available = endTicksAvailable - offset;
+ ticksWritten += available;
+ toWrite -= available;
+ offset = endTicksAvailable;
+ }
+ output.AppendNullData(toWrite);
+ LOG(LogLevel::Verbose,
+ ("%p MediaTrack %p writing %" PRId64 " padding slsamples for %f to "
+ "%f (samples %" PRId64 " to %" PRId64 ")",
+ this, track, toWrite, MediaTimeToSeconds(t),
+ MediaTimeToSeconds(end), offset, endTicksNeeded));
+ ticksWritten += toWrite;
+ }
+ output.ApplyVolume(mGlobalVolume * aOutput.mVolume);
+ }
+ t = end;
+
+ output.Mix(mMixer, aOutputChannelCount, mSampleRate);
+ }
+ return ticksWritten;
+}
+
+DeviceInputTrack* MediaTrackGraph::GetDeviceInputTrackMainThread(
+ CubebUtils::AudioDeviceID aID) {
+ MOZ_ASSERT(NS_IsMainThread());
+ auto* impl = static_cast<MediaTrackGraphImpl*>(this);
+ return impl->mDeviceInputTrackManagerMainThread.GetDeviceInputTrack(aID);
+}
+
+NativeInputTrack* MediaTrackGraph::GetNativeInputTrackMainThread() {
+ MOZ_ASSERT(NS_IsMainThread());
+ auto* impl = static_cast<MediaTrackGraphImpl*>(this);
+ return impl->mDeviceInputTrackManagerMainThread.GetNativeInputTrack();
+}
+
+void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack* aTrack) {
+ MOZ_ASSERT(OnGraphThread());
+ LOG(LogLevel::Debug,
+ ("%p OpenAudioInputImpl: device %p", this, aTrack->mDeviceId));
+
+ mDeviceInputTrackManagerGraphThread.Add(aTrack);
+
+ if (aTrack->AsNativeInputTrack()) {
+ // Switch Drivers since we're adding input (to input-only or full-duplex)
+ AudioCallbackDriver* driver = new AudioCallbackDriver(
+ this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
+ AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(),
+ aTrack->mDeviceId, AudioInputDevicePreference(aTrack->mDeviceId));
+ LOG(LogLevel::Debug,
+ ("%p OpenAudioInputImpl: starting new AudioCallbackDriver(input) %p",
+ this, driver));
+ SwitchAtNextIteration(driver);
+ } else {
+ NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack();
+ MOZ_ASSERT(nonNative);
+ // Start non-native input right away.
+ nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
+ MakeRefPtr<AudioInputSourceListener>(nonNative),
+ nonNative->GenerateSourceId(), nonNative->mDeviceId,
+ AudioInputChannelCount(nonNative->mDeviceId),
+ AudioInputDevicePreference(nonNative->mDeviceId) ==
+ AudioInputType::Voice,
+ nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate()));
+ }
+}
+
+void MediaTrackGraphImpl::OpenAudioInput(DeviceInputTrack* aTrack) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(aTrack);
+
+ LOG(LogLevel::Debug, ("%p OpenInput: DeviceInputTrack %p for device %p", this,
+ aTrack, aTrack->mDeviceId));
+
+ class Message : public ControlMessage {
+ public:
+ Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
+ : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
+ void Run() override {
+ TRACE("MTG::OpenAudioInputImpl ControlMessage");
+ mGraph->OpenAudioInputImpl(mInputTrack);
+ }
+ MediaTrackGraphImpl* mGraph;
+ DeviceInputTrack* mInputTrack;
+ };
+
+ mDeviceInputTrackManagerMainThread.Add(aTrack);
+
+ this->AppendMessage(MakeUnique<Message>(this, aTrack));
+}
+
+void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack* aTrack) {
+ MOZ_ASSERT(OnGraphThread());
+
+ LOG(LogLevel::Debug,
+ ("%p CloseAudioInputImpl: device %p", this, aTrack->mDeviceId));
+
+ if (NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack()) {
+ nonNative->StopAudio();
+ mDeviceInputTrackManagerGraphThread.Remove(aTrack);
+ return;
+ }
+
+ MOZ_ASSERT(aTrack->AsNativeInputTrack());
+
+ mDeviceInputTrackManagerGraphThread.Remove(aTrack);
+
+ // Switch Drivers since we're adding or removing an input (to nothing/system
+ // or output only)
+ bool audioTrackPresent = AudioTrackPresent();
+
+ GraphDriver* driver;
+ if (audioTrackPresent) {
+ // We still have audio output
+ LOG(LogLevel::Debug,
+ ("%p: CloseInput: output present (AudioCallback)", this));
+
+ driver = new AudioCallbackDriver(
+ this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
+ AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(),
+ nullptr, AudioInputDevicePreference(aTrack->mDeviceId));
+ SwitchAtNextIteration(driver);
+ } else if (CurrentDriver()->AsAudioCallbackDriver()) {
+ LOG(LogLevel::Debug,
+ ("%p: CloseInput: no output present (SystemClockCallback)", this));
+
+ driver = new SystemClockDriver(this, CurrentDriver(), mSampleRate);
+ SwitchAtNextIteration(driver);
+ } // else SystemClockDriver->SystemClockDriver, no switch
+}
+
+void MediaTrackGraphImpl::UnregisterAllAudioOutputs(MediaTrack* aTrack) {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+ mOutputDevices.RemoveElementsBy([&](OutputDeviceEntry& aDeviceRef) {
+ aDeviceRef.mTrackOutputs.RemoveElement(aTrack);
+ // mReceiver is null for the primary output device, which is retained for
+ // AudioCallbackDriver output even when no tracks have audio outputs.
+ return aDeviceRef.mTrackOutputs.IsEmpty() && aDeviceRef.mReceiver;
+ });
+}
+
+void MediaTrackGraphImpl::CloseAudioInput(DeviceInputTrack* aTrack) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(aTrack);
+
+ LOG(LogLevel::Debug, ("%p CloseInput: DeviceInputTrack %p for device %p",
+ this, aTrack, aTrack->mDeviceId));
+
+ class Message : public ControlMessage {
+ public:
+ Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack)
+ : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {}
+ void Run() override {
+ TRACE("MTG::CloseAudioInputImpl ControlMessage");
+ mGraph->CloseAudioInputImpl(mInputTrack);
+ }
+ MediaTrackGraphImpl* mGraph;
+ DeviceInputTrack* mInputTrack;
+ };
+
+ // DeviceInputTrack is still alive (in mTracks) even we remove it here, since
+ // aTrack->Destroy() is called after this. See DeviceInputTrack::CloseAudio
+ // for more details.
+ mDeviceInputTrackManagerMainThread.Remove(aTrack);
+
+ this->AppendMessage(MakeUnique<Message>(this, aTrack));
+
+ if (aTrack->AsNativeInputTrack()) {
+ LOG(LogLevel::Debug,
+ ("%p Native input device %p is closed!", this, aTrack->mDeviceId));
+ SetNewNativeInput();
+ }
+}
+
+// All AudioInput listeners get the same speaker data (at least for now).
+void MediaTrackGraphImpl::NotifyOutputData(const AudioChunk& aChunk) {
+ if (!mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()) {
+ return;
+ }
+
+#if defined(MOZ_WEBRTC)
+ for (const auto& track : mTracks) {
+ if (const auto& t = track->AsAudioProcessingTrack()) {
+ t->NotifyOutputData(this, aChunk);
+ }
+ }
+#endif
+}
+
+void MediaTrackGraphImpl::NotifyInputStopped() {
+ NativeInputTrack* native =
+ mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
+ if (!native) {
+ return;
+ }
+ native->NotifyInputStopped(this);
+}
+
+void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer,
+ size_t aFrames, TrackRate aRate,
+ uint32_t aChannels,
+ uint32_t aAlreadyBuffered) {
+ // Either we have an audio input device, or we just removed the audio input
+ // this iteration, and we're switching back to an output-only driver next
+ // iteration.
+ NativeInputTrack* native =
+ mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
+ MOZ_ASSERT(native || Switching());
+ if (!native) {
+ return;
+ }
+ native->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels,
+ aAlreadyBuffered);
+}
+
+void MediaTrackGraphImpl::DeviceChangedImpl() {
+ MOZ_ASSERT(OnGraphThread());
+ NativeInputTrack* native =
+ mDeviceInputTrackManagerGraphThread.GetNativeInputTrack();
+ if (!native) {
+ return;
+ }
+ native->DeviceChanged(this);
+}
+
+void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount) {
+ MOZ_ASSERT(OnGraphThread());
+ mMaxOutputChannelCount = aMaxChannelCount;
+}
+
+void MediaTrackGraphImpl::DeviceChanged() {
+ // This is safe to be called from any thread: this message comes from an
+ // underlying platform API, and we don't have much guarantees. If it is not
+ // called from the main thread (and it probably will rarely be), it will post
+ // itself to the main thread, and the actual device change message will be ran
+ // and acted upon on the graph thread.
+ if (!NS_IsMainThread()) {
+ RefPtr<nsIRunnable> runnable = WrapRunnable(
+ RefPtr<MediaTrackGraphImpl>(this), &MediaTrackGraphImpl::DeviceChanged);
+ mMainThread->Dispatch(runnable.forget());
+ return;
+ }
+
+ class Message : public ControlMessage {
+ public:
+ explicit Message(MediaTrackGraph* aGraph)
+ : ControlMessage(nullptr),
+ mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)) {}
+ void Run() override {
+ TRACE("MTG::DeviceChangeImpl ControlMessage");
+ mGraphImpl->DeviceChangedImpl();
+ }
+ // We know that this is valid, because the graph can't shutdown if it has
+ // messages.
+ MediaTrackGraphImpl* mGraphImpl;
+ };
+
+ if (mMainThreadTrackCount == 0 && mMainThreadPortCount == 0) {
+ // This is a special case where the origin of this event cannot control the
+ // lifetime of the graph, because the graph is controling the lifetime of
+ // the AudioCallbackDriver where the event originated.
+ // We know the graph is soon going away, so there's no need to notify about
+ // this device change.
+ return;
+ }
+
+ // Reset the latency, it will get fetched again next time it's queried.
+ MOZ_ASSERT(NS_IsMainThread());
+ mAudioOutputLatency = 0.0;
+
+ // Dispatch to the bg thread to do the (potentially expensive) query of the
+ // maximum channel count, and then dispatch back to the main thread, then to
+ // the graph, with the new info.
+ RefPtr<MediaTrackGraphImpl> self = this;
+ NS_DispatchBackgroundTask(NS_NewRunnableFunction(
+ "MaxChannelCountUpdateOnBgThread", [self{std::move(self)}]() {
+ uint32_t maxChannelCount = CubebUtils::MaxNumberOfChannels();
+ self->Dispatch(NS_NewRunnableFunction(
+ "MaxChannelCountUpdateToMainThread",
+ [self{self}, maxChannelCount]() {
+ class MessageToGraph : public ControlMessage {
+ public:
+ explicit MessageToGraph(MediaTrackGraph* aGraph,
+ uint32_t aMaxChannelCount)
+ : ControlMessage(nullptr),
+ mGraphImpl(static_cast<MediaTrackGraphImpl*>(aGraph)),
+ mMaxChannelCount(aMaxChannelCount) {}
+ void Run() override {
+ TRACE("MTG::SetMaxOutputChannelCount ControlMessage")
+ mGraphImpl->SetMaxOutputChannelCount(mMaxChannelCount);
+ }
+ MediaTrackGraphImpl* mGraphImpl;
+ uint32_t mMaxChannelCount;
+ };
+ self->AppendMessage(
+ MakeUnique<MessageToGraph>(self, maxChannelCount));
+ }));
+ }));
+
+ AppendMessage(MakeUnique<Message>(this));
+}
+
+static const char* GetAudioInputTypeString(const AudioInputType& aType) {
+ return aType == AudioInputType::Voice ? "Voice" : "Unknown";
+}
+
+void MediaTrackGraph::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
+ MOZ_ASSERT(OnGraphThread());
+ auto* impl = static_cast<MediaTrackGraphImpl*>(this);
+ impl->ReevaluateInputDevice(aID);
+}
+
+void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) {
+ MOZ_ASSERT(OnGraphThread());
+ LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: device %p", this, aID));
+
+ DeviceInputTrack* track =
+ mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
+ if (!track) {
+ LOG(LogLevel::Debug,
+ ("%p: No DeviceInputTrack for this device. Ignore", this));
+ return;
+ }
+
+ bool needToSwitch = false;
+
+ if (NonNativeInputTrack* nonNative = track->AsNonNativeInputTrack()) {
+ if (nonNative->NumberOfChannels() != AudioInputChannelCount(aID)) {
+ LOG(LogLevel::Debug,
+ ("%p: %u-channel non-native input device %p (track %p) is "
+ "re-configured to %d-channel",
+ this, nonNative->NumberOfChannels(), aID, track,
+ AudioInputChannelCount(aID)));
+ needToSwitch = true;
+ }
+ if (nonNative->DevicePreference() != AudioInputDevicePreference(aID)) {
+ LOG(LogLevel::Debug,
+ ("%p: %s-type non-native input device %p (track %p) is re-configured "
+ "to %s-type",
+ this, GetAudioInputTypeString(nonNative->DevicePreference()), aID,
+ track, GetAudioInputTypeString(AudioInputDevicePreference(aID))));
+ needToSwitch = true;
+ }
+
+ if (needToSwitch) {
+ nonNative->StopAudio();
+ nonNative->StartAudio(MakeRefPtr<AudioInputSource>(
+ MakeRefPtr<AudioInputSourceListener>(nonNative),
+ nonNative->GenerateSourceId(), aID, AudioInputChannelCount(aID),
+ AudioInputDevicePreference(aID) == AudioInputType::Voice,
+ nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate()));
+ }
+
+ return;
+ }
+
+ MOZ_ASSERT(track->AsNativeInputTrack());
+
+ if (AudioCallbackDriver* audioCallbackDriver =
+ CurrentDriver()->AsAudioCallbackDriver()) {
+ if (audioCallbackDriver->InputChannelCount() !=
+ AudioInputChannelCount(aID)) {
+ LOG(LogLevel::Debug,
+ ("%p: ReevaluateInputDevice: %u-channel AudioCallbackDriver %p is "
+ "re-configured to %d-channel",
+ this, audioCallbackDriver->InputChannelCount(), audioCallbackDriver,
+ AudioInputChannelCount(aID)));
+ needToSwitch = true;
+ }
+ if (audioCallbackDriver->InputDevicePreference() !=
+ AudioInputDevicePreference(aID)) {
+ LOG(LogLevel::Debug,
+ ("%p: ReevaluateInputDevice: %s-type AudioCallbackDriver %p is "
+ "re-configured to %s-type",
+ this,
+ GetAudioInputTypeString(
+ audioCallbackDriver->InputDevicePreference()),
+ audioCallbackDriver,
+ GetAudioInputTypeString(AudioInputDevicePreference(aID))));
+ needToSwitch = true;
+ }
+ } else if (Switching() && NextDriver()->AsAudioCallbackDriver()) {
+ // We're already in the process of switching to a audio callback driver,
+ // which will happen at the next iteration.
+ // However, maybe it's not the correct number of channels. Re-query the
+ // correct channel amount at this time.
+ needToSwitch = true;
+ }
+
+ if (needToSwitch) {
+ AudioCallbackDriver* newDriver = new AudioCallbackDriver(
+ this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(),
+ AudioInputChannelCount(aID), PrimaryOutputDeviceID(), aID,
+ AudioInputDevicePreference(aID));
+ SwitchAtNextIteration(newDriver);
+ }
+}
+
+bool MediaTrackGraphImpl::OnGraphThreadOrNotRunning() const {
+ // either we're on the right thread (and calling CurrentDriver() is safe),
+ // or we're going to fail the assert anyway, so don't cross-check
+ // via CurrentDriver().
+ return mGraphDriverRunning ? OnGraphThread() : NS_IsMainThread();
+}
+
+bool MediaTrackGraphImpl::OnGraphThread() const {
+ // we're on the right thread (and calling mDriver is safe),
+ MOZ_ASSERT(mDriver);
+ if (mGraphRunner && mGraphRunner->OnThread()) {
+ return true;
+ }
+ return mDriver->OnThread();
+}
+
+bool MediaTrackGraphImpl::Destroyed() const {
+ MOZ_ASSERT(NS_IsMainThread());
+ return !mSelfRef;
+}
+
+bool MediaTrackGraphImpl::ShouldUpdateMainThread() {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+ if (mRealtime) {
+ return true;
+ }
+
+ TimeStamp now = TimeStamp::Now();
+ // For offline graphs, update now if it has been long enough since the last
+ // update, or if it has reached the end.
+ if ((now - mLastMainThreadUpdate).ToMilliseconds() >
+ CurrentDriver()->IterationDuration() ||
+ mStateComputedTime >= mEndTime) {
+ mLastMainThreadUpdate = now;
+ return true;
+ }
+ return false;
+}
+
+void MediaTrackGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+ mMonitor.AssertCurrentThreadOwns();
+
+ // We don't want to frequently update the main thread about timing update
+ // when we are not running in realtime.
+ if (aFinalUpdate || ShouldUpdateMainThread()) {
+ // Strip updates that will be obsoleted below, so as to keep the length of
+ // mTrackUpdates sane.
+ size_t keptUpdateCount = 0;
+ for (size_t i = 0; i < mTrackUpdates.Length(); ++i) {
+ MediaTrack* track = mTrackUpdates[i].mTrack;
+ // RemoveTrackGraphThread() clears mTrack in updates for
+ // tracks that are removed from the graph.
+ MOZ_ASSERT(!track || track->GraphImpl() == this);
+ if (!track || track->MainThreadNeedsUpdates()) {
+ // Discard this update as it has either been cleared when the track
+ // was destroyed or there will be a newer update below.
+ continue;
+ }
+ if (keptUpdateCount != i) {
+ mTrackUpdates[keptUpdateCount] = std::move(mTrackUpdates[i]);
+ MOZ_ASSERT(!mTrackUpdates[i].mTrack);
+ }
+ ++keptUpdateCount;
+ }
+ mTrackUpdates.TruncateLength(keptUpdateCount);
+
+ mTrackUpdates.SetCapacity(mTrackUpdates.Length() + mTracks.Length() +
+ mSuspendedTracks.Length());
+ for (MediaTrack* track : AllTracks()) {
+ if (!track->MainThreadNeedsUpdates()) {
+ continue;
+ }
+ TrackUpdate* update = mTrackUpdates.AppendElement();
+ update->mTrack = track;
+ // No blocking to worry about here, since we've passed
+ // UpdateCurrentTimeForTracks.
+ update->mNextMainThreadCurrentTime =
+ track->GraphTimeToTrackTime(mProcessedTime);
+ update->mNextMainThreadEnded = track->mNotifiedEnded;
+ }
+ mNextMainThreadGraphTime = mProcessedTime;
+ if (!mPendingUpdateRunnables.IsEmpty()) {
+ mUpdateRunnables.AppendElements(std::move(mPendingUpdateRunnables));
+ }
+ }
+
+ // If this is the final update, then a stable state event will soon be
+ // posted just before this thread finishes, and so there is no need to also
+ // post here.
+ if (!aFinalUpdate &&
+ // Don't send the message to the main thread if it's not going to have
+ // any work to do.
+ !(mUpdateRunnables.IsEmpty() && mTrackUpdates.IsEmpty())) {
+ EnsureStableStateEventPosted();
+ }
+}
+
+GraphTime MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(GraphTime aTime) {
+ if (aTime % WEBAUDIO_BLOCK_SIZE == 0) {
+ return aTime;
+ }
+ return RoundUpToNextAudioBlock(aTime);
+}
+
+GraphTime MediaTrackGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime) {
+ uint64_t block = aTime >> WEBAUDIO_BLOCK_SIZE_BITS;
+ uint64_t nextBlock = block + 1;
+ GraphTime nextTime = nextBlock << WEBAUDIO_BLOCK_SIZE_BITS;
+ return nextTime;
+}
+
+void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock(
+ uint32_t aTrackIndex, TrackRate aSampleRate) {
+ MOZ_ASSERT(OnGraphThread());
+ MOZ_ASSERT(aTrackIndex <= mFirstCycleBreaker,
+ "Cycle breaker is not AudioNodeTrack?");
+
+ while (mProcessedTime < mStateComputedTime) {
+ // Microtask checkpoints are in between render quanta.
+ nsAutoMicroTask mt;
+
+ GraphTime next = RoundUpToNextAudioBlock(mProcessedTime);
+ for (uint32_t i = mFirstCycleBreaker; i < mTracks.Length(); ++i) {
+ auto nt = static_cast<AudioNodeTrack*>(mTracks[i]);
+ MOZ_ASSERT(nt->AsAudioNodeTrack());
+ nt->ProduceOutputBeforeInput(mProcessedTime);
+ }
+ for (uint32_t i = aTrackIndex; i < mTracks.Length(); ++i) {
+ ProcessedMediaTrack* pt = mTracks[i]->AsProcessedTrack();
+ if (pt) {
+ pt->ProcessInput(
+ mProcessedTime, next,
+ (next == mStateComputedTime) ? ProcessedMediaTrack::ALLOW_END : 0);
+ }
+ }
+ mProcessedTime = next;
+ }
+ NS_ASSERTION(mProcessedTime == mStateComputedTime,
+ "Something went wrong with rounding to block boundaries");
+}
+
+void MediaTrackGraphImpl::RunMessageAfterProcessing(
+ UniquePtr<ControlMessageInterface> aMessage) {
+ MOZ_ASSERT(OnGraphThread());
+
+ if (mFrontMessageQueue.IsEmpty()) {
+ mFrontMessageQueue.AppendElement();
+ }
+
+ // Only one block is used for messages from the graph thread.
+ MOZ_ASSERT(mFrontMessageQueue.Length() == 1);
+ mFrontMessageQueue[0].mMessages.AppendElement(std::move(aMessage));
+}
+
+void MediaTrackGraphImpl::RunMessagesInQueue() {
+ TRACE("MTG::RunMessagesInQueue");
+ MOZ_ASSERT(OnGraphThread());
+ // Calculate independent action times for each batch of messages (each
+ // batch corresponding to an event loop task). This isolates the performance
+ // of different scripts to some extent.
+ for (uint32_t i = 0; i < mFrontMessageQueue.Length(); ++i) {
+ nsTArray<UniquePtr<ControlMessageInterface>>& messages =
+ mFrontMessageQueue[i].mMessages;
+
+ for (uint32_t j = 0; j < messages.Length(); ++j) {
+ TRACE("ControlMessage::Run");
+ messages[j]->Run();
+ }
+ }
+ mFrontMessageQueue.Clear();
+}
+
+void MediaTrackGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) {
+ TRACE("MTG::UpdateGraph");
+ MOZ_ASSERT(OnGraphThread());
+ MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime);
+ // The next state computed time can be the same as the previous: it
+ // means the driver would have been blocking indefinitly, but the graph has
+ // been woken up right after having been to sleep.
+ MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime);
+
+ CheckDriver();
+ UpdateTrackOrder();
+
+ // Always do another iteration if there are tracks waiting to resume.
+ bool ensureNextIteration = !mPendingResumeOperations.IsEmpty();
+
+ for (MediaTrack* track : mTracks) {
+ if (SourceMediaTrack* is = track->AsSourceTrack()) {
+ ensureNextIteration |= is->PullNewData(aEndBlockingDecisions);
+ is->ExtractPendingInput(mStateComputedTime, aEndBlockingDecisions);
+ }
+ if (track->mEnded) {
+ // The track's not suspended, and since it's ended, underruns won't
+ // stop it playing out. So there's no blocking other than what we impose
+ // here.
+ GraphTime endTime = track->GetEnd() + track->mStartTime;
+ if (endTime <= mStateComputedTime) {
+ LOG(LogLevel::Verbose,
+ ("%p: MediaTrack %p is blocked due to being ended", this, track));
+ track->mStartBlocking = mStateComputedTime;
+ } else {
+ LOG(LogLevel::Verbose,
+ ("%p: MediaTrack %p has ended, but is not blocked yet (current "
+ "time %f, end at %f)",
+ this, track, MediaTimeToSeconds(mStateComputedTime),
+ MediaTimeToSeconds(endTime)));
+ // Data can't be added to a ended track, so underruns are irrelevant.
+ MOZ_ASSERT(endTime <= aEndBlockingDecisions);
+ track->mStartBlocking = endTime;
+ }
+ } else {
+ track->mStartBlocking = WillUnderrun(track, aEndBlockingDecisions);
+
+#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
+ if (SourceMediaTrack* s = track->AsSourceTrack()) {
+ if (s->Ended()) {
+ continue;
+ }
+ {
+ MutexAutoLock lock(s->mMutex);
+ if (!s->mUpdateTrack->mPullingEnabled) {
+ // The invariant that data must be provided is only enforced when
+ // pulling.
+ continue;
+ }
+ }
+ if (track->GetEnd() <
+ track->GraphTimeToTrackTime(aEndBlockingDecisions)) {
+ LOG(LogLevel::Error,
+ ("%p: SourceMediaTrack %p (%s) is live and pulled, "
+ "but wasn't fed "
+ "enough data. TrackListeners=%zu. Track-end=%f, "
+ "Iteration-end=%f",
+ this, track,
+ (track->mType == MediaSegment::AUDIO ? "audio" : "video"),
+ track->mTrackListeners.Length(),
+ MediaTimeToSeconds(track->GetEnd()),
+ MediaTimeToSeconds(
+ track->GraphTimeToTrackTime(aEndBlockingDecisions))));
+ MOZ_DIAGNOSTIC_ASSERT(false,
+ "A non-ended SourceMediaTrack wasn't fed "
+ "enough data by NotifyPull");
+ }
+ }
+#endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */
+ }
+ }
+
+ for (MediaTrack* track : mSuspendedTracks) {
+ track->mStartBlocking = mStateComputedTime;
+ }
+
+ // If the loop is woken up so soon that IterationEnd() barely advances or
+ // if an offline graph is not currently rendering, we end up having
+ // aEndBlockingDecisions == mStateComputedTime.
+ // Since the process interval [mStateComputedTime, aEndBlockingDecision) is
+ // empty, Process() will not find any unblocked track and so will not
+ // ensure another iteration. If the graph should be rendering, then ensure
+ // another iteration to render.
+ if (ensureNextIteration || (aEndBlockingDecisions == mStateComputedTime &&
+ mStateComputedTime < mEndTime)) {
+ EnsureNextIteration();
+ }
+}
+
+void MediaTrackGraphImpl::SelectOutputDeviceForAEC() {
+ MOZ_ASSERT(OnGraphThread());
+ size_t currentDeviceIndex = mOutputDevices.IndexOf(mOutputDeviceForAEC);
+ if (currentDeviceIndex == mOutputDevices.NoIndex) {
+ // Outputs for this device have been removed.
+ // Fall back to the primary output device.
+ mOutputDeviceForAEC = PrimaryOutputDeviceID();
+ currentDeviceIndex = 0;
+ MOZ_ASSERT(mOutputDevices[0].mDeviceID == mOutputDeviceForAEC);
+ }
+ if (mOutputDevices.Length() == 1) {
+ // No other output devices so there is no choice.
+ return;
+ }
+
+ // The output is considered silent intentionally only if the whole duration
+ // (often more than just this processing interval) of audio data in the
+ // MediaSegment is null so as to reduce switching between output devices
+ // should there be short durations of silence.
+ auto HasNonNullAudio = [](const TrackAndVolume& aTV) {
+ return aTV.mVolume != 0 && !aTV.mTrack->IsSuspended() &&
+ !aTV.mTrack->GetData()->IsNull();
+ };
+ // Keep using the same output device stream if it has non-null data,
+ // so as to stay with a stream having ongoing audio. If the output stream
+ // is switched, the echo cancellation algorithm can take some time to adjust
+ // to the change in delay, so there is less value in switching back and
+ // forth between output devices for very short sounds.
+ for (const auto& output : mOutputDevices[currentDeviceIndex].mTrackOutputs) {
+ if (HasNonNullAudio(output)) {
+ return;
+ }
+ }
+ // The current output device is silent. Use another if it has non-null data.
+ for (const auto& outputDeviceEntry : mOutputDevices) {
+ for (const auto& output : outputDeviceEntry.mTrackOutputs) {
+ if (HasNonNullAudio(output)) {
+ // Switch to this device.
+ mOutputDeviceForAEC = outputDeviceEntry.mDeviceID;
+ return;
+ }
+ }
+ }
+ // Null data for all outputs. Keep using the same device.
+}
+
+void MediaTrackGraphImpl::Process(MixerCallbackReceiver* aMixerReceiver) {
+ TRACE("MTG::Process");
+ MOZ_ASSERT(OnGraphThread());
+ // Play track contents.
+ bool allBlockedForever = true;
+ // True when we've done ProcessInput for all processed tracks.
+ bool doneAllProducing = false;
+ const GraphTime oldProcessedTime = mProcessedTime;
+
+ // Figure out what each track wants to do
+ for (uint32_t i = 0; i < mTracks.Length(); ++i) {
+ MediaTrack* track = mTracks[i];
+ if (!doneAllProducing) {
+ ProcessedMediaTrack* pt = track->AsProcessedTrack();
+ if (pt) {
+ AudioNodeTrack* n = track->AsAudioNodeTrack();
+ if (n) {
+#ifdef DEBUG
+ // Verify that the sampling rate for all of the following tracks is
+ // the same
+ for (uint32_t j = i + 1; j < mTracks.Length(); ++j) {
+ AudioNodeTrack* nextTrack = mTracks[j]->AsAudioNodeTrack();
+ if (nextTrack) {
+ MOZ_ASSERT(n->mSampleRate == nextTrack->mSampleRate,
+ "All AudioNodeTracks in the graph must have the same "
+ "sampling rate");
+ }
+ }
+#endif
+ // Since an AudioNodeTrack is present, go ahead and
+ // produce audio block by block for all the rest of the tracks.
+ ProduceDataForTracksBlockByBlock(i, n->mSampleRate);
+ doneAllProducing = true;
+ } else {
+ pt->ProcessInput(mProcessedTime, mStateComputedTime,
+ ProcessedMediaTrack::ALLOW_END);
+ // Assert that a live track produced enough data
+ MOZ_ASSERT_IF(!track->mEnded,
+ track->GetEnd() >= GraphTimeToTrackTimeWithBlocking(
+ track, mStateComputedTime));
+ }
+ }
+ }
+ if (track->mStartBlocking > oldProcessedTime) {
+ allBlockedForever = false;
+ }
+ }
+ mProcessedTime = mStateComputedTime;
+
+ SelectOutputDeviceForAEC();
+ for (const auto& outputDeviceEntry : mOutputDevices) {
+ uint32_t outputChannelCount;
+ if (!outputDeviceEntry.mReceiver) { // primary output
+ if (!aMixerReceiver) {
+ // Running off a system clock driver. No need to mix output.
+ continue;
+ }
+ MOZ_ASSERT(CurrentDriver()->AsAudioCallbackDriver(),
+ "Driver must be AudioCallbackDriver if aMixerReceiver");
+ // Use the number of channel the driver expects: this is the number of
+ // channel that can be output by the underlying system level audio stream.
+ outputChannelCount =
+ CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
+ } else {
+ outputChannelCount = AudioOutputChannelCount(outputDeviceEntry);
+ }
+ MOZ_ASSERT(mRealtime,
+ "If there's an output device, this graph must be realtime");
+ mMixer.StartMixing();
+ // This is the number of frames that are written to the output buffer, for
+ // this iteration.
+ TrackTime ticksPlayed = 0;
+ for (const auto& t : outputDeviceEntry.mTrackOutputs) {
+ TrackTime ticksPlayedForThisTrack =
+ PlayAudio(t, oldProcessedTime, outputChannelCount);
+ if (ticksPlayed == 0) {
+ ticksPlayed = ticksPlayedForThisTrack;
+ } else {
+ MOZ_ASSERT(
+ !ticksPlayedForThisTrack || ticksPlayedForThisTrack == ticksPlayed,
+ "Each track should have the same number of frames.");
+ }
+ }
+
+ if (ticksPlayed == 0) {
+ // Nothing was played, so the mixer doesn't know how many frames were
+ // processed. We still tell it so AudioCallbackDriver knows how much has
+ // been processed. (bug 1406027)
+ mMixer.Mix(nullptr, outputChannelCount,
+ mStateComputedTime - oldProcessedTime, mSampleRate);
+ }
+ AudioChunk* outputChunk = mMixer.MixedChunk();
+ if (outputDeviceEntry.mDeviceID == mOutputDeviceForAEC) {
+ // Callback any observers for the AEC speaker data. Note that one
+ // (maybe) of these will be full-duplex, the others will get their input
+ // data off separate cubeb callbacks.
+ NotifyOutputData(*outputChunk);
+ }
+ if (!outputDeviceEntry.mReceiver) { // primary output
+ aMixerReceiver->MixerCallback(outputChunk, mSampleRate);
+ } else {
+ outputDeviceEntry.mReceiver->EnqueueAudio(*outputChunk);
+ }
+ }
+
+ if (!allBlockedForever) {
+ EnsureNextIteration();
+ }
+}
+
+bool MediaTrackGraphImpl::UpdateMainThreadState() {
+ MOZ_ASSERT(OnGraphThread());
+ if (mForceShutDownReceived) {
+ for (MediaTrack* track : AllTracks()) {
+ track->OnGraphThreadDone();
+ }
+ }
+ {
+ MonitorAutoLock lock(mMonitor);
+ bool finalUpdate =
+ mForceShutDownReceived || (IsEmpty() && mBackMessageQueue.IsEmpty());
+ PrepareUpdatesToMainThreadState(finalUpdate);
+ if (!finalUpdate) {
+ SwapMessageQueues();
+ return true;
+ }
+ // The JSContext will not be used again.
+ // Clear main thread access while under monitor.
+ mJSContext = nullptr;
+ }
+ dom::WorkletThread::DeleteCycleCollectedJSContext();
+ // Enter shutdown mode when this iteration is completed.
+ // No need to Destroy tracks here. The main-thread owner of each
+ // track is responsible for calling Destroy on them.
+ return false;
+}
+
+auto MediaTrackGraphImpl::OneIteration(GraphTime aStateTime,
+ GraphTime aIterationEnd,
+ MixerCallbackReceiver* aMixerReceiver)
+ -> IterationResult {
+ if (mGraphRunner) {
+ return mGraphRunner->OneIteration(aStateTime, aIterationEnd,
+ aMixerReceiver);
+ }
+
+ return OneIterationImpl(aStateTime, aIterationEnd, aMixerReceiver);
+}
+
+auto MediaTrackGraphImpl::OneIterationImpl(
+ GraphTime aStateTime, GraphTime aIterationEnd,
+ MixerCallbackReceiver* aMixerReceiver) -> IterationResult {
+ TRACE("MTG::OneIterationImpl");
+
+ mIterationEndTime = aIterationEnd;
+
+ if (SoftRealTimeLimitReached()) {
+ TRACE("MTG::Demoting real-time thread!");
+ DemoteThreadFromRealTime();
+ }
+
+ // Changes to LIFECYCLE_RUNNING occur before starting or reviving the graph
+ // thread, and so the monitor need not be held to check mLifecycleState.
+ // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
+ // graphs that have not started.
+
+ // While changes occur on mainthread, this assert confirms that
+ // this code shouldn't run if mainthread might be changing the state (to
+ // > LIFECYCLE_RUNNING)
+
+ // Ignore mutex warning: static during execution of the graph
+ MOZ_PUSH_IGNORE_THREAD_SAFETY
+ MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
+ MOZ_POP_THREAD_SAFETY
+
+ MOZ_ASSERT(OnGraphThread());
+
+ WebCore::DenormalDisabler disabler;
+
+ // Process graph message from the main thread for this iteration.
+ RunMessagesInQueue();
+
+ // Process MessagePort events.
+ // These require a single thread, which has an nsThread with an event queue.
+ if (mGraphRunner || !mRealtime) {
+ TRACE("MTG::MessagePort events");
+ NS_ProcessPendingEvents(nullptr);
+ }
+
+ GraphTime stateTime = std::min(aStateTime, GraphTime(mEndTime));
+ UpdateGraph(stateTime);
+
+ mStateComputedTime = stateTime;
+
+ GraphTime oldProcessedTime = mProcessedTime;
+ Process(aMixerReceiver);
+ MOZ_ASSERT(mProcessedTime == stateTime);
+
+ UpdateCurrentTimeForTracks(oldProcessedTime);
+
+ ProcessChunkMetadata(oldProcessedTime);
+
+ // Process graph messages queued from RunMessageAfterProcessing() on this
+ // thread during the iteration.
+ RunMessagesInQueue();
+
+ if (!UpdateMainThreadState()) {
+ if (Switching()) {
+ // We'll never get to do this switch. Clear mNextDriver to break the
+ // ref-cycle graph->nextDriver->currentDriver->graph.
+ SwitchAtNextIteration(nullptr);
+ }
+ return IterationResult::CreateStop(
+ NewRunnableMethod("MediaTrackGraphImpl::SignalMainThreadCleanup", this,
+ &MediaTrackGraphImpl::SignalMainThreadCleanup));
+ }
+
+ if (Switching()) {
+ RefPtr<GraphDriver> nextDriver = std::move(mNextDriver);
+ return IterationResult::CreateSwitchDriver(
+ nextDriver, NewRunnableMethod<StoreRefPtrPassByPtr<GraphDriver>>(
+ "MediaTrackGraphImpl::SetCurrentDriver", this,
+ &MediaTrackGraphImpl::SetCurrentDriver, nextDriver));
+ }
+
+ return IterationResult::CreateStillProcessing();
+}
+
+void MediaTrackGraphImpl::ApplyTrackUpdate(TrackUpdate* aUpdate) {
+ MOZ_ASSERT(NS_IsMainThread());
+ mMonitor.AssertCurrentThreadOwns();
+
+ MediaTrack* track = aUpdate->mTrack;
+ if (!track) return;
+ track->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime;
+ track->mMainThreadEnded = aUpdate->mNextMainThreadEnded;
+
+ if (track->ShouldNotifyTrackEnded()) {
+ track->NotifyMainThreadListeners();
+ }
+}
+
+void MediaTrackGraphImpl::ForceShutDown() {
+ MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
+ LOG(LogLevel::Debug, ("%p: MediaTrackGraph::ForceShutdown", this));
+
+ if (mShutdownBlocker) {
+ // Avoid waiting forever for a graph to shut down
+ // synchronously. Reports are that some 3rd-party audio drivers
+ // occasionally hang in shutdown (both for us and Chrome).
+ NS_NewTimerWithCallback(
+ getter_AddRefs(mShutdownTimer), this,
+ MediaTrackGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT,
+ nsITimer::TYPE_ONE_SHOT);
+ }
+
+ class Message final : public ControlMessage {
+ public:
+ explicit Message(MediaTrackGraphImpl* aGraph)
+ : ControlMessage(nullptr), mGraph(aGraph) {}
+ void Run() override {
+ TRACE("MTG::ForceShutdown ControlMessage");
+ mGraph->mForceShutDownReceived = true;
+ }
+ // The graph owns this message.
+ MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
+ };
+
+ if (mMainThreadTrackCount > 0 || mMainThreadPortCount > 0) {
+ // If both the track and port counts are zero, the regular shutdown
+ // sequence will progress shortly to shutdown threads and destroy the graph.
+ AppendMessage(MakeUnique<Message>(this));
+ InterruptJS();
+ }
+}
+
+NS_IMETHODIMP
+MediaTrackGraphImpl::Notify(nsITimer* aTimer) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(!mShutdownBlocker, "MediaTrackGraph took too long to shut down!");
+ // Sigh, graph took too long to shut down. Stop blocking system
+ // shutdown and hope all is well.
+ RemoveShutdownBlocker();
+ return NS_OK;
+}
+
+static nsCString GetDocumentTitle(uint64_t aWindowID) {
+ MOZ_ASSERT(NS_IsMainThread());
+ nsCString title;
+ auto* win = nsGlobalWindowInner::GetInnerWindowWithId(aWindowID);
+ if (!win) {
+ return title;
+ }
+ Document* doc = win->GetExtantDoc();
+ if (!doc) {
+ return title;
+ }
+ nsAutoString titleUTF16;
+ doc->GetTitle(titleUTF16);
+ CopyUTF16toUTF8(titleUTF16, title);
+ return title;
+}
+
+NS_IMETHODIMP
+MediaTrackGraphImpl::Observe(nsISupports* aSubject, const char* aTopic,
+ const char16_t* aData) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(strcmp(aTopic, "document-title-changed") == 0);
+ nsCString streamName = GetDocumentTitle(mWindowID);
+ LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get()));
+ if (streamName.IsEmpty()) {
+ return NS_OK;
+ }
+ QueueControlMessageWithNoShutdown(
+ [self = RefPtr{this}, this, streamName = std::move(streamName)] {
+ CurrentDriver()->SetStreamName(streamName);
+ });
+ return NS_OK;
+}
+
+bool MediaTrackGraphImpl::AddShutdownBlocker() {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(!mShutdownBlocker);
+
+ class Blocker : public media::ShutdownBlocker {
+ const RefPtr<MediaTrackGraphImpl> mGraph;
+
+ public:
+ Blocker(MediaTrackGraphImpl* aGraph, const nsString& aName)
+ : media::ShutdownBlocker(aName), mGraph(aGraph) {}
+
+ NS_IMETHOD
+ BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override {
+ mGraph->ForceShutDown();
+ return NS_OK;
+ }
+ };
+
+ nsCOMPtr<nsIAsyncShutdownClient> barrier = media::GetShutdownBarrier();
+ if (!barrier) {
+ // We're already shutting down, we won't be able to add a blocker, bail.
+ LOG(LogLevel::Error,
+ ("%p: Couldn't get shutdown barrier, won't add shutdown blocker",
+ this));
+ return false;
+ }
+
+ // Blocker names must be distinct.
+ nsString blockerName;
+ blockerName.AppendPrintf("MediaTrackGraph %p shutdown", this);
+ mShutdownBlocker = MakeAndAddRef<Blocker>(this, blockerName);
+ nsresult rv = barrier->AddBlocker(mShutdownBlocker,
+ NS_LITERAL_STRING_FROM_CSTRING(__FILE__),
+ __LINE__, u"MediaTrackGraph shutdown"_ns);
+ MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
+ return true;
+}
+
+void MediaTrackGraphImpl::RemoveShutdownBlocker() {
+ if (!mShutdownBlocker) {
+ return;
+ }
+ media::MustGetShutdownBarrier()->RemoveBlocker(mShutdownBlocker);
+ mShutdownBlocker = nullptr;
+}
+
+NS_IMETHODIMP
+MediaTrackGraphImpl::GetName(nsACString& aName) {
+ aName.AssignLiteral("MediaTrackGraphImpl");
+ return NS_OK;
+}
+
+namespace {
+
+class MediaTrackGraphShutDownRunnable : public Runnable {
+ public:
+ explicit MediaTrackGraphShutDownRunnable(MediaTrackGraphImpl* aGraph)
+ : Runnable("MediaTrackGraphShutDownRunnable"), mGraph(aGraph) {}
+ // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
+ // See bug 1535398.
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
+ TRACE("MTG::MediaTrackGraphShutDownRunnable runnable");
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(!mGraph->mGraphDriverRunning && mGraph->mDriver,
+ "We should know the graph thread control loop isn't running!");
+
+ LOG(LogLevel::Debug, ("%p: Shutting down graph", mGraph.get()));
+
+ // We've asserted the graph isn't running. Use mDriver instead of
+ // CurrentDriver to avoid thread-safety checks
+#if 0 // AudioCallbackDrivers are released asynchronously anyways
+ // XXX a better test would be have setting mGraphDriverRunning make sure
+ // any current callback has finished and block future ones -- or just
+ // handle it all in Shutdown()!
+ if (mGraph->mDriver->AsAudioCallbackDriver()) {
+ MOZ_ASSERT(!mGraph->mDriver->AsAudioCallbackDriver()->InCallback());
+ }
+#endif
+
+ for (MediaTrackGraphImpl::PendingResumeOperation& op :
+ mGraph->mPendingResumeOperations) {
+ op.Abort();
+ }
+
+ if (mGraph->mGraphRunner) {
+ RefPtr<GraphRunner>(mGraph->mGraphRunner)->Shutdown();
+ }
+
+ RefPtr<GraphDriver>(mGraph->mDriver)->Shutdown();
+
+ // Release the driver now so that an AudioCallbackDriver will release its
+ // SharedThreadPool reference. Each SharedThreadPool reference must be
+ // released before SharedThreadPool::SpinUntilEmpty() runs on
+ // xpcom-shutdown-threads. Don't wait for GC/CC to release references to
+ // objects owning tracks, or for expiration of mGraph->mShutdownTimer,
+ // which won't otherwise release its reference on the graph until
+ // nsTimerImpl::Shutdown(), which runs after xpcom-shutdown-threads.
+ mGraph->SetCurrentDriver(nullptr);
+
+ // Safe to access these without the monitor since the graph isn't running.
+ // We may be one of several graphs. Drop ticket to eventually unblock
+ // shutdown.
+ if (mGraph->mShutdownTimer && !mGraph->mShutdownBlocker) {
+ MOZ_ASSERT(
+ false,
+ "AudioCallbackDriver took too long to shut down and we let shutdown"
+ " continue - freezing and leaking");
+
+ // The timer fired, so we may be deeper in shutdown now. Block any
+ // further teardown and just leak, for safety.
+ return NS_OK;
+ }
+
+ // mGraph's thread is not running so it's OK to do whatever here
+ for (MediaTrack* track : mGraph->AllTracks()) {
+ // Clean up all MediaSegments since we cannot release Images too
+ // late during shutdown. Also notify listeners that they were removed
+ // so they can clean up any gfx resources.
+ track->RemoveAllResourcesAndListenersImpl();
+ }
+
+#ifdef DEBUG
+ {
+ MonitorAutoLock lock(mGraph->mMonitor);
+ MOZ_ASSERT(mGraph->mUpdateRunnables.IsEmpty());
+ }
+#endif
+ mGraph->mPendingUpdateRunnables.Clear();
+
+ mGraph->RemoveShutdownBlocker();
+
+ // We can't block past the final LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION
+ // stage, since completion of that stage requires all tracks to be freed,
+ // which requires shutdown to proceed.
+
+ if (mGraph->IsEmpty()) {
+ // mGraph is no longer needed, so delete it.
+ mGraph->Destroy();
+ } else {
+ // The graph is not empty. We must be in a forced shutdown.
+ // Some later AppendMessage will detect that the graph has
+ // been emptied, and delete it.
+ NS_ASSERTION(mGraph->mForceShutDownReceived, "Not in forced shutdown?");
+ mGraph->LifecycleStateRef() =
+ MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
+ }
+ return NS_OK;
+ }
+
+ private:
+ RefPtr<MediaTrackGraphImpl> mGraph;
+};
+
+class MediaTrackGraphStableStateRunnable : public Runnable {
+ public:
+ explicit MediaTrackGraphStableStateRunnable(MediaTrackGraphImpl* aGraph,
+ bool aSourceIsMTG)
+ : Runnable("MediaTrackGraphStableStateRunnable"),
+ mGraph(aGraph),
+ mSourceIsMTG(aSourceIsMTG) {}
+ NS_IMETHOD Run() override {
+ TRACE("MTG::MediaTrackGraphStableStateRunnable ControlMessage");
+ if (mGraph) {
+ mGraph->RunInStableState(mSourceIsMTG);
+ }
+ return NS_OK;
+ }
+
+ private:
+ RefPtr<MediaTrackGraphImpl> mGraph;
+ bool mSourceIsMTG;
+};
+
+/*
+ * Control messages forwarded from main thread to graph manager thread
+ */
+class CreateMessage : public ControlMessage {
+ public:
+ explicit CreateMessage(MediaTrack* aTrack) : ControlMessage(aTrack) {}
+ void Run() override {
+ TRACE("MTG::AddTrackGraphThread ControlMessage");
+ mTrack->GraphImpl()->AddTrackGraphThread(mTrack);
+ }
+ void RunDuringShutdown() override {
+ // Make sure to run this message during shutdown too, to make sure
+ // that we balance the number of tracks registered with the graph
+ // as they're destroyed during shutdown.
+ Run();
+ }
+};
+
+} // namespace
+
+void MediaTrackGraphImpl::RunInStableState(bool aSourceIsMTG) {
+ MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread");
+
+ nsTArray<nsCOMPtr<nsIRunnable>> runnables;
+ // When we're doing a forced shutdown, pending control messages may be
+ // run on the main thread via RunDuringShutdown. Those messages must
+ // run without the graph monitor being held. So, we collect them here.
+ nsTArray<UniquePtr<ControlMessageInterface>>
+ controlMessagesToRunDuringShutdown;
+
+ {
+ MonitorAutoLock lock(mMonitor);
+ if (aSourceIsMTG) {
+ MOZ_ASSERT(mPostedRunInStableStateEvent);
+ mPostedRunInStableStateEvent = false;
+ }
+
+ // This should be kept in sync with the LifecycleState enum in
+ // MediaTrackGraphImpl.h
+ const char* LifecycleState_str[] = {
+ "LIFECYCLE_THREAD_NOT_STARTED", "LIFECYCLE_RUNNING",
+ "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP",
+ "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN",
+ "LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION"};
+
+ if (LifecycleStateRef() != LIFECYCLE_RUNNING) {
+ LOG(LogLevel::Debug,
+ ("%p: Running stable state callback. Current state: %s", this,
+ LifecycleState_str[LifecycleStateRef()]));
+ }
+
+ runnables = std::move(mUpdateRunnables);
+ for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) {
+ TrackUpdate* update = &mTrackUpdates[i];
+ if (update->mTrack) {
+ ApplyTrackUpdate(update);
+ }
+ }
+ mTrackUpdates.Clear();
+
+ mMainThreadGraphTime = mNextMainThreadGraphTime;
+
+ if (mCurrentTaskMessageQueue.IsEmpty()) {
+ if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
+ IsEmpty()) {
+ // Complete shutdown. First, ensure that this graph is no longer used.
+ // A new graph graph will be created if one is needed.
+ // Asynchronously clean up old graph. We don't want to do this
+ // synchronously because it spins the event loop waiting for threads
+ // to shut down, and we don't want to do that in a stable state handler.
+ LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
+ LOG(LogLevel::Debug,
+ ("%p: Sending MediaTrackGraphShutDownRunnable", this));
+ nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
+ mMainThread->Dispatch(event.forget());
+ }
+ } else {
+ if (LifecycleStateRef() <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
+ MessageBlock* block = mBackMessageQueue.AppendElement();
+ block->mMessages = std::move(mCurrentTaskMessageQueue);
+ EnsureNextIteration();
+ }
+
+ // If this MediaTrackGraph has entered regular (non-forced) shutdown it
+ // is not able to process any more messages. Those messages being added to
+ // the graph in the first place is an error.
+ MOZ_DIAGNOSTIC_ASSERT(LifecycleStateRef() <
+ LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP ||
+ mForceShutDownReceived);
+ }
+
+ if (LifecycleStateRef() == LIFECYCLE_THREAD_NOT_STARTED) {
+ // Start the driver now. We couldn't start it earlier because the graph
+ // might exit immediately on finding it has no tracks. The first message
+ // for a new graph must create a track. Ensure that his message runs on
+ // the first iteration.
+ MOZ_ASSERT(MessagesQueued());
+ SwapMessageQueues();
+
+ LOG(LogLevel::Debug,
+ ("%p: Starting a graph with a %s", this,
+ CurrentDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver"
+ : "SystemClockDriver"));
+ LifecycleStateRef() = LIFECYCLE_RUNNING;
+ mGraphDriverRunning = true;
+ RefPtr<GraphDriver> driver = CurrentDriver();
+ driver->Start();
+ // It's not safe to Shutdown() a thread from StableState, and
+ // releasing this may shutdown a SystemClockDriver thread.
+ // Proxy the release to outside of StableState.
+ NS_ReleaseOnMainThread("MediaTrackGraphImpl::CurrentDriver",
+ driver.forget(),
+ true); // always proxy
+ }
+
+ if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP &&
+ mForceShutDownReceived) {
+ // Defer calls to RunDuringShutdown() to happen while mMonitor is not
+ // held.
+ for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) {
+ MessageBlock& mb = mBackMessageQueue[i];
+ controlMessagesToRunDuringShutdown.AppendElements(
+ std::move(mb.mMessages));
+ }
+ mBackMessageQueue.Clear();
+ MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty());
+ // Stop MediaTrackGraph threads.
+ LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
+ nsCOMPtr<nsIRunnable> event = new MediaTrackGraphShutDownRunnable(this);
+ mMainThread->Dispatch(event.forget());
+ }
+
+ mGraphDriverRunning = LifecycleStateRef() == LIFECYCLE_RUNNING;
+ }
+
+ // Make sure we get a new current time in the next event loop task
+ if (!aSourceIsMTG) {
+ MOZ_ASSERT(mPostedRunInStableState);
+ mPostedRunInStableState = false;
+ }
+
+ for (uint32_t i = 0; i < controlMessagesToRunDuringShutdown.Length(); ++i) {
+ controlMessagesToRunDuringShutdown[i]->RunDuringShutdown();
+ }
+
+#ifdef DEBUG
+ mCanRunMessagesSynchronously =
+ !mGraphDriverRunning &&
+ LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
+#endif
+
+ for (uint32_t i = 0; i < runnables.Length(); ++i) {
+ runnables[i]->Run();
+ }
+}
+
+void MediaTrackGraphImpl::EnsureRunInStableState() {
+ MOZ_ASSERT(NS_IsMainThread(), "main thread only");
+
+ if (mPostedRunInStableState) return;
+ mPostedRunInStableState = true;
+ nsCOMPtr<nsIRunnable> event =
+ new MediaTrackGraphStableStateRunnable(this, false);
+ nsContentUtils::RunInStableState(event.forget());
+}
+
+void MediaTrackGraphImpl::EnsureStableStateEventPosted() {
+ MOZ_ASSERT(OnGraphThread());
+ mMonitor.AssertCurrentThreadOwns();
+
+ if (mPostedRunInStableStateEvent) return;
+ mPostedRunInStableStateEvent = true;
+ nsCOMPtr<nsIRunnable> event =
+ new MediaTrackGraphStableStateRunnable(this, true);
+ mMainThread->Dispatch(event.forget());
+}
+
+void MediaTrackGraphImpl::SignalMainThreadCleanup() {
+ MOZ_ASSERT(mDriver->OnThread());
+
+ MonitorAutoLock lock(mMonitor);
+ // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline
+ // graphs that have not started.
+ MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING);
+ LOG(LogLevel::Debug,
+ ("%p: MediaTrackGraph waiting for main thread cleanup", this));
+ LifecycleStateRef() =
+ MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP;
+ EnsureStableStateEventPosted();
+}
+
+void MediaTrackGraphImpl::AppendMessage(
+ UniquePtr<ControlMessageInterface> aMessage) {
+ MOZ_ASSERT(NS_IsMainThread(), "main thread only");
+ MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0 || mMainThreadPortCount > 0);
+
+ if (!mGraphDriverRunning &&
+ LifecycleStateRef() > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) {
+ // The graph control loop is not running and main thread cleanup has
+ // happened. From now on we can't append messages to
+ // mCurrentTaskMessageQueue, because that will never be processed again, so
+ // just RunDuringShutdown this message. This should only happen during
+ // forced shutdown, or after a non-realtime graph has finished processing.
+#ifdef DEBUG
+ MOZ_ASSERT(mCanRunMessagesSynchronously);
+ mCanRunMessagesSynchronously = false;
+#endif
+ aMessage->RunDuringShutdown();
+#ifdef DEBUG
+ mCanRunMessagesSynchronously = true;
+#endif
+ if (IsEmpty() &&
+ LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION) {
+ Destroy();
+ }
+ return;
+ }
+
+ mCurrentTaskMessageQueue.AppendElement(std::move(aMessage));
+ EnsureRunInStableState();
+}
+
+void MediaTrackGraphImpl::Dispatch(already_AddRefed<nsIRunnable>&& aRunnable) {
+ mMainThread->Dispatch(std::move(aRunnable));
+}
+
+MediaTrack::MediaTrack(TrackRate aSampleRate, MediaSegment::Type aType,
+ MediaSegment* aSegment)
+ : mSampleRate(aSampleRate),
+ mType(aType),
+ mSegment(aSegment),
+ mStartTime(0),
+ mForgottenTime(0),
+ mEnded(false),
+ mNotifiedEnded(false),
+ mDisabledMode(DisabledTrackMode::ENABLED),
+ mStartBlocking(GRAPH_TIME_MAX),
+ mSuspendedCount(0),
+ mMainThreadCurrentTime(0),
+ mMainThreadEnded(false),
+ mEndedNotificationSent(false),
+ mMainThreadDestroyed(false),
+ mGraph(nullptr) {
+ MOZ_COUNT_CTOR(MediaTrack);
+ MOZ_ASSERT_IF(mSegment, mSegment->GetType() == aType);
+}
+
+MediaTrack::~MediaTrack() {
+ MOZ_COUNT_DTOR(MediaTrack);
+ NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already");
+ NS_ASSERTION(mMainThreadListeners.IsEmpty(),
+ "All main thread listeners should have been removed");
+}
+
+size_t MediaTrack::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const {
+ size_t amount = 0;
+
+ // Not owned:
+ // - mGraph - Not reported here
+ // - mConsumers - elements
+ // Future:
+ // - mLastPlayedVideoFrame
+ // - mTrackListeners - elements
+
+ amount += mTrackListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
+ amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf);
+ amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf);
+
+ return amount;
+}
+
+size_t MediaTrack::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
+ return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+}
+
+void MediaTrack::IncrementSuspendCount() {
+ ++mSuspendedCount;
+ if (mSuspendedCount != 1 || !mGraph) {
+ MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
+ return;
+ }
+ AssertOnGraphThreadOrNotRunning();
+ auto* graph = GraphImpl();
+ for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
+ mConsumers[i]->Suspended();
+ }
+ MOZ_ASSERT(graph->mTracks.Contains(this));
+ graph->mTracks.RemoveElement(this);
+ graph->mSuspendedTracks.AppendElement(this);
+ graph->SetTrackOrderDirty();
+}
+
+void MediaTrack::DecrementSuspendCount() {
+ MOZ_ASSERT(mSuspendedCount > 0, "Suspend count underrun");
+ --mSuspendedCount;
+ if (mSuspendedCount != 0 || !mGraph) {
+ MOZ_ASSERT(mGraph || mConsumers.IsEmpty());
+ return;
+ }
+ AssertOnGraphThreadOrNotRunning();
+ auto* graph = GraphImpl();
+ for (uint32_t i = 0; i < mConsumers.Length(); ++i) {
+ mConsumers[i]->Resumed();
+ }
+ MOZ_ASSERT(graph->mSuspendedTracks.Contains(this));
+ graph->mSuspendedTracks.RemoveElement(this);
+ graph->mTracks.AppendElement(this);
+ graph->SetTrackOrderDirty();
+}
+
+void ProcessedMediaTrack::DecrementSuspendCount() {
+ mCycleMarker = NOT_VISITED;
+ MediaTrack::DecrementSuspendCount();
+}
+
+MediaTrackGraphImpl* MediaTrack::GraphImpl() {
+ return static_cast<MediaTrackGraphImpl*>(mGraph);
+}
+
+const MediaTrackGraphImpl* MediaTrack::GraphImpl() const {
+ return static_cast<MediaTrackGraphImpl*>(mGraph);
+}
+
+void MediaTrack::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
+ MOZ_ASSERT(!mGraph, "Should only be called once");
+ MOZ_ASSERT(mSampleRate == aGraph->GraphRate());
+ mGraph = aGraph;
+}
+
+void MediaTrack::SetGraphImpl(MediaTrackGraph* aGraph) {
+ MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(aGraph);
+ SetGraphImpl(graph);
+}
+
+TrackTime MediaTrack::GraphTimeToTrackTime(GraphTime aTime) const {
+ NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
+ aTime <= mStartBlocking,
+ "Incorrectly ignoring blocking!");
+ return aTime - mStartTime;
+}
+
+GraphTime MediaTrack::TrackTimeToGraphTime(TrackTime aTime) const {
+ NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime ||
+ aTime + mStartTime <= mStartBlocking,
+ "Incorrectly ignoring blocking!");
+ return aTime + mStartTime;
+}
+
+TrackTime MediaTrack::GraphTimeToTrackTimeWithBlocking(GraphTime aTime) const {
+ return GraphImpl()->GraphTimeToTrackTimeWithBlocking(this, aTime);
+}
+
+void MediaTrack::RemoveAllResourcesAndListenersImpl() {
+ GraphImpl()->AssertOnGraphThreadOrNotRunning();
+
+ for (auto& l : mTrackListeners.Clone()) {
+ l->NotifyRemoved(Graph());
+ }
+ mTrackListeners.Clear();
+
+ RemoveAllDirectListenersImpl();
+
+ if (mSegment) {
+ mSegment->Clear();
+ }
+}
+
+void MediaTrack::DestroyImpl() {
+ for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
+ mConsumers[i]->Disconnect();
+ }
+ if (mSegment) {
+ mSegment->Clear();
+ }
+ mGraph = nullptr;
+}
+
+void MediaTrack::Destroy() {
+ // Keep this track alive until we leave this method
+ RefPtr<MediaTrack> kungFuDeathGrip = this;
+ // Keep a reference to the graph, since Message might RunDuringShutdown()
+ // synchronously and make GraphImpl() invalid.
+ RefPtr<MediaTrackGraphImpl> graph = GraphImpl();
+
+ QueueControlOrShutdownMessage(
+ [self = RefPtr{this}, this](IsInShutdown aInShutdown) {
+ if (aInShutdown == IsInShutdown::No) {
+ OnGraphThreadDone();
+ }
+ TRACE("MediaTrack::Destroy ControlMessage");
+ RemoveAllResourcesAndListenersImpl();
+ auto* graph = GraphImpl();
+ DestroyImpl();
+ graph->RemoveTrackGraphThread(this);
+ });
+ graph->RemoveTrack(this);
+ // Message::RunDuringShutdown may have removed this track from the graph,
+ // but our kungFuDeathGrip above will have kept this track alive if
+ // necessary.
+ mMainThreadDestroyed = true;
+}
+
+TrackTime MediaTrack::GetEnd() const {
+ return mSegment ? mSegment->GetDuration() : 0;
+}
+
+void MediaTrack::AddAudioOutput(void* aKey, const AudioDeviceInfo* aSink) {
+ MOZ_ASSERT(NS_IsMainThread());
+ AudioDeviceID deviceID = nullptr;
+ TrackRate preferredSampleRate = 0;
+ if (aSink) {
+ deviceID = aSink->DeviceID();
+ preferredSampleRate = static_cast<TrackRate>(aSink->DefaultRate());
+ }
+ AddAudioOutput(aKey, deviceID, preferredSampleRate);
+}
+
+void MediaTrack::AddAudioOutput(void* aKey, CubebUtils::AudioDeviceID aDeviceID,
+ TrackRate aPreferredSampleRate) {
+ MOZ_ASSERT(NS_IsMainThread());
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ LOG(LogLevel::Info, ("MediaTrack %p adding AudioOutput", this));
+ GraphImpl()->RegisterAudioOutput(this, aKey, aDeviceID, aPreferredSampleRate);
+}
+
+void MediaTrackGraphImpl::SetAudioOutputVolume(MediaTrack* aTrack, void* aKey,
+ float aVolume) {
+ MOZ_ASSERT(NS_IsMainThread());
+ for (auto& params : mAudioOutputParams) {
+ if (params.mKey == aKey && aTrack == params.mTrack) {
+ params.mVolume = aVolume;
+ UpdateAudioOutput(aTrack, params.mDeviceID);
+ return;
+ }
+ }
+ MOZ_CRASH("Audio output key not found when setting the volume.");
+}
+
+void MediaTrack::SetAudioOutputVolume(void* aKey, float aVolume) {
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ GraphImpl()->SetAudioOutputVolume(this, aKey, aVolume);
+}
+
+void MediaTrack::RemoveAudioOutput(void* aKey) {
+ MOZ_ASSERT(NS_IsMainThread());
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ LOG(LogLevel::Info, ("MediaTrack %p removing AudioOutput", this));
+ GraphImpl()->UnregisterAudioOutput(this, aKey);
+}
+
+void MediaTrackGraphImpl::RegisterAudioOutput(
+ MediaTrack* aTrack, void* aKey, CubebUtils::AudioDeviceID aDeviceID,
+ TrackRate aPreferredSampleRate) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(!mAudioOutputParams.Contains(TrackAndKey{aTrack, aKey}));
+
+ IncrementOutputDeviceRefCnt(aDeviceID, aPreferredSampleRate);
+
+ mAudioOutputParams.EmplaceBack(
+ TrackKeyDeviceAndVolume{aTrack, aKey, aDeviceID, 1.f});
+
+ UpdateAudioOutput(aTrack, aDeviceID);
+}
+
+void MediaTrackGraphImpl::UnregisterAudioOutput(MediaTrack* aTrack,
+ void* aKey) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ size_t index = mAudioOutputParams.IndexOf(TrackAndKey{aTrack, aKey});
+ MOZ_ASSERT(index != mAudioOutputParams.NoIndex);
+ AudioDeviceID deviceID = mAudioOutputParams[index].mDeviceID;
+ mAudioOutputParams.UnorderedRemoveElementAt(index);
+
+ UpdateAudioOutput(aTrack, deviceID);
+
+ DecrementOutputDeviceRefCnt(deviceID);
+}
+
+void MediaTrackGraphImpl::UpdateAudioOutput(MediaTrack* aTrack,
+ AudioDeviceID aDeviceID) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(!aTrack->IsDestroyed());
+
+ float volume = 0.f;
+ bool found = false;
+ for (const auto& params : mAudioOutputParams) {
+ if (params.mTrack == aTrack && params.mDeviceID == aDeviceID) {
+ volume += params.mVolume;
+ found = true;
+ }
+ }
+
+ QueueControlMessageWithNoShutdown(
+ // track has a strong reference to this.
+ [track = RefPtr{aTrack}, aDeviceID, volume, found] {
+ TRACE("MediaTrack::UpdateAudioOutput ControlMessage");
+ MediaTrackGraphImpl* graph = track->GraphImpl();
+ auto& outputDevicesRef = graph->mOutputDevices;
+ size_t deviceIndex = outputDevicesRef.IndexOf(aDeviceID);
+ MOZ_ASSERT(deviceIndex != outputDevicesRef.NoIndex);
+ auto& deviceOutputsRef = outputDevicesRef[deviceIndex].mTrackOutputs;
+ if (found) {
+ for (auto& outputRef : deviceOutputsRef) {
+ if (outputRef.mTrack == track) {
+ outputRef.mVolume = volume;
+ return;
+ }
+ }
+ deviceOutputsRef.EmplaceBack(TrackAndVolume{track, volume});
+ } else {
+ DebugOnly<bool> removed = deviceOutputsRef.RemoveElement(track);
+ MOZ_ASSERT(removed);
+ // mOutputDevices[0] is retained for AudioCallbackDriver output even
+ // when no tracks have audio outputs.
+ if (deviceIndex != 0 && deviceOutputsRef.IsEmpty()) {
+ // The device is no longer in use.
+ outputDevicesRef.UnorderedRemoveElementAt(deviceIndex);
+ }
+ }
+ });
+}
+
+void MediaTrackGraphImpl::IncrementOutputDeviceRefCnt(
+ AudioDeviceID aDeviceID, TrackRate aPreferredSampleRate) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ for (auto& elementRef : mOutputDeviceRefCnts) {
+ if (elementRef.mDeviceID == aDeviceID) {
+ ++elementRef.mRefCnt;
+ return;
+ }
+ }
+ MOZ_ASSERT(aDeviceID != mPrimaryOutputDeviceID,
+ "mOutputDeviceRefCnts should always have the primary device");
+ // Need to add an output device.
+ // Output via another graph for this device.
+ // This sample rate is not exposed to content.
+ TrackRate sampleRate =
+ aPreferredSampleRate != 0
+ ? aPreferredSampleRate
+ : static_cast<TrackRate>(CubebUtils::PreferredSampleRate(
+ /*aShouldResistFingerprinting*/ false));
+ MediaTrackGraph* newGraph = MediaTrackGraphImpl::GetInstance(
+ MediaTrackGraph::AUDIO_THREAD_DRIVER, mWindowID, sampleRate, aDeviceID,
+ GetMainThreadSerialEventTarget());
+ // CreateCrossGraphReceiver wants the sample rate of this graph.
+ RefPtr receiver = newGraph->CreateCrossGraphReceiver(mSampleRate);
+ receiver->AddAudioOutput(nullptr, aDeviceID, sampleRate);
+ mOutputDeviceRefCnts.EmplaceBack(
+ DeviceReceiverAndCount{aDeviceID, receiver, 1});
+
+ QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aDeviceID,
+ receiver = std::move(receiver)]() mutable {
+ TRACE("MediaTrackGraph add output device ControlMessage");
+ MOZ_ASSERT(!mOutputDevices.Contains(aDeviceID));
+ mOutputDevices.EmplaceBack(
+ OutputDeviceEntry{aDeviceID, std::move(receiver)});
+ });
+}
+
+void MediaTrackGraphImpl::DecrementOutputDeviceRefCnt(AudioDeviceID aDeviceID) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID);
+ MOZ_ASSERT(index != mOutputDeviceRefCnts.NoIndex);
+ // mOutputDeviceRefCnts[0] is retained for consistency with
+ // mOutputDevices[0], which is retained for AudioCallbackDriver output even
+ // when no tracks have audio outputs.
+ if (--mOutputDeviceRefCnts[index].mRefCnt == 0 && index != 0) {
+ mOutputDeviceRefCnts[index].mReceiver->Destroy();
+ mOutputDeviceRefCnts.UnorderedRemoveElementAt(index);
+ }
+}
+
+void MediaTrack::Suspend() {
+ // This can happen if this method has been called asynchronously, and the
+ // track has been destroyed since then.
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] {
+ TRACE("MediaTrack::IncrementSuspendCount ControlMessage");
+ IncrementSuspendCount();
+ });
+}
+
+void MediaTrack::Resume() {
+ // This can happen if this method has been called asynchronously, and the
+ // track has been destroyed since then.
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] {
+ TRACE("MediaTrack::DecrementSuspendCount ControlMessage");
+ DecrementSuspendCount();
+ });
+}
+
+void MediaTrack::AddListenerImpl(
+ already_AddRefed<MediaTrackListener> aListener) {
+ RefPtr<MediaTrackListener> l(aListener);
+ mTrackListeners.AppendElement(std::move(l));
+
+ PrincipalHandle lastPrincipalHandle = mSegment->GetLastPrincipalHandle();
+ mTrackListeners.LastElement()->NotifyPrincipalHandleChanged(
+ Graph(), lastPrincipalHandle);
+ if (mNotifiedEnded) {
+ mTrackListeners.LastElement()->NotifyEnded(Graph());
+ }
+ if (CombinedDisabledMode() == DisabledTrackMode::SILENCE_BLACK) {
+ mTrackListeners.LastElement()->NotifyEnabledStateChanged(Graph(), false);
+ }
+}
+
+void MediaTrack::AddListener(MediaTrackListener* aListener) {
+ MOZ_ASSERT(mSegment, "Segment-less tracks do not support listeners");
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ QueueControlMessageWithNoShutdown(
+ [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable {
+ TRACE("MediaTrack::AddListenerImpl ControlMessage");
+ AddListenerImpl(listener.forget());
+ });
+}
+
+void MediaTrack::RemoveListenerImpl(MediaTrackListener* aListener) {
+ for (size_t i = 0; i < mTrackListeners.Length(); ++i) {
+ if (mTrackListeners[i] == aListener) {
+ mTrackListeners[i]->NotifyRemoved(Graph());
+ mTrackListeners.RemoveElementAt(i);
+ return;
+ }
+ }
+}
+
+RefPtr<GenericPromise> MediaTrack::RemoveListener(
+ MediaTrackListener* aListener) {
+ MozPromiseHolder<GenericPromise> promiseHolder;
+ RefPtr<GenericPromise> p = promiseHolder.Ensure(__func__);
+ if (mMainThreadDestroyed) {
+ promiseHolder.Reject(NS_ERROR_FAILURE, __func__);
+ return p;
+ }
+ QueueControlOrShutdownMessage(
+ [self = RefPtr{this}, this, listener = RefPtr{aListener},
+ promiseHolder = std::move(promiseHolder)](IsInShutdown) mutable {
+ TRACE("MediaTrack::RemoveListenerImpl ControlMessage");
+ // During shutdown we still want the listener's NotifyRemoved to be
+ // called, since not doing that might block shutdown of other modules.
+ RemoveListenerImpl(listener);
+ promiseHolder.Resolve(true, __func__);
+ });
+ return p;
+}
+
+void MediaTrack::AddDirectListenerImpl(
+ already_AddRefed<DirectMediaTrackListener> aListener) {
+ AssertOnGraphThread();
+ // Base implementation, for tracks that don't support direct track listeners.
+ RefPtr<DirectMediaTrackListener> listener = aListener;
+ listener->NotifyDirectListenerInstalled(
+ DirectMediaTrackListener::InstallationResult::TRACK_NOT_SUPPORTED);
+}
+
+void MediaTrack::AddDirectListener(DirectMediaTrackListener* aListener) {
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ QueueControlMessageWithNoShutdown(
+ [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable {
+ TRACE("MediaTrack::AddDirectListenerImpl ControlMessage");
+ AddDirectListenerImpl(listener.forget());
+ });
+}
+
+void MediaTrack::RemoveDirectListenerImpl(DirectMediaTrackListener* aListener) {
+ // Base implementation, the listener was never added so nothing to do.
+}
+
+void MediaTrack::RemoveDirectListener(DirectMediaTrackListener* aListener) {
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ QueueControlOrShutdownMessage(
+ [self = RefPtr{this}, this, listener = RefPtr{aListener}](IsInShutdown) {
+ TRACE("MediaTrack::RemoveDirectListenerImpl ControlMessage");
+ // During shutdown we still want the listener's
+ // NotifyDirectListenerUninstalled to be called, since not doing that
+ // might block shutdown of other modules.
+ RemoveDirectListenerImpl(listener);
+ });
+}
+
+void MediaTrack::RunAfterPendingUpdates(
+ already_AddRefed<nsIRunnable> aRunnable) {
+ MOZ_ASSERT(NS_IsMainThread());
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ QueueControlOrShutdownMessage(
+ [self = RefPtr{this}, this,
+ runnable = nsCOMPtr{aRunnable}](IsInShutdown aInShutdown) mutable {
+ TRACE("MediaTrack::DispatchToMainThreadStableState ControlMessage");
+ if (aInShutdown == IsInShutdown::No) {
+ Graph()->DispatchToMainThreadStableState(runnable.forget());
+ } else {
+ // Don't run mRunnable now as it may call AppendMessage() which would
+ // assume that there are no remaining
+ // controlMessagesToRunDuringShutdown.
+ MOZ_ASSERT(NS_IsMainThread());
+ GraphImpl()->Dispatch(runnable.forget());
+ }
+ });
+}
+
+void MediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
+ AssertOnGraphThread();
+ MOZ_DIAGNOSTIC_ASSERT(
+ aMode == DisabledTrackMode::ENABLED ||
+ mDisabledMode == DisabledTrackMode::ENABLED,
+ "Changing disabled track mode for a track is not allowed");
+ DisabledTrackMode oldMode = CombinedDisabledMode();
+ mDisabledMode = aMode;
+ NotifyIfDisabledModeChangedFrom(oldMode);
+}
+
+void MediaTrack::SetDisabledTrackMode(DisabledTrackMode aMode) {
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aMode]() {
+ TRACE("MediaTrack::SetDisabledTrackModeImpl ControlMessage");
+ SetDisabledTrackModeImpl(aMode);
+ });
+}
+
+void MediaTrack::ApplyTrackDisabling(MediaSegment* aSegment,
+ MediaSegment* aRawSegment) {
+ AssertOnGraphThread();
+ mozilla::ApplyTrackDisabling(mDisabledMode, aSegment, aRawSegment);
+}
+
+void MediaTrack::AddMainThreadListener(
+ MainThreadMediaTrackListener* aListener) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(aListener);
+ MOZ_ASSERT(!mMainThreadListeners.Contains(aListener));
+
+ mMainThreadListeners.AppendElement(aListener);
+
+ // If it is not yet time to send the notification, then exit here.
+ if (!mEndedNotificationSent) {
+ return;
+ }
+
+ class NotifyRunnable final : public Runnable {
+ public:
+ explicit NotifyRunnable(MediaTrack* aTrack)
+ : Runnable("MediaTrack::NotifyRunnable"), mTrack(aTrack) {}
+
+ NS_IMETHOD Run() override {
+ TRACE("MediaTrack::NotifyMainThreadListeners Runnable");
+ MOZ_ASSERT(NS_IsMainThread());
+ mTrack->NotifyMainThreadListeners();
+ return NS_OK;
+ }
+
+ private:
+ ~NotifyRunnable() = default;
+
+ RefPtr<MediaTrack> mTrack;
+ };
+
+ nsCOMPtr<nsIRunnable> runnable = new NotifyRunnable(this);
+ GraphImpl()->Dispatch(runnable.forget());
+}
+
+void MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
+ GraphTime aBlockedTime) {
+ mStartTime += aBlockedTime;
+
+ if (!mSegment) {
+ // No data to be forgotten.
+ return;
+ }
+
+ TrackTime time = aCurrentTime - mStartTime;
+ // Only prune if there is a reasonable chunk (50ms) to forget, so we don't
+ // spend too much time pruning segments.
+ const TrackTime minChunkSize = mSampleRate * 50 / 1000;
+ if (time < mForgottenTime + minChunkSize) {
+ return;
+ }
+
+ mForgottenTime = std::min(GetEnd() - 1, time);
+ mSegment->ForgetUpTo(mForgottenTime);
+}
+
+void MediaTrack::NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode) {
+ DisabledTrackMode mode = CombinedDisabledMode();
+ if (aOldMode == mode) {
+ return;
+ }
+
+ for (const auto& listener : mTrackListeners) {
+ listener->NotifyEnabledStateChanged(
+ Graph(), mode != DisabledTrackMode::SILENCE_BLACK);
+ }
+
+ for (const auto& c : mConsumers) {
+ if (c->GetDestination()) {
+ c->GetDestination()->OnInputDisabledModeChanged(mode);
+ }
+ }
+}
+
+void MediaTrack::QueueMessage(UniquePtr<ControlMessageInterface> aMessage) {
+ MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
+ MOZ_RELEASE_ASSERT(!IsDestroyed());
+ GraphImpl()->AppendMessage(std::move(aMessage));
+}
+
+void MediaTrack::RunMessageAfterProcessing(
+ UniquePtr<ControlMessageInterface> aMessage) {
+ AssertOnGraphThread();
+ GraphImpl()->RunMessageAfterProcessing(std::move(aMessage));
+}
+
+SourceMediaTrack::SourceMediaTrack(MediaSegment::Type aType,
+ TrackRate aSampleRate)
+ : MediaTrack(aSampleRate, aType,
+ aType == MediaSegment::AUDIO
+ ? static_cast<MediaSegment*>(new AudioSegment())
+ : static_cast<MediaSegment*>(new VideoSegment())),
+ mMutex("mozilla::media::SourceMediaTrack") {
+ mUpdateTrack = MakeUnique<TrackData>();
+ mUpdateTrack->mInputRate = aSampleRate;
+ mUpdateTrack->mResamplerChannelCount = 0;
+ mUpdateTrack->mData = UniquePtr<MediaSegment>(mSegment->CreateEmptyClone());
+ mUpdateTrack->mEnded = false;
+ mUpdateTrack->mPullingEnabled = false;
+ mUpdateTrack->mGraphThreadDone = false;
+}
+
+void SourceMediaTrack::DestroyImpl() {
+ GraphImpl()->AssertOnGraphThreadOrNotRunning();
+ for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) {
+ // Disconnect before we come under mMutex's lock since it can call back
+ // through RemoveDirectListenerImpl() and deadlock.
+ mConsumers[i]->Disconnect();
+ }
+
+ // Hold mMutex while mGraph is reset so that other threads holding mMutex
+ // can null-check know that the graph will not destroyed.
+ MutexAutoLock lock(mMutex);
+ mUpdateTrack = nullptr;
+ MediaTrack::DestroyImpl();
+}
+
+void SourceMediaTrack::SetPullingEnabled(bool aEnabled) {
+ class Message : public ControlMessage {
+ public:
+ Message(SourceMediaTrack* aTrack, bool aEnabled)
+ : ControlMessage(nullptr), mTrack(aTrack), mEnabled(aEnabled) {}
+ void Run() override {
+ TRACE("SourceMediaTrack::SetPullingEnabled ControlMessage");
+ MutexAutoLock lock(mTrack->mMutex);
+ if (!mTrack->mUpdateTrack) {
+ // We can't enable pulling for a track that has ended. We ignore
+ // this if we're disabling pulling, since shutdown sequences are
+ // complex. If there's truly an issue we'll have issues enabling anyway.
+ MOZ_ASSERT_IF(mEnabled, mTrack->mEnded);
+ return;
+ }
+ MOZ_ASSERT(mTrack->mType == MediaSegment::AUDIO,
+ "Pulling is not allowed for video");
+ mTrack->mUpdateTrack->mPullingEnabled = mEnabled;
+ }
+ SourceMediaTrack* mTrack;
+ bool mEnabled;
+ };
+ GraphImpl()->AppendMessage(MakeUnique<Message>(this, aEnabled));
+}
+
+bool SourceMediaTrack::PullNewData(GraphTime aDesiredUpToTime) {
+ TRACE_COMMENT("SourceMediaTrack::PullNewData", "%p", this);
+ TrackTime t;
+ TrackTime current;
+ {
+ if (mEnded) {
+ return false;
+ }
+ MutexAutoLock lock(mMutex);
+ if (mUpdateTrack->mEnded) {
+ return false;
+ }
+ if (!mUpdateTrack->mPullingEnabled) {
+ return false;
+ }
+ // Compute how much track time we'll need assuming we don't block
+ // the track at all.
+ t = GraphTimeToTrackTime(aDesiredUpToTime);
+ current = GetEnd() + mUpdateTrack->mData->GetDuration();
+ }
+ if (t <= current) {
+ return false;
+ }
+ LOG(LogLevel::Verbose, ("%p: Calling NotifyPull track=%p t=%f current end=%f",
+ GraphImpl(), this, GraphImpl()->MediaTimeToSeconds(t),
+ GraphImpl()->MediaTimeToSeconds(current)));
+ for (auto& l : mTrackListeners) {
+ l->NotifyPull(Graph(), current, t);
+ }
+ return true;
+}
+
+/**
+ * This moves chunks from aIn to aOut. For audio this is simple. For video
+ * we carry durations over if present, or extend up to aDesiredUpToTime if not.
+ *
+ * We also handle "resetters" from captured media elements. This type of source
+ * pushes future frames into the track, and should it need to remove some, e.g.,
+ * because of a seek or pause, it tells us by letting time go backwards. Without
+ * this, tracks would be live for too long after a seek or pause.
+ */
+static void MoveToSegment(SourceMediaTrack* aTrack, MediaSegment* aIn,
+ MediaSegment* aOut, TrackTime aCurrentTime,
+ TrackTime aDesiredUpToTime)
+ MOZ_REQUIRES(aTrack->GetMutex()) {
+ MOZ_ASSERT(aIn->GetType() == aOut->GetType());
+ MOZ_ASSERT(aOut->GetDuration() >= aCurrentTime);
+ MOZ_ASSERT(aDesiredUpToTime >= aCurrentTime);
+ if (aIn->GetType() == MediaSegment::AUDIO) {
+ AudioSegment* in = static_cast<AudioSegment*>(aIn);
+ AudioSegment* out = static_cast<AudioSegment*>(aOut);
+ TrackTime desiredDurationToMove = aDesiredUpToTime - aCurrentTime;
+ TrackTime end = std::min(in->GetDuration(), desiredDurationToMove);
+
+ out->AppendSlice(*in, 0, end);
+ in->RemoveLeading(end);
+
+ aTrack->GetMutex().AssertCurrentThreadOwns();
+ out->ApplyVolume(aTrack->GetVolumeLocked());
+ } else {
+ VideoSegment* in = static_cast<VideoSegment*>(aIn);
+ VideoSegment* out = static_cast<VideoSegment*>(aOut);
+ for (VideoSegment::ConstChunkIterator c(*in); !c.IsEnded(); c.Next()) {
+ MOZ_ASSERT(!c->mTimeStamp.IsNull());
+ VideoChunk* last = out->GetLastChunk();
+ if (!last || last->mTimeStamp.IsNull()) {
+ // This is the first frame, or the last frame pushed to `out` has been
+ // all consumed. Just append and we deal with its duration later.
+ out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
+ c->mFrame.GetIntrinsicSize(),
+ c->mFrame.GetPrincipalHandle(),
+ c->mFrame.GetForceBlack(), c->mTimeStamp);
+ if (c->GetDuration() > 0) {
+ out->ExtendLastFrameBy(c->GetDuration());
+ }
+ continue;
+ }
+
+ // We now know when this frame starts, aka when the last frame ends.
+
+ if (c->mTimeStamp < last->mTimeStamp) {
+ // Time is going backwards. This is a resetting frame from
+ // DecodedStream. Clear everything up to currentTime.
+ out->Clear();
+ out->AppendNullData(aCurrentTime);
+ }
+
+ // Append the current frame (will have duration 0).
+ out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
+ c->mFrame.GetIntrinsicSize(),
+ c->mFrame.GetPrincipalHandle(),
+ c->mFrame.GetForceBlack(), c->mTimeStamp);
+ if (c->GetDuration() > 0) {
+ out->ExtendLastFrameBy(c->GetDuration());
+ }
+ }
+ if (out->GetDuration() < aDesiredUpToTime) {
+ out->ExtendLastFrameBy(aDesiredUpToTime - out->GetDuration());
+ }
+ in->Clear();
+ MOZ_ASSERT(aIn->GetDuration() == 0, "aIn must be consumed");
+ }
+}
+
+void SourceMediaTrack::ExtractPendingInput(GraphTime aCurrentTime,
+ GraphTime aDesiredUpToTime) {
+ MutexAutoLock lock(mMutex);
+
+ if (!mUpdateTrack) {
+ MOZ_ASSERT(mEnded);
+ return;
+ }
+
+ TrackTime trackCurrentTime = GraphTimeToTrackTime(aCurrentTime);
+
+ ApplyTrackDisabling(mUpdateTrack->mData.get());
+
+ if (!mUpdateTrack->mData->IsEmpty()) {
+ for (const auto& l : mTrackListeners) {
+ l->NotifyQueuedChanges(GraphImpl(), GetEnd(), *mUpdateTrack->mData);
+ }
+ }
+ TrackTime trackDesiredUpToTime = GraphTimeToTrackTime(aDesiredUpToTime);
+ TrackTime endTime = trackDesiredUpToTime;
+ if (mUpdateTrack->mEnded) {
+ endTime = std::min(trackDesiredUpToTime,
+ GetEnd() + mUpdateTrack->mData->GetDuration());
+ }
+ LOG(LogLevel::Verbose,
+ ("%p: SourceMediaTrack %p advancing end from %" PRId64 " to %" PRId64,
+ GraphImpl(), this, int64_t(trackCurrentTime), int64_t(endTime)));
+ MoveToSegment(this, mUpdateTrack->mData.get(), mSegment.get(),
+ trackCurrentTime, endTime);
+ if (mUpdateTrack->mEnded && GetEnd() < trackDesiredUpToTime) {
+ mEnded = true;
+ mUpdateTrack = nullptr;
+ }
+}
+
+void SourceMediaTrack::ResampleAudioToGraphSampleRate(MediaSegment* aSegment) {
+ mMutex.AssertCurrentThreadOwns();
+ if (aSegment->GetType() != MediaSegment::AUDIO ||
+ mUpdateTrack->mInputRate == GraphImpl()->GraphRate()) {
+ return;
+ }
+ AudioSegment* segment = static_cast<AudioSegment*>(aSegment);
+ segment->ResampleChunks(mUpdateTrack->mResampler,
+ &mUpdateTrack->mResamplerChannelCount,
+ mUpdateTrack->mInputRate, GraphImpl()->GraphRate());
+}
+
+void SourceMediaTrack::AdvanceTimeVaryingValuesToCurrentTime(
+ GraphTime aCurrentTime, GraphTime aBlockedTime) {
+ MutexAutoLock lock(mMutex);
+ MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(aCurrentTime, aBlockedTime);
+}
+
+void SourceMediaTrack::SetAppendDataSourceRate(TrackRate aRate) {
+ MutexAutoLock lock(mMutex);
+ if (!mUpdateTrack) {
+ return;
+ }
+ MOZ_DIAGNOSTIC_ASSERT(mSegment->GetType() == MediaSegment::AUDIO);
+ // Set the new input rate and reset the resampler.
+ mUpdateTrack->mInputRate = aRate;
+ mUpdateTrack->mResampler.own(nullptr);
+ mUpdateTrack->mResamplerChannelCount = 0;
+}
+
+TrackTime SourceMediaTrack::AppendData(MediaSegment* aSegment,
+ MediaSegment* aRawSegment) {
+ MutexAutoLock lock(mMutex);
+ MOZ_DIAGNOSTIC_ASSERT(aSegment->GetType() == mType);
+ TrackTime appended = 0;
+ if (!mUpdateTrack || mUpdateTrack->mEnded || mUpdateTrack->mGraphThreadDone) {
+ aSegment->Clear();
+ return appended;
+ }
+
+ // Data goes into mData, and on the next iteration of the MTG moves
+ // into the track's segment after NotifyQueuedTrackChanges(). This adds
+ // 0-10ms of delay before data gets to direct listeners.
+ // Indirect listeners (via subsequent TrackUnion nodes) are synced to
+ // playout time, and so can be delayed by buffering.
+
+ // Apply track disabling before notifying any consumers directly
+ // or inserting into the graph
+ mozilla::ApplyTrackDisabling(mDirectDisabledMode, aSegment, aRawSegment);
+
+ ResampleAudioToGraphSampleRate(aSegment);
+
+ // Must notify first, since AppendFrom() will empty out aSegment
+ NotifyDirectConsumers(aRawSegment ? aRawSegment : aSegment);
+ appended = aSegment->GetDuration();
+ mUpdateTrack->mData->AppendFrom(aSegment); // note: aSegment is now dead
+ {
+ auto graph = GraphImpl();
+ MonitorAutoLock lock(graph->GetMonitor());
+ if (graph->CurrentDriver()) { // graph has not completed forced shutdown
+ graph->EnsureNextIteration();
+ }
+ }
+
+ return appended;
+}
+
+TrackTime SourceMediaTrack::ClearFutureData() {
+ MutexAutoLock lock(mMutex);
+ auto graph = GraphImpl();
+ if (!mUpdateTrack || !graph) {
+ return 0;
+ }
+
+ TrackTime duration = mUpdateTrack->mData->GetDuration();
+ mUpdateTrack->mData->Clear();
+ return duration;
+}
+
+void SourceMediaTrack::NotifyDirectConsumers(MediaSegment* aSegment) {
+ mMutex.AssertCurrentThreadOwns();
+
+ for (const auto& l : mDirectTrackListeners) {
+ TrackTime offset = 0; // FIX! need a separate TrackTime.... or the end of
+ // the internal buffer
+ l->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset,
+ *aSegment);
+ }
+}
+
+void SourceMediaTrack::AddDirectListenerImpl(
+ already_AddRefed<DirectMediaTrackListener> aListener) {
+ AssertOnGraphThread();
+ MutexAutoLock lock(mMutex);
+
+ RefPtr<DirectMediaTrackListener> listener = aListener;
+ LOG(LogLevel::Debug,
+ ("%p: Adding direct track listener %p to source track %p", GraphImpl(),
+ listener.get(), this));
+
+ MOZ_ASSERT(mType == MediaSegment::VIDEO);
+ for (const auto& l : mDirectTrackListeners) {
+ if (l == listener) {
+ listener->NotifyDirectListenerInstalled(
+ DirectMediaTrackListener::InstallationResult::ALREADY_EXISTS);
+ return;
+ }
+ }
+
+ mDirectTrackListeners.AppendElement(listener);
+
+ LOG(LogLevel::Debug,
+ ("%p: Added direct track listener %p", GraphImpl(), listener.get()));
+ listener->NotifyDirectListenerInstalled(
+ DirectMediaTrackListener::InstallationResult::SUCCESS);
+
+ if (mDisabledMode != DisabledTrackMode::ENABLED) {
+ listener->IncreaseDisabled(mDisabledMode);
+ }
+
+ if (mEnded) {
+ return;
+ }
+
+ // Pass buffered data to the listener
+ VideoSegment bufferedData;
+ size_t videoFrames = 0;
+ VideoSegment& segment = *GetData<VideoSegment>();
+ for (VideoSegment::ConstChunkIterator iter(segment); !iter.IsEnded();
+ iter.Next()) {
+ if (iter->mTimeStamp.IsNull()) {
+ // No timestamp means this is only for the graph's internal book-keeping,
+ // denoting a late start of the track.
+ continue;
+ }
+ ++videoFrames;
+ bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
+ iter->mFrame.GetIntrinsicSize(),
+ iter->mFrame.GetPrincipalHandle(),
+ iter->mFrame.GetForceBlack(), iter->mTimeStamp);
+ }
+
+ VideoSegment& video = static_cast<VideoSegment&>(*mUpdateTrack->mData);
+ for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded();
+ iter.Next()) {
+ ++videoFrames;
+ MOZ_ASSERT(!iter->mTimeStamp.IsNull());
+ bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
+ iter->mFrame.GetIntrinsicSize(),
+ iter->mFrame.GetPrincipalHandle(),
+ iter->mFrame.GetForceBlack(), iter->mTimeStamp);
+ }
+
+ LOG(LogLevel::Info,
+ ("%p: Notifying direct listener %p of %zu video frames and duration "
+ "%" PRId64,
+ GraphImpl(), listener.get(), videoFrames, bufferedData.GetDuration()));
+ listener->NotifyRealtimeTrackData(Graph(), 0, bufferedData);
+}
+
+void SourceMediaTrack::RemoveDirectListenerImpl(
+ DirectMediaTrackListener* aListener) {
+ mGraph->AssertOnGraphThreadOrNotRunning();
+ MutexAutoLock lock(mMutex);
+ for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) {
+ const RefPtr<DirectMediaTrackListener>& l = mDirectTrackListeners[i];
+ if (l == aListener) {
+ if (mDisabledMode != DisabledTrackMode::ENABLED) {
+ aListener->DecreaseDisabled(mDisabledMode);
+ }
+ aListener->NotifyDirectListenerUninstalled();
+ mDirectTrackListeners.RemoveElementAt(i);
+ }
+ }
+}
+
+void SourceMediaTrack::End() {
+ MutexAutoLock lock(mMutex);
+ if (!mUpdateTrack) {
+ // Already ended
+ return;
+ }
+ mUpdateTrack->mEnded = true;
+ if (auto graph = GraphImpl()) {
+ MonitorAutoLock lock(graph->GetMonitor());
+ if (graph->CurrentDriver()) { // graph has not completed forced shutdown
+ graph->EnsureNextIteration();
+ }
+ }
+}
+
+void SourceMediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) {
+ AssertOnGraphThread();
+ {
+ MutexAutoLock lock(mMutex);
+ const DisabledTrackMode oldMode = mDirectDisabledMode;
+ const bool oldEnabled = oldMode == DisabledTrackMode::ENABLED;
+ const bool enabled = aMode == DisabledTrackMode::ENABLED;
+ mDirectDisabledMode = aMode;
+ for (const auto& l : mDirectTrackListeners) {
+ if (!oldEnabled && enabled) {
+ LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
+ "direct listener enabled",
+ GraphImpl(), this));
+ l->DecreaseDisabled(oldMode);
+ } else if (oldEnabled && !enabled) {
+ LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting "
+ "direct listener disabled",
+ GraphImpl(), this));
+ l->IncreaseDisabled(aMode);
+ }
+ }
+ }
+ MediaTrack::SetDisabledTrackModeImpl(aMode);
+}
+
+uint32_t SourceMediaTrack::NumberOfChannels() const {
+ AudioSegment* audio = GetData<AudioSegment>();
+ MOZ_DIAGNOSTIC_ASSERT(audio);
+ if (!audio) {
+ return 0;
+ }
+ return audio->MaxChannelCount();
+}
+
+void SourceMediaTrack::RemoveAllDirectListenersImpl() {
+ GraphImpl()->AssertOnGraphThreadOrNotRunning();
+ MutexAutoLock lock(mMutex);
+
+ for (auto& l : mDirectTrackListeners.Clone()) {
+ l->NotifyDirectListenerUninstalled();
+ }
+ mDirectTrackListeners.Clear();
+}
+
+void SourceMediaTrack::SetVolume(float aVolume) {
+ MutexAutoLock lock(mMutex);
+ mVolume = aVolume;
+}
+
+float SourceMediaTrack::GetVolumeLocked() {
+ mMutex.AssertCurrentThreadOwns();
+ return mVolume;
+}
+
+SourceMediaTrack::~SourceMediaTrack() = default;
+
+void MediaInputPort::Init() {
+ mGraph->AssertOnGraphThreadOrNotRunning();
+ LOG(LogLevel::Debug, ("%p: Adding MediaInputPort %p (from %p to %p)", mGraph,
+ this, mSource, mDest));
+ // Only connect the port if it wasn't disconnected on allocation.
+ if (mSource) {
+ mSource->AddConsumer(this);
+ mDest->AddInput(this);
+ }
+ // mPortCount decremented via MediaInputPort::Destroy's message
+ ++mGraph->mPortCount;
+}
+
+void MediaInputPort::Disconnect() {
+ mGraph->AssertOnGraphThreadOrNotRunning();
+ NS_ASSERTION(!mSource == !mDest,
+ "mSource and mDest must either both be null or both non-null");
+
+ if (!mSource) {
+ return;
+ }
+
+ mSource->RemoveConsumer(this);
+ mDest->RemoveInput(this);
+ mSource = nullptr;
+ mDest = nullptr;
+
+ mGraph->SetTrackOrderDirty();
+}
+
+MediaTrack* MediaInputPort::GetSource() const {
+ mGraph->AssertOnGraphThreadOrNotRunning();
+ return mSource;
+}
+
+ProcessedMediaTrack* MediaInputPort::GetDestination() const {
+ mGraph->AssertOnGraphThreadOrNotRunning();
+ return mDest;
+}
+
+MediaInputPort::InputInterval MediaInputPort::GetNextInputInterval(
+ MediaInputPort const* aPort, GraphTime aTime) {
+ InputInterval result = {GRAPH_TIME_MAX, GRAPH_TIME_MAX, false};
+ if (!aPort) {
+ result.mStart = aTime;
+ result.mInputIsBlocked = true;
+ return result;
+ }
+ aPort->mGraph->AssertOnGraphThreadOrNotRunning();
+ if (aTime >= aPort->mDest->mStartBlocking) {
+ return result;
+ }
+ result.mStart = aTime;
+ result.mEnd = aPort->mDest->mStartBlocking;
+ result.mInputIsBlocked = aTime >= aPort->mSource->mStartBlocking;
+ if (!result.mInputIsBlocked) {
+ result.mEnd = std::min(result.mEnd, aPort->mSource->mStartBlocking);
+ }
+ return result;
+}
+
+void MediaInputPort::Suspended() {
+ mGraph->AssertOnGraphThreadOrNotRunning();
+ mDest->InputSuspended(this);
+}
+
+void MediaInputPort::Resumed() {
+ mGraph->AssertOnGraphThreadOrNotRunning();
+ mDest->InputResumed(this);
+}
+
+void MediaInputPort::Destroy() {
+ class Message : public ControlMessage {
+ public:
+ explicit Message(MediaInputPort* aPort)
+ : ControlMessage(nullptr), mPort(aPort) {}
+ void Run() override {
+ TRACE("MediaInputPort::Destroy ControlMessage");
+ mPort->Disconnect();
+ --mPort->GraphImpl()->mPortCount;
+ mPort->SetGraphImpl(nullptr);
+ NS_RELEASE(mPort);
+ }
+ void RunDuringShutdown() override { Run(); }
+ MediaInputPort* mPort;
+ };
+ // Keep a reference to the graph, since Message might RunDuringShutdown()
+ // synchronously and make GraphImpl() invalid.
+ RefPtr<MediaTrackGraphImpl> graph = mGraph;
+ graph->AppendMessage(MakeUnique<Message>(this));
+ --graph->mMainThreadPortCount;
+}
+
+MediaTrackGraphImpl* MediaInputPort::GraphImpl() const {
+ mGraph->AssertOnGraphThreadOrNotRunning();
+ return mGraph;
+}
+
+MediaTrackGraph* MediaInputPort::Graph() const {
+ mGraph->AssertOnGraphThreadOrNotRunning();
+ return mGraph;
+}
+
+void MediaInputPort::SetGraphImpl(MediaTrackGraphImpl* aGraph) {
+ MOZ_ASSERT(!mGraph || !aGraph, "Should only be set once");
+ DebugOnly<MediaTrackGraphImpl*> graph = mGraph ? mGraph : aGraph;
+ MOZ_ASSERT(graph->OnGraphThreadOrNotRunning());
+ mGraph = aGraph;
+}
+
+already_AddRefed<MediaInputPort> ProcessedMediaTrack::AllocateInputPort(
+ MediaTrack* aTrack, uint16_t aInputNumber, uint16_t aOutputNumber) {
+ // This method creates two references to the MediaInputPort: one for
+ // the main thread, and one for the MediaTrackGraph.
+ class Message : public ControlMessage {
+ public:
+ explicit Message(MediaInputPort* aPort)
+ : ControlMessage(aPort->mDest), mPort(aPort) {}
+ void Run() override {
+ TRACE("ProcessedMediaTrack::AllocateInputPort ControlMessage");
+ mPort->Init();
+ // The graph holds its reference implicitly
+ mPort->GraphImpl()->SetTrackOrderDirty();
+ Unused << mPort.forget();
+ }
+ void RunDuringShutdown() override { Run(); }
+ RefPtr<MediaInputPort> mPort;
+ };
+
+ MOZ_DIAGNOSTIC_ASSERT(aTrack->mType == mType);
+ RefPtr<MediaInputPort> port;
+ if (aTrack->IsDestroyed()) {
+ // Create a port that's disconnected, which is what it'd be after its source
+ // track is Destroy()ed normally. Disconnect() is idempotent so destroying
+ // this later is fine.
+ port = new MediaInputPort(GraphImpl(), nullptr, nullptr, aInputNumber,
+ aOutputNumber);
+ } else {
+ MOZ_ASSERT(aTrack->GraphImpl() == GraphImpl());
+ port = new MediaInputPort(GraphImpl(), aTrack, this, aInputNumber,
+ aOutputNumber);
+ }
+ ++GraphImpl()->mMainThreadPortCount;
+ GraphImpl()->AppendMessage(MakeUnique<Message>(port));
+ return port.forget();
+}
+
+void ProcessedMediaTrack::QueueSetAutoend(bool aAutoend) {
+ class Message : public ControlMessage {
+ public:
+ Message(ProcessedMediaTrack* aTrack, bool aAutoend)
+ : ControlMessage(aTrack), mAutoend(aAutoend) {}
+ void Run() override {
+ TRACE("ProcessedMediaTrack::SetAutoendImpl ControlMessage");
+ static_cast<ProcessedMediaTrack*>(mTrack)->SetAutoendImpl(mAutoend);
+ }
+ bool mAutoend;
+ };
+ if (mMainThreadDestroyed) {
+ return;
+ }
+ GraphImpl()->AppendMessage(MakeUnique<Message>(this, aAutoend));
+}
+
+void ProcessedMediaTrack::DestroyImpl() {
+ for (int32_t i = mInputs.Length() - 1; i >= 0; --i) {
+ mInputs[i]->Disconnect();
+ }
+
+ for (int32_t i = mSuspendedInputs.Length() - 1; i >= 0; --i) {
+ mSuspendedInputs[i]->Disconnect();
+ }
+
+ MediaTrack::DestroyImpl();
+ // The track order is only important if there are connections, in which
+ // case MediaInputPort::Disconnect() called SetTrackOrderDirty().
+ // MediaTrackGraphImpl::RemoveTrackGraphThread() will also call
+ // SetTrackOrderDirty(), for other reasons.
+}
+
+MediaTrackGraphImpl::MediaTrackGraphImpl(uint64_t aWindowID,
+ TrackRate aSampleRate,
+ AudioDeviceID aPrimaryOutputDeviceID,
+ nsISerialEventTarget* aMainThread)
+ : MediaTrackGraph(aSampleRate, aPrimaryOutputDeviceID),
+ mWindowID(aWindowID),
+ mFirstCycleBreaker(0)
+ // An offline graph is not initially processing.
+ ,
+ mPortCount(0),
+ mMonitor("MediaTrackGraphImpl"),
+ mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED),
+ mPostedRunInStableStateEvent(false),
+ mGraphDriverRunning(false),
+ mPostedRunInStableState(false),
+ mTrackOrderDirty(false),
+ mMainThread(aMainThread),
+ mGlobalVolume(CubebUtils::GetVolumeScale())
+#ifdef DEBUG
+ ,
+ mCanRunMessagesSynchronously(false)
+#endif
+ ,
+ mMainThreadGraphTime(0, "MediaTrackGraphImpl::mMainThreadGraphTime"),
+ mAudioOutputLatency(0.0),
+ mMaxOutputChannelCount(std::min(8u, CubebUtils::MaxNumberOfChannels())) {
+}
+
+void MediaTrackGraphImpl::Init(GraphDriverType aDriverRequested,
+ GraphRunType aRunTypeRequested,
+ uint32_t aChannelCount) {
+ mSelfRef = this;
+ mEndTime = aDriverRequested == OFFLINE_THREAD_DRIVER ? 0 : GRAPH_TIME_MAX;
+ mRealtime = aDriverRequested != OFFLINE_THREAD_DRIVER;
+ // The primary output device always exists because an AudioCallbackDriver
+ // may exist, and want to be fed data, even when no tracks have audio
+ // outputs.
+ mOutputDeviceRefCnts.EmplaceBack(
+ DeviceReceiverAndCount{mPrimaryOutputDeviceID, nullptr, 0});
+ mOutputDevices.EmplaceBack(OutputDeviceEntry{mPrimaryOutputDeviceID});
+
+ bool failedToGetShutdownBlocker = false;
+ if (!IsNonRealtime()) {
+ failedToGetShutdownBlocker = !AddShutdownBlocker();
+ }
+
+ mGraphRunner = aRunTypeRequested == SINGLE_THREAD
+ ? GraphRunner::Create(this)
+ : already_AddRefed<GraphRunner>(nullptr);
+
+ if ((aRunTypeRequested == SINGLE_THREAD && !mGraphRunner) ||
+ failedToGetShutdownBlocker) {
+ MonitorAutoLock lock(mMonitor);
+ // At least one of the following happened
+ // - Failed to create thread.
+ // - Failed to install a shutdown blocker when one is needed.
+ // Because we have a fail state, jump to last phase of the lifecycle.
+ mLifecycleState = LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION;
+ RemoveShutdownBlocker(); // No-op if blocker wasn't added.
+#ifdef DEBUG
+ mCanRunMessagesSynchronously = true;
+#endif
+ return;
+ }
+ if (mRealtime) {
+ if (aDriverRequested == AUDIO_THREAD_DRIVER) {
+ // Always start with zero input channels, and no particular preferences
+ // for the input channel.
+ mDriver = new AudioCallbackDriver(
+ this, nullptr, mSampleRate, aChannelCount, 0, PrimaryOutputDeviceID(),
+ nullptr, AudioInputType::Unknown);
+ } else {
+ mDriver = new SystemClockDriver(this, nullptr, mSampleRate);
+ }
+ nsCString streamName = GetDocumentTitle(mWindowID);
+ LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get()));
+ mDriver->SetStreamName(streamName);
+ } else {
+ mDriver =
+ new OfflineClockDriver(this, mSampleRate, MEDIA_GRAPH_TARGET_PERIOD_MS);
+ }
+
+ mLastMainThreadUpdate = TimeStamp::Now();
+
+ RegisterWeakAsyncMemoryReporter(this);
+}
+
+#ifdef DEBUG
+bool MediaTrackGraphImpl::InDriverIteration(const GraphDriver* aDriver) const {
+ return aDriver->OnThread() ||
+ (mGraphRunner && mGraphRunner->InDriverIteration(aDriver));
+}
+#endif
+
+void MediaTrackGraphImpl::Destroy() {
+ // First unregister from memory reporting.
+ UnregisterWeakMemoryReporter(this);
+
+ // Clear the self reference which will destroy this instance if all
+ // associated GraphDrivers are destroyed.
+ mSelfRef = nullptr;
+}
+
+// Internal method has a Window ID parameter so that TestAudioTrackGraph
+// GTests can create a graph without a window.
+/* static */
+MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstanceIfExists(
+ uint64_t aWindowID, TrackRate aSampleRate,
+ AudioDeviceID aPrimaryOutputDeviceID) {
+ MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
+ MOZ_ASSERT(aSampleRate > 0);
+
+ GraphHashSet::Ptr p =
+ Graphs()->lookup({aWindowID, aSampleRate, aPrimaryOutputDeviceID});
+ return p ? *p : nullptr;
+}
+
+// Public method has an nsPIDOMWindowInner* parameter to ensure that the
+// window is a real inner Window, not a WindowProxy.
+/* static */
+MediaTrackGraph* MediaTrackGraph::GetInstanceIfExists(
+ nsPIDOMWindowInner* aWindow, TrackRate aSampleRate,
+ AudioDeviceID aPrimaryOutputDeviceID) {
+ TrackRate sampleRate =
+ aSampleRate ? aSampleRate
+ : CubebUtils::PreferredSampleRate(
+ aWindow->AsGlobal()->ShouldResistFingerprinting(
+ RFPTarget::AudioSampleRate));
+ return MediaTrackGraphImpl::GetInstanceIfExists(
+ aWindow->WindowID(), sampleRate, aPrimaryOutputDeviceID);
+}
+
+/* static */
+MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstance(
+ GraphDriverType aGraphDriverRequested, uint64_t aWindowID,
+ TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID,
+ nsISerialEventTarget* aMainThread) {
+ MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
+ MOZ_ASSERT(aSampleRate > 0);
+ MOZ_ASSERT(aGraphDriverRequested != OFFLINE_THREAD_DRIVER,
+ "Use CreateNonRealtimeInstance() for offline graphs");
+
+ GraphHashSet* graphs = Graphs();
+ GraphHashSet::AddPtr addPtr =
+ graphs->lookupForAdd({aWindowID, aSampleRate, aPrimaryOutputDeviceID});
+ if (addPtr) { // graph already exists
+ return *addPtr;
+ }
+
+ GraphRunType runType = DIRECT_DRIVER;
+ if (Preferences::GetBool("media.audiograph.single_thread.enabled", true)) {
+ runType = SINGLE_THREAD;
+ }
+
+ // In a real time graph, the number of output channels is determined by
+ // the underlying number of channel of the default audio output device, and
+ // capped to 8.
+ uint32_t channelCount =
+ std::min<uint32_t>(8, CubebUtils::MaxNumberOfChannels());
+ MediaTrackGraphImpl* graph = new MediaTrackGraphImpl(
+ aWindowID, aSampleRate, aPrimaryOutputDeviceID, aMainThread);
+ graph->Init(aGraphDriverRequested, runType, channelCount);
+ MOZ_ALWAYS_TRUE(graphs->add(addPtr, graph));
+
+ LOG(LogLevel::Debug, ("Starting up MediaTrackGraph %p for window 0x%" PRIx64,
+ graph, aWindowID));
+
+ return graph;
+}
+
+/* static */
+MediaTrackGraph* MediaTrackGraph::GetInstance(
+ GraphDriverType aGraphDriverRequested, nsPIDOMWindowInner* aWindow,
+ TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID) {
+ TrackRate sampleRate =
+ aSampleRate ? aSampleRate
+ : CubebUtils::PreferredSampleRate(
+ aWindow->AsGlobal()->ShouldResistFingerprinting(
+ RFPTarget::AudioSampleRate));
+ return MediaTrackGraphImpl::GetInstance(
+ aGraphDriverRequested, aWindow->WindowID(), sampleRate,
+ aPrimaryOutputDeviceID, GetMainThreadSerialEventTarget());
+}
+
+MediaTrackGraph* MediaTrackGraphImpl::CreateNonRealtimeInstance(
+ TrackRate aSampleRate) {
+ MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
+
+ nsISerialEventTarget* mainThread = GetMainThreadSerialEventTarget();
+ // Offline graphs have 0 output channel count: they write the output to a
+ // buffer, not an audio output track.
+ MediaTrackGraphImpl* graph = new MediaTrackGraphImpl(
+ 0, aSampleRate, DEFAULT_OUTPUT_DEVICE, mainThread);
+ graph->Init(OFFLINE_THREAD_DRIVER, DIRECT_DRIVER, 0);
+
+ LOG(LogLevel::Debug, ("Starting up Offline MediaTrackGraph %p", graph));
+
+ return graph;
+}
+
+MediaTrackGraph* MediaTrackGraph::CreateNonRealtimeInstance(
+ TrackRate aSampleRate) {
+ return MediaTrackGraphImpl::CreateNonRealtimeInstance(aSampleRate);
+}
+
+void MediaTrackGraph::ForceShutDown() {
+ MOZ_ASSERT(NS_IsMainThread(), "Main thread only");
+
+ MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
+
+ graph->ForceShutDown();
+}
+
+NS_IMPL_ISUPPORTS(MediaTrackGraphImpl, nsIMemoryReporter, nsIObserver,
+ nsIThreadObserver, nsITimerCallback, nsINamed)
+
+NS_IMETHODIMP
+MediaTrackGraphImpl::CollectReports(nsIHandleReportCallback* aHandleReport,
+ nsISupports* aData, bool aAnonymize) {
+ MOZ_ASSERT(NS_IsMainThread());
+ if (mMainThreadTrackCount == 0) {
+ // No tracks to report.
+ FinishCollectReports(aHandleReport, aData, nsTArray<AudioNodeSizes>());
+ return NS_OK;
+ }
+
+ class Message final : public ControlMessage {
+ public:
+ Message(MediaTrackGraphImpl* aGraph, nsIHandleReportCallback* aHandleReport,
+ nsISupports* aHandlerData)
+ : ControlMessage(nullptr),
+ mGraph(aGraph),
+ mHandleReport(aHandleReport),
+ mHandlerData(aHandlerData) {}
+ void Run() override {
+ TRACE("MTG::CollectSizesForMemoryReport ControlMessage");
+ mGraph->CollectSizesForMemoryReport(mHandleReport.forget(),
+ mHandlerData.forget());
+ }
+ void RunDuringShutdown() override {
+ // Run this message during shutdown too, so that endReports is called.
+ Run();
+ }
+ MediaTrackGraphImpl* mGraph;
+ // nsMemoryReporterManager keeps the callback and data alive only if it
+ // does not time out.
+ nsCOMPtr<nsIHandleReportCallback> mHandleReport;
+ nsCOMPtr<nsISupports> mHandlerData;
+ };
+
+ AppendMessage(MakeUnique<Message>(this, aHandleReport, aData));
+
+ return NS_OK;
+}
+
+void MediaTrackGraphImpl::CollectSizesForMemoryReport(
+ already_AddRefed<nsIHandleReportCallback> aHandleReport,
+ already_AddRefed<nsISupports> aHandlerData) {
+ class FinishCollectRunnable final : public Runnable {
+ public:
+ explicit FinishCollectRunnable(
+ already_AddRefed<nsIHandleReportCallback> aHandleReport,
+ already_AddRefed<nsISupports> aHandlerData)
+ : mozilla::Runnable("FinishCollectRunnable"),
+ mHandleReport(aHandleReport),
+ mHandlerData(aHandlerData) {}
+
+ NS_IMETHOD Run() override {
+ TRACE("MTG::FinishCollectReports ControlMessage");
+ MediaTrackGraphImpl::FinishCollectReports(mHandleReport, mHandlerData,
+ std::move(mAudioTrackSizes));
+ return NS_OK;
+ }
+
+ nsTArray<AudioNodeSizes> mAudioTrackSizes;
+
+ private:
+ ~FinishCollectRunnable() = default;
+
+ // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its
+ // constructor modifies the ref-count, which cannot be done off main
+ // thread.
+ RefPtr<nsIHandleReportCallback> mHandleReport;
+ RefPtr<nsISupports> mHandlerData;
+ };
+
+ RefPtr<FinishCollectRunnable> runnable = new FinishCollectRunnable(
+ std::move(aHandleReport), std::move(aHandlerData));
+
+ auto audioTrackSizes = &runnable->mAudioTrackSizes;
+
+ for (MediaTrack* t : AllTracks()) {
+ AudioNodeTrack* track = t->AsAudioNodeTrack();
+ if (track) {
+ AudioNodeSizes* usage = audioTrackSizes->AppendElement();
+ track->SizeOfAudioNodesIncludingThis(MallocSizeOf, *usage);
+ }
+ }
+
+ mMainThread->Dispatch(runnable.forget());
+}
+
+void MediaTrackGraphImpl::FinishCollectReports(
+ nsIHandleReportCallback* aHandleReport, nsISupports* aData,
+ const nsTArray<AudioNodeSizes>& aAudioTrackSizes) {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ nsCOMPtr<nsIMemoryReporterManager> manager =
+ do_GetService("@mozilla.org/memory-reporter-manager;1");
+
+ if (!manager) return;
+
+#define REPORT(_path, _amount, _desc) \
+ aHandleReport->Callback(""_ns, _path, KIND_HEAP, UNITS_BYTES, _amount, \
+ nsLiteralCString(_desc), aData);
+
+ for (size_t i = 0; i < aAudioTrackSizes.Length(); i++) {
+ const AudioNodeSizes& usage = aAudioTrackSizes[i];
+ const char* const nodeType =
+ usage.mNodeType ? usage.mNodeType : "<unknown>";
+
+ nsPrintfCString enginePath("explicit/webaudio/audio-node/%s/engine-objects",
+ nodeType);
+ REPORT(enginePath, usage.mEngine,
+ "Memory used by AudioNode engine objects (Web Audio).");
+
+ nsPrintfCString trackPath("explicit/webaudio/audio-node/%s/track-objects",
+ nodeType);
+ REPORT(trackPath, usage.mTrack,
+ "Memory used by AudioNode track objects (Web Audio).");
+ }
+
+ size_t hrtfLoaders = WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf);
+ if (hrtfLoaders) {
+ REPORT(nsLiteralCString(
+ "explicit/webaudio/audio-node/PannerNode/hrtf-databases"),
+ hrtfLoaders, "Memory used by PannerNode databases (Web Audio).");
+ }
+
+#undef REPORT
+
+ manager->EndReport();
+}
+
+SourceMediaTrack* MediaTrackGraph::CreateSourceTrack(MediaSegment::Type aType) {
+ SourceMediaTrack* track = new SourceMediaTrack(aType, GraphRate());
+ AddTrack(track);
+ return track;
+}
+
+ProcessedMediaTrack* MediaTrackGraph::CreateForwardedInputTrack(
+ MediaSegment::Type aType) {
+ ForwardedInputTrack* track = new ForwardedInputTrack(GraphRate(), aType);
+ AddTrack(track);
+ return track;
+}
+
+AudioCaptureTrack* MediaTrackGraph::CreateAudioCaptureTrack() {
+ AudioCaptureTrack* track = new AudioCaptureTrack(GraphRate());
+ AddTrack(track);
+ return track;
+}
+
+CrossGraphTransmitter* MediaTrackGraph::CreateCrossGraphTransmitter(
+ CrossGraphReceiver* aReceiver) {
+ CrossGraphTransmitter* track =
+ new CrossGraphTransmitter(GraphRate(), aReceiver);
+ AddTrack(track);
+ return track;
+}
+
+CrossGraphReceiver* MediaTrackGraph::CreateCrossGraphReceiver(
+ TrackRate aTransmitterRate) {
+ CrossGraphReceiver* track =
+ new CrossGraphReceiver(GraphRate(), aTransmitterRate);
+ AddTrack(track);
+ return track;
+}
+
+void MediaTrackGraph::AddTrack(MediaTrack* aTrack) {
+ MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
+ MOZ_ASSERT(NS_IsMainThread());
+#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
+ if (graph->mRealtime) {
+ GraphHashSet::Ptr p = Graphs()->lookup(*graph);
+ MOZ_DIAGNOSTIC_ASSERT(p, "Graph must not be shutting down");
+ }
+#endif
+ if (graph->mMainThreadTrackCount == 0) {
+ nsCOMPtr<nsIObserverService> observerService =
+ mozilla::services::GetObserverService();
+ if (observerService) {
+ observerService->AddObserver(graph, "document-title-changed", false);
+ }
+ }
+
+ NS_ADDREF(aTrack);
+ aTrack->SetGraphImpl(graph);
+ ++graph->mMainThreadTrackCount;
+ graph->AppendMessage(MakeUnique<CreateMessage>(aTrack));
+}
+
+void MediaTrackGraphImpl::RemoveTrack(MediaTrack* aTrack) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0);
+
+ mAudioOutputParams.RemoveElementsBy(
+ [&](const TrackKeyDeviceAndVolume& aElement) {
+ if (aElement.mTrack != aTrack) {
+ return false;
+ };
+ DecrementOutputDeviceRefCnt(aElement.mDeviceID);
+ return true;
+ });
+
+ if (--mMainThreadTrackCount == 0) {
+ LOG(LogLevel::Info, ("MediaTrackGraph %p, last track %p removed from "
+ "main thread. Graph will shut down.",
+ this, aTrack));
+ if (mRealtime) {
+ // Find the graph in the hash table and remove it.
+ GraphHashSet* graphs = Graphs();
+ GraphHashSet::Ptr p = graphs->lookup(*this);
+ MOZ_ASSERT(*p == this);
+ graphs->remove(p);
+
+ nsCOMPtr<nsIObserverService> observerService =
+ mozilla::services::GetObserverService();
+ if (observerService) {
+ observerService->RemoveObserver(this, "document-title-changed");
+ }
+ }
+ // The graph thread will shut itself down soon, but won't be able to do
+ // that if JS continues to run.
+ InterruptJS();
+ }
+}
+
+auto MediaTrackGraphImpl::NotifyWhenDeviceStarted(AudioDeviceID aDeviceID)
+ -> RefPtr<GraphStartedPromise> {
+ MOZ_ASSERT(NS_IsMainThread());
+
+ size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID);
+ if (index == decltype(mOutputDeviceRefCnts)::NoIndex) {
+ return GraphStartedPromise::CreateAndReject(NS_ERROR_INVALID_ARG, __func__);
+ }
+
+ MozPromiseHolder<GraphStartedPromise> h;
+ RefPtr<GraphStartedPromise> p = h.Ensure(__func__);
+
+ if (CrossGraphReceiver* receiver = mOutputDeviceRefCnts[index].mReceiver) {
+ receiver->GraphImpl()->NotifyWhenPrimaryDeviceStarted(std::move(h));
+ return p;
+ }
+
+ // aSink corresponds to the primary audio output device of this graph.
+ NotifyWhenPrimaryDeviceStarted(std::move(h));
+ return p;
+}
+
+void MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted(
+ MozPromiseHolder<GraphStartedPromise>&& aHolder) {
+ MOZ_ASSERT(NS_IsMainThread());
+ if (mOutputDeviceRefCnts[0].mRefCnt == 0) {
+ // There are no track outputs that require the device, so the creator of
+ // this promise no longer needs to know when the graph is running. Don't
+ // keep the graph alive with another message.
+ aHolder.Reject(NS_ERROR_NOT_AVAILABLE, __func__);
+ return;
+ }
+
+ QueueControlOrShutdownMessage(
+ [self = RefPtr{this}, this,
+ holder = std::move(aHolder)](IsInShutdown aInShutdown) mutable {
+ if (aInShutdown == IsInShutdown::Yes) {
+ holder.Reject(NS_ERROR_ILLEGAL_DURING_SHUTDOWN, __func__);
+ return;
+ }
+
+ TRACE("MTG::NotifyWhenPrimaryDeviceStarted ControlMessage");
+ // This runs on the graph thread, so when this runs, and the current
+ // driver is an AudioCallbackDriver, we know the audio hardware is
+ // started. If not, we are going to switch soon, keep reposting this
+ // ControlMessage.
+ if (CurrentDriver()->AsAudioCallbackDriver() &&
+ CurrentDriver()->ThreadRunning() &&
+ !CurrentDriver()->AsAudioCallbackDriver()->OnFallback()) {
+ // Avoid Resolve's locking on the graph thread by doing it on main.
+ Dispatch(NS_NewRunnableFunction(
+ "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted::Resolver",
+ [holder = std::move(holder)]() mutable {
+ holder.Resolve(true, __func__);
+ }));
+ } else {
+ DispatchToMainThreadStableState(
+ NewRunnableMethod<
+ StoreCopyPassByRRef<MozPromiseHolder<GraphStartedPromise>>>(
+ "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted", this,
+ &MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted,
+ std::move(holder)));
+ }
+ });
+}
+
+class AudioContextOperationControlMessage : public ControlMessage {
+ using AudioContextOperationPromise =
+ MediaTrackGraph::AudioContextOperationPromise;
+
+ public:
+ AudioContextOperationControlMessage(
+ MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
+ AudioContextOperation aOperation,
+ MozPromiseHolder<AudioContextOperationPromise>&& aHolder)
+ : ControlMessage(aDestinationTrack),
+ mTracks(std::move(aTracks)),
+ mAudioContextOperation(aOperation),
+ mHolder(std::move(aHolder)) {}
+ void Run() override {
+ TRACE_COMMENT("MTG::ApplyAudioContextOperationImpl ControlMessage",
+ kAudioContextOptionsStrings[static_cast<uint8_t>(
+ mAudioContextOperation)]);
+ mTrack->GraphImpl()->ApplyAudioContextOperationImpl(this);
+ }
+ void RunDuringShutdown() override {
+ MOZ_ASSERT(mAudioContextOperation == AudioContextOperation::Close,
+ "We should be reviving the graph?");
+ mHolder.Reject(false, __func__);
+ }
+
+ nsTArray<RefPtr<MediaTrack>> mTracks;
+ AudioContextOperation mAudioContextOperation;
+ MozPromiseHolder<AudioContextOperationPromise> mHolder;
+};
+
+void MediaTrackGraphImpl::ApplyAudioContextOperationImpl(
+ AudioContextOperationControlMessage* aMessage) {
+ MOZ_ASSERT(OnGraphThread());
+ // Initialize state to zero. This silences a GCC warning about uninitialized
+ // values, because although the switch below initializes state for all valid
+ // enum values, the actual value could be any integer that fits in the enum.
+ AudioContextState state{0};
+ switch (aMessage->mAudioContextOperation) {
+ // Suspend and Close operations may be performed immediately because no
+ // specific kind of GraphDriver is required. CheckDriver() will schedule
+ // a change to a SystemCallbackDriver if all tracks are suspended.
+ case AudioContextOperation::Suspend:
+ state = AudioContextState::Suspended;
+ break;
+ case AudioContextOperation::Close:
+ state = AudioContextState::Closed;
+ break;
+ case AudioContextOperation::Resume:
+ // Resume operations require an AudioCallbackDriver. CheckDriver() will
+ // schedule an AudioCallbackDriver if necessary and process pending
+ // operations if and when an AudioCallbackDriver is running.
+ mPendingResumeOperations.EmplaceBack(aMessage);
+ return;
+ }
+ // First resolve any pending Resume promises for the same AudioContext so as
+ // to resolve its associated promises in the same order as they were
+ // created. These Resume operations are considered complete and immediately
+ // canceled by the Suspend or Close.
+ MediaTrack* destinationTrack = aMessage->GetTrack();
+ bool shrinking = false;
+ auto moveDest = mPendingResumeOperations.begin();
+ for (PendingResumeOperation& op : mPendingResumeOperations) {
+ if (op.DestinationTrack() == destinationTrack) {
+ op.Apply(this);
+ shrinking = true;
+ continue;
+ }
+ if (shrinking) { // Fill-in gaps in the array.
+ *moveDest = std::move(op);
+ }
+ ++moveDest;
+ }
+ mPendingResumeOperations.TruncateLength(moveDest -
+ mPendingResumeOperations.begin());
+
+ for (MediaTrack* track : aMessage->mTracks) {
+ track->IncrementSuspendCount();
+ }
+ // Resolve after main thread state is up to date with completed processing.
+ DispatchToMainThreadStableState(NS_NewRunnableFunction(
+ "MediaTrackGraphImpl::ApplyAudioContextOperationImpl",
+ [holder = std::move(aMessage->mHolder), state]() mutable {
+ holder.Resolve(state, __func__);
+ }));
+}
+
+MediaTrackGraphImpl::PendingResumeOperation::PendingResumeOperation(
+ AudioContextOperationControlMessage* aMessage)
+ : mDestinationTrack(aMessage->GetTrack()),
+ mTracks(std::move(aMessage->mTracks)),
+ mHolder(std::move(aMessage->mHolder)) {
+ MOZ_ASSERT(aMessage->mAudioContextOperation == AudioContextOperation::Resume);
+}
+
+void MediaTrackGraphImpl::PendingResumeOperation::Apply(
+ MediaTrackGraphImpl* aGraph) {
+ MOZ_ASSERT(aGraph->OnGraphThread());
+ for (MediaTrack* track : mTracks) {
+ track->DecrementSuspendCount();
+ }
+ // The graph is provided through the parameter so that it is available even
+ // when the track is destroyed.
+ aGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction(
+ "PendingResumeOperation::Apply", [holder = std::move(mHolder)]() mutable {
+ holder.Resolve(AudioContextState::Running, __func__);
+ }));
+}
+
+void MediaTrackGraphImpl::PendingResumeOperation::Abort() {
+ // The graph is shutting down before the operation completed.
+ MOZ_ASSERT(!mDestinationTrack->GraphImpl() ||
+ mDestinationTrack->GraphImpl()->LifecycleStateRef() ==
+ MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN);
+ mHolder.Reject(false, __func__);
+}
+
+auto MediaTrackGraph::ApplyAudioContextOperation(
+ MediaTrack* aDestinationTrack, nsTArray<RefPtr<MediaTrack>> aTracks,
+ AudioContextOperation aOperation) -> RefPtr<AudioContextOperationPromise> {
+ MozPromiseHolder<AudioContextOperationPromise> holder;
+ RefPtr<AudioContextOperationPromise> p = holder.Ensure(__func__);
+ MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
+ graphImpl->AppendMessage(MakeUnique<AudioContextOperationControlMessage>(
+ aDestinationTrack, std::move(aTracks), aOperation, std::move(holder)));
+ return p;
+}
+
+uint32_t MediaTrackGraphImpl::PrimaryOutputChannelCount() const {
+ MOZ_ASSERT(!mOutputDevices[0].mReceiver);
+ return AudioOutputChannelCount(mOutputDevices[0]);
+}
+
+uint32_t MediaTrackGraphImpl::AudioOutputChannelCount(
+ const OutputDeviceEntry& aDevice) const {
+ MOZ_ASSERT(OnGraphThread());
+ // The audio output channel count for a graph is the maximum of the output
+ // channel count of all the tracks with outputs to this device, or the max
+ // audio output channel count the machine can do, whichever is smaller.
+ uint32_t channelCount = 0;
+ for (const auto& output : aDevice.mTrackOutputs) {
+ channelCount = std::max(channelCount, output.mTrack->NumberOfChannels());
+ }
+ channelCount = std::min(channelCount, mMaxOutputChannelCount);
+ if (channelCount) {
+ return channelCount;
+ } else {
+ // null aDevice.mReceiver indicates the primary graph output device.
+ if (!aDevice.mReceiver && CurrentDriver()->AsAudioCallbackDriver()) {
+ return CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount();
+ }
+ return 2;
+ }
+}
+
+double MediaTrackGraph::AudioOutputLatency() {
+ return static_cast<MediaTrackGraphImpl*>(this)->AudioOutputLatency();
+}
+
+double MediaTrackGraphImpl::AudioOutputLatency() {
+ MOZ_ASSERT(NS_IsMainThread());
+ if (mAudioOutputLatency != 0.0) {
+ return mAudioOutputLatency;
+ }
+ MonitorAutoLock lock(mMonitor);
+ if (CurrentDriver()->AsAudioCallbackDriver()) {
+ mAudioOutputLatency = CurrentDriver()
+ ->AsAudioCallbackDriver()
+ ->AudioOutputLatency()
+ .ToSeconds();
+ } else {
+ // Failure mode: return 0.0 if running on a normal thread.
+ mAudioOutputLatency = 0.0;
+ }
+
+ return mAudioOutputLatency;
+}
+
+bool MediaTrackGraph::IsNonRealtime() const {
+ return !static_cast<const MediaTrackGraphImpl*>(this)->mRealtime;
+}
+
+void MediaTrackGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess) {
+ MOZ_ASSERT(NS_IsMainThread(), "main thread only");
+
+ MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this);
+ NS_ASSERTION(!graph->mRealtime, "non-realtime only");
+
+ class Message : public ControlMessage {
+ public:
+ explicit Message(MediaTrackGraphImpl* aGraph, uint32_t aTicksToProcess)
+ : ControlMessage(nullptr),
+ mGraph(aGraph),
+ mTicksToProcess(aTicksToProcess) {}
+ void Run() override {
+ TRACE("MTG::StartNonRealtimeProcessing ControlMessage");
+ MOZ_ASSERT(mGraph->mEndTime == 0,
+ "StartNonRealtimeProcessing should be called only once");
+ mGraph->mEndTime = mGraph->RoundUpToEndOfAudioBlock(
+ mGraph->mStateComputedTime + mTicksToProcess);
+ }
+ // The graph owns this message.
+ MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph;
+ uint32_t mTicksToProcess;
+ };
+
+ graph->AppendMessage(MakeUnique<Message>(graph, aTicksToProcess));
+}
+
+void MediaTrackGraphImpl::InterruptJS() {
+ MonitorAutoLock lock(mMonitor);
+ mInterruptJSCalled = true;
+ if (mJSContext) {
+ JS_RequestInterruptCallback(mJSContext);
+ }
+}
+
+static bool InterruptCallback(JSContext* aCx) {
+ // Interrupt future calls also.
+ JS_RequestInterruptCallback(aCx);
+ // Stop execution.
+ return false;
+}
+
+void MediaTrackGraph::NotifyJSContext(JSContext* aCx) {
+ MOZ_ASSERT(OnGraphThread());
+ MOZ_ASSERT(aCx);
+
+ auto* impl = static_cast<MediaTrackGraphImpl*>(this);
+ MonitorAutoLock lock(impl->mMonitor);
+ if (impl->mJSContext) {
+ MOZ_ASSERT(impl->mJSContext == aCx);
+ return;
+ }
+ JS_AddInterruptCallback(aCx, InterruptCallback);
+ impl->mJSContext = aCx;
+ if (impl->mInterruptJSCalled) {
+ JS_RequestInterruptCallback(aCx);
+ }
+}
+
+void ProcessedMediaTrack::AddInput(MediaInputPort* aPort) {
+ MediaTrack* t = aPort->GetSource();
+ if (!t->IsSuspended()) {
+ mInputs.AppendElement(aPort);
+ } else {
+ mSuspendedInputs.AppendElement(aPort);
+ }
+ GraphImpl()->SetTrackOrderDirty();
+}
+
+void ProcessedMediaTrack::InputSuspended(MediaInputPort* aPort) {
+ GraphImpl()->AssertOnGraphThreadOrNotRunning();
+ mInputs.RemoveElement(aPort);
+ mSuspendedInputs.AppendElement(aPort);
+ GraphImpl()->SetTrackOrderDirty();
+}
+
+void ProcessedMediaTrack::InputResumed(MediaInputPort* aPort) {
+ GraphImpl()->AssertOnGraphThreadOrNotRunning();
+ mSuspendedInputs.RemoveElement(aPort);
+ mInputs.AppendElement(aPort);
+ GraphImpl()->SetTrackOrderDirty();
+}
+
+void MediaTrackGraphImpl::SwitchAtNextIteration(GraphDriver* aNextDriver) {
+ MOZ_ASSERT(OnGraphThread());
+ LOG(LogLevel::Debug, ("%p: Switching to new driver: %p", this, aNextDriver));
+ if (GraphDriver* nextDriver = NextDriver()) {
+ if (nextDriver != CurrentDriver()) {
+ LOG(LogLevel::Debug,
+ ("%p: Discarding previous next driver: %p", this, nextDriver));
+ }
+ }
+ mNextDriver = aNextDriver;
+}
+
+void MediaTrackGraph::RegisterCaptureTrackForWindow(
+ uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
+ graphImpl->RegisterCaptureTrackForWindow(aWindowId, aCaptureTrack);
+}
+
+void MediaTrackGraphImpl::RegisterCaptureTrackForWindow(
+ uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) {
+ MOZ_ASSERT(NS_IsMainThread());
+ WindowAndTrack winAndTrack;
+ winAndTrack.mWindowId = aWindowId;
+ winAndTrack.mCaptureTrackSink = aCaptureTrack;
+ mWindowCaptureTracks.AppendElement(winAndTrack);
+}
+
+void MediaTrackGraph::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
+ MOZ_ASSERT(NS_IsMainThread());
+ MediaTrackGraphImpl* graphImpl = static_cast<MediaTrackGraphImpl*>(this);
+ graphImpl->UnregisterCaptureTrackForWindow(aWindowId);
+}
+
+void MediaTrackGraphImpl::UnregisterCaptureTrackForWindow(uint64_t aWindowId) {
+ MOZ_ASSERT(NS_IsMainThread());
+ mWindowCaptureTracks.RemoveElementsBy(
+ [aWindowId](const auto& track) { return track.mWindowId == aWindowId; });
+}
+
+already_AddRefed<MediaInputPort> MediaTrackGraph::ConnectToCaptureTrack(
+ uint64_t aWindowId, MediaTrack* aMediaTrack) {
+ return aMediaTrack->GraphImpl()->ConnectToCaptureTrack(aWindowId,
+ aMediaTrack);
+}
+
+already_AddRefed<MediaInputPort> MediaTrackGraphImpl::ConnectToCaptureTrack(
+ uint64_t aWindowId, MediaTrack* aMediaTrack) {
+ MOZ_ASSERT(NS_IsMainThread());
+ for (uint32_t i = 0; i < mWindowCaptureTracks.Length(); i++) {
+ if (mWindowCaptureTracks[i].mWindowId == aWindowId) {
+ ProcessedMediaTrack* sink = mWindowCaptureTracks[i].mCaptureTrackSink;
+ return sink->AllocateInputPort(aMediaTrack);
+ }
+ }
+ return nullptr;
+}
+
+void MediaTrackGraph::DispatchToMainThreadStableState(
+ already_AddRefed<nsIRunnable> aRunnable) {
+ AssertOnGraphThreadOrNotRunning();
+ static_cast<MediaTrackGraphImpl*>(this)
+ ->mPendingUpdateRunnables.AppendElement(std::move(aRunnable));
+}
+
+Watchable<mozilla::GraphTime>& MediaTrackGraphImpl::CurrentTime() {
+ MOZ_ASSERT(NS_IsMainThread());
+ return mMainThreadGraphTime;
+}
+
+GraphTime MediaTrackGraph::ProcessedTime() const {
+ AssertOnGraphThreadOrNotRunning();
+ return static_cast<const MediaTrackGraphImpl*>(this)->mProcessedTime;
+}
+
+void* MediaTrackGraph::CurrentDriver() const {
+ AssertOnGraphThreadOrNotRunning();
+ return static_cast<const MediaTrackGraphImpl*>(this)->mDriver;
+}
+
+uint32_t MediaTrackGraphImpl::AudioInputChannelCount(
+ CubebUtils::AudioDeviceID aID) {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+ DeviceInputTrack* t =
+ mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
+ return t ? t->MaxRequestedInputChannels() : 0;
+}
+
+AudioInputType MediaTrackGraphImpl::AudioInputDevicePreference(
+ CubebUtils::AudioDeviceID aID) {
+ MOZ_ASSERT(OnGraphThreadOrNotRunning());
+ DeviceInputTrack* t =
+ mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID);
+ return t && t->HasVoiceInput() ? AudioInputType::Voice
+ : AudioInputType::Unknown;
+}
+
+void MediaTrackGraphImpl::SetNewNativeInput() {
+ MOZ_ASSERT(NS_IsMainThread());
+ MOZ_ASSERT(!mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
+
+ LOG(LogLevel::Debug, ("%p SetNewNativeInput", this));
+
+ NonNativeInputTrack* track =
+ mDeviceInputTrackManagerMainThread.GetFirstNonNativeInputTrack();
+ if (!track) {
+ LOG(LogLevel::Debug, ("%p No other devices opened. Do nothing", this));
+ return;
+ }
+
+ const CubebUtils::AudioDeviceID deviceId = track->mDeviceId;
+ const PrincipalHandle principal = track->mPrincipalHandle;
+
+ LOG(LogLevel::Debug,
+ ("%p Select device %p as the new native input device", this, deviceId));
+
+ struct TrackListener {
+ DeviceInputConsumerTrack* track;
+ // Keep its reference so it won't be dropped when after
+ // DisconnectDeviceInput().
+ RefPtr<AudioDataListener> listener;
+ };
+ nsTArray<TrackListener> pairs;
+
+ for (const auto& t : track->GetConsumerTracks()) {
+ pairs.AppendElement(
+ TrackListener{t.get(), t->GetAudioDataListener().get()});
+ }
+
+ for (TrackListener& pair : pairs) {
+ pair.track->DisconnectDeviceInput();
+ }
+
+ for (TrackListener& pair : pairs) {
+ pair.track->ConnectDeviceInput(deviceId, pair.listener.get(), principal);
+ LOG(LogLevel::Debug,
+ ("%p: Reinitialize AudioProcessingTrack %p for device %p", this,
+ pair.track, deviceId));
+ }
+
+ LOG(LogLevel::Debug,
+ ("%p Native input device is set to device %p now", this, deviceId));
+
+ MOZ_ASSERT(mDeviceInputTrackManagerMainThread.GetNativeInputTrack());
+}
+
+// nsIThreadObserver methods
+
+NS_IMETHODIMP
+MediaTrackGraphImpl::OnDispatchedEvent() {
+ MonitorAutoLock lock(mMonitor);
+ EnsureNextIteration();
+ return NS_OK;
+}
+
+NS_IMETHODIMP
+MediaTrackGraphImpl::OnProcessNextEvent(nsIThreadInternal*, bool) {
+ return NS_OK;
+}
+
+NS_IMETHODIMP
+MediaTrackGraphImpl::AfterProcessNextEvent(nsIThreadInternal*, bool) {
+ return NS_OK;
+}
+} // namespace mozilla