/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "MediaTrackGraphImpl.h" #include "mozilla/MathAlgorithms.h" #include "mozilla/Unused.h" #include "AudioSegment.h" #include "CrossGraphPort.h" #include "VideoSegment.h" #include "nsContentUtils.h" #include "nsGlobalWindowInner.h" #include "nsPrintfCString.h" #include "nsServiceManagerUtils.h" #include "prerror.h" #include "mozilla/Logging.h" #include "mozilla/Attributes.h" #include "ForwardedInputTrack.h" #include "ImageContainer.h" #include "AudioCaptureTrack.h" #include "AudioDeviceInfo.h" #include "AudioNodeTrack.h" #include "AudioNodeExternalInputTrack.h" #if defined(MOZ_WEBRTC) # include "MediaEngineWebRTCAudio.h" #endif // MOZ_WEBRTC #include "MediaTrackListener.h" #include "mozilla/dom/BaseAudioContextBinding.h" #include "mozilla/dom/Document.h" #include "mozilla/dom/WorkletThread.h" #include "mozilla/media/MediaUtils.h" #include #include "GeckoProfiler.h" #include "VideoFrameContainer.h" #include "mozilla/AbstractThread.h" #include "mozilla/StaticPrefs_dom.h" #include "mozilla/StaticPrefs_media.h" #include "transport/runnable_utils.h" #include "VideoUtils.h" #include "GraphRunner.h" #include "Tracing.h" #include "UnderrunHandler.h" #include "mozilla/CycleCollectedJSRuntime.h" #include "mozilla/Preferences.h" #include "webaudio/blink/DenormalDisabler.h" #include "webaudio/blink/HRTFDatabaseLoader.h" using namespace mozilla::layers; using namespace mozilla::dom; using namespace mozilla::gfx; using namespace mozilla::media; namespace mozilla { using AudioDeviceID = CubebUtils::AudioDeviceID; using IsInShutdown = MediaTrack::IsInShutdown; LazyLogModule gMediaTrackGraphLog("MediaTrackGraph"); #ifdef LOG # undef LOG #endif // LOG #define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg) NativeInputTrack* DeviceInputTrackManager::GetNativeInputTrack() { return mNativeInputTrack.get(); } DeviceInputTrack* DeviceInputTrackManager::GetDeviceInputTrack( CubebUtils::AudioDeviceID aID) { if (mNativeInputTrack && mNativeInputTrack->mDeviceId == aID) { return mNativeInputTrack.get(); } for (const RefPtr& t : mNonNativeInputTracks) { if (t->mDeviceId == aID) { return t.get(); } } return nullptr; } NonNativeInputTrack* DeviceInputTrackManager::GetFirstNonNativeInputTrack() { if (mNonNativeInputTracks.IsEmpty()) { return nullptr; } return mNonNativeInputTracks[0].get(); } void DeviceInputTrackManager::Add(DeviceInputTrack* aTrack) { if (NativeInputTrack* native = aTrack->AsNativeInputTrack()) { MOZ_ASSERT(!mNativeInputTrack); mNativeInputTrack = native; } else { NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack(); MOZ_ASSERT(nonNative); struct DeviceTrackComparator { public: bool Equals(const RefPtr& aTrack, CubebUtils::AudioDeviceID aDeviceId) const { return aTrack->mDeviceId == aDeviceId; } }; MOZ_ASSERT(!mNonNativeInputTracks.Contains(aTrack->mDeviceId, DeviceTrackComparator())); mNonNativeInputTracks.AppendElement(nonNative); } } void DeviceInputTrackManager::Remove(DeviceInputTrack* aTrack) { if (aTrack->AsNativeInputTrack()) { MOZ_ASSERT(mNativeInputTrack); MOZ_ASSERT(mNativeInputTrack.get() == aTrack->AsNativeInputTrack()); mNativeInputTrack = nullptr; } else { NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack(); MOZ_ASSERT(nonNative); DebugOnly removed = mNonNativeInputTracks.RemoveElement(nonNative); MOZ_ASSERT(removed); } } /** * A hash table containing the graph instances, one per Window ID, * sample rate, and device ID combination. */ struct MediaTrackGraphImpl::Lookup final { HashNumber Hash() const { return HashGeneric(mWindowID, mSampleRate, mOutputDeviceID); } const uint64_t mWindowID; const TrackRate mSampleRate; const CubebUtils::AudioDeviceID mOutputDeviceID; }; // Implicit to support GraphHashSet.lookup(*graph). MOZ_IMPLICIT MediaTrackGraphImpl::operator MediaTrackGraphImpl::Lookup() const { return {mWindowID, mSampleRate, PrimaryOutputDeviceID()}; } namespace { struct GraphHasher { // for HashSet using Lookup = const MediaTrackGraphImpl::Lookup; static HashNumber hash(const Lookup& aLookup) { return aLookup.Hash(); } static bool match(const MediaTrackGraphImpl* aGraph, const Lookup& aLookup) { return aGraph->mWindowID == aLookup.mWindowID && aGraph->GraphRate() == aLookup.mSampleRate && aGraph->PrimaryOutputDeviceID() == aLookup.mOutputDeviceID; } }; // The weak reference to the graph is removed when its last track is removed. using GraphHashSet = HashSet; GraphHashSet* Graphs() { MOZ_ASSERT(NS_IsMainThread()); static GraphHashSet sGraphs(4); // 4 is minimum HashSet capacity return &sGraphs; } } // anonymous namespace static void ApplyTrackDisabling(DisabledTrackMode aDisabledMode, MediaSegment* aSegment, MediaSegment* aRawSegment) { if (aDisabledMode == DisabledTrackMode::ENABLED) { return; } if (aDisabledMode == DisabledTrackMode::SILENCE_BLACK) { aSegment->ReplaceWithDisabled(); if (aRawSegment) { aRawSegment->ReplaceWithDisabled(); } } else if (aDisabledMode == DisabledTrackMode::SILENCE_FREEZE) { aSegment->ReplaceWithNull(); if (aRawSegment) { aRawSegment->ReplaceWithNull(); } } else { MOZ_CRASH("Unsupported mode"); } } MediaTrackGraphImpl::~MediaTrackGraphImpl() { MOZ_ASSERT(mTracks.IsEmpty() && mSuspendedTracks.IsEmpty(), "All tracks should have been destroyed by messages from the main " "thread"); LOG(LogLevel::Debug, ("MediaTrackGraph %p destroyed", this)); LOG(LogLevel::Debug, ("MediaTrackGraphImpl::~MediaTrackGraphImpl")); } void MediaTrackGraphImpl::AddTrackGraphThread(MediaTrack* aTrack) { MOZ_ASSERT(OnGraphThreadOrNotRunning()); aTrack->mStartTime = mProcessedTime; if (aTrack->IsSuspended()) { mSuspendedTracks.AppendElement(aTrack); LOG(LogLevel::Debug, ("%p: Adding media track %p, in the suspended track array", this, aTrack)); } else { mTracks.AppendElement(aTrack); LOG(LogLevel::Debug, ("%p: Adding media track %p, count %zu", this, aTrack, mTracks.Length())); } SetTrackOrderDirty(); } void MediaTrackGraphImpl::RemoveTrackGraphThread(MediaTrack* aTrack) { MOZ_ASSERT(OnGraphThreadOrNotRunning()); // Remove references in mTrackUpdates before we allow aTrack to die. // Pending updates are not needed (since the main thread has already given // up the track) so we will just drop them. { MonitorAutoLock lock(mMonitor); for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) { if (mTrackUpdates[i].mTrack == aTrack) { mTrackUpdates[i].mTrack = nullptr; } } } // Ensure that mFirstCycleBreaker is updated when necessary. SetTrackOrderDirty(); UnregisterAllAudioOutputs(aTrack); if (aTrack->IsSuspended()) { mSuspendedTracks.RemoveElement(aTrack); } else { mTracks.RemoveElement(aTrack); } LOG(LogLevel::Debug, ("%p: Removed media track %p, count %zu", this, aTrack, mTracks.Length())); NS_RELEASE(aTrack); // probably destroying it } TrackTime MediaTrackGraphImpl::GraphTimeToTrackTimeWithBlocking( const MediaTrack* aTrack, GraphTime aTime) const { MOZ_ASSERT( aTime <= mStateComputedTime, "Don't ask about times where we haven't made blocking decisions yet"); return std::max( 0, std::min(aTime, aTrack->mStartBlocking) - aTrack->mStartTime); } GraphTime MediaTrackGraphImpl::IterationEnd() const { MOZ_ASSERT(OnGraphThread()); return mIterationEndTime; } void MediaTrackGraphImpl::UpdateCurrentTimeForTracks( GraphTime aPrevCurrentTime) { MOZ_ASSERT(OnGraphThread()); for (MediaTrack* track : AllTracks()) { // Shouldn't have already notified of ended *and* have output! MOZ_ASSERT_IF(track->mStartBlocking > aPrevCurrentTime, !track->mNotifiedEnded); // Calculate blocked time and fire Blocked/Unblocked events GraphTime blockedTime = mStateComputedTime - track->mStartBlocking; NS_ASSERTION(blockedTime >= 0, "Error in blocking time"); track->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime, blockedTime); LOG(LogLevel::Verbose, ("%p: MediaTrack %p bufferStartTime=%f blockedTime=%f", this, track, MediaTimeToSeconds(track->mStartTime), MediaTimeToSeconds(blockedTime))); track->mStartBlocking = mStateComputedTime; TrackTime trackCurrentTime = track->GraphTimeToTrackTime(mStateComputedTime); if (track->mEnded) { MOZ_ASSERT(track->GetEnd() <= trackCurrentTime); if (!track->mNotifiedEnded) { // Playout of this track ended and listeners have not been notified. track->mNotifiedEnded = true; SetTrackOrderDirty(); for (const auto& listener : track->mTrackListeners) { listener->NotifyOutput(this, track->GetEnd()); listener->NotifyEnded(this); } } } else { for (const auto& listener : track->mTrackListeners) { listener->NotifyOutput(this, trackCurrentTime); } } } } template void MediaTrackGraphImpl::ProcessChunkMetadataForInterval(MediaTrack* aTrack, C& aSegment, TrackTime aStart, TrackTime aEnd) { MOZ_ASSERT(OnGraphThreadOrNotRunning()); MOZ_ASSERT(aTrack); TrackTime offset = 0; for (typename C::ConstChunkIterator chunk(aSegment); !chunk.IsEnded(); chunk.Next()) { if (offset >= aEnd) { break; } offset += chunk->GetDuration(); if (chunk->IsNull() || offset < aStart) { continue; } const PrincipalHandle& principalHandle = chunk->GetPrincipalHandle(); if (principalHandle != aSegment.GetLastPrincipalHandle()) { aSegment.SetLastPrincipalHandle(principalHandle); LOG(LogLevel::Debug, ("%p: MediaTrack %p, principalHandle " "changed in %sChunk with duration %lld", this, aTrack, aSegment.GetType() == MediaSegment::AUDIO ? "Audio" : "Video", (long long)chunk->GetDuration())); for (const auto& listener : aTrack->mTrackListeners) { listener->NotifyPrincipalHandleChanged(this, principalHandle); } } } } void MediaTrackGraphImpl::ProcessChunkMetadata(GraphTime aPrevCurrentTime) { MOZ_ASSERT(OnGraphThreadOrNotRunning()); for (MediaTrack* track : AllTracks()) { TrackTime iterationStart = track->GraphTimeToTrackTime(aPrevCurrentTime); TrackTime iterationEnd = track->GraphTimeToTrackTime(mProcessedTime); if (!track->mSegment) { continue; } if (track->mType == MediaSegment::AUDIO) { ProcessChunkMetadataForInterval( track, *track->GetData(), iterationStart, iterationEnd); } else if (track->mType == MediaSegment::VIDEO) { ProcessChunkMetadataForInterval( track, *track->GetData(), iterationStart, iterationEnd); } else { MOZ_CRASH("Unknown track type"); } } } GraphTime MediaTrackGraphImpl::WillUnderrun(MediaTrack* aTrack, GraphTime aEndBlockingDecisions) { // Ended tracks can't underrun. ProcessedMediaTracks also can't cause // underrun currently, since we'll always be able to produce data for them // unless they block on some other track. if (aTrack->mEnded || aTrack->AsProcessedTrack()) { return aEndBlockingDecisions; } // This track isn't ended or suspended. We don't need to call // TrackTimeToGraphTime since an underrun is the only thing that can block // it. GraphTime bufferEnd = aTrack->GetEnd() + aTrack->mStartTime; #ifdef DEBUG if (bufferEnd < mProcessedTime) { LOG(LogLevel::Error, ("%p: MediaTrack %p underrun, " "bufferEnd %f < mProcessedTime %f (%" PRId64 " < %" PRId64 "), TrackTime %" PRId64, this, aTrack, MediaTimeToSeconds(bufferEnd), MediaTimeToSeconds(mProcessedTime), bufferEnd, mProcessedTime, aTrack->GetEnd())); NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran"); } #endif return std::min(bufferEnd, aEndBlockingDecisions); } namespace { // Value of mCycleMarker for unvisited tracks in cycle detection. const uint32_t NOT_VISITED = UINT32_MAX; // Value of mCycleMarker for ordered tracks in muted cycles. const uint32_t IN_MUTED_CYCLE = 1; } // namespace bool MediaTrackGraphImpl::AudioTrackPresent() { MOZ_ASSERT(OnGraphThreadOrNotRunning()); bool audioTrackPresent = false; for (MediaTrack* track : mTracks) { if (track->AsAudioNodeTrack()) { audioTrackPresent = true; break; } if (track->mType == MediaSegment::AUDIO && !track->mNotifiedEnded) { audioTrackPresent = true; break; } } // We may not have audio input device when we only have AudioNodeTracks. But // if audioTrackPresent is false, we must have no input device. MOZ_DIAGNOSTIC_ASSERT_IF( !audioTrackPresent, !mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()); return audioTrackPresent; } void MediaTrackGraphImpl::CheckDriver() { MOZ_ASSERT(OnGraphThread()); // An offline graph has only one driver. // Otherwise, if a switch is already pending, let that happen. if (!mRealtime || Switching()) { return; } AudioCallbackDriver* audioCallbackDriver = CurrentDriver()->AsAudioCallbackDriver(); if (audioCallbackDriver && !audioCallbackDriver->OnFallback()) { for (PendingResumeOperation& op : mPendingResumeOperations) { op.Apply(this); } mPendingResumeOperations.Clear(); } // Note that this looks for any audio tracks, input or output, and switches // to a SystemClockDriver if there are none active or no resume operations // to make any active. bool needAudioCallbackDriver = !mPendingResumeOperations.IsEmpty() || AudioTrackPresent(); if (!needAudioCallbackDriver) { if (audioCallbackDriver && audioCallbackDriver->IsStarted()) { SwitchAtNextIteration( new SystemClockDriver(this, CurrentDriver(), mSampleRate)); } return; } NativeInputTrack* native = mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); CubebUtils::AudioDeviceID inputDevice = native ? native->mDeviceId : nullptr; uint32_t inputChannelCount = AudioInputChannelCount(inputDevice); AudioInputType inputPreference = AudioInputDevicePreference(inputDevice); cubeb_input_processing_params inputProcessingParams = RequestedAudioInputProcessingParams(inputDevice); uint32_t primaryOutputChannelCount = PrimaryOutputChannelCount(); if (!audioCallbackDriver) { if (primaryOutputChannelCount > 0) { AudioCallbackDriver* driver = new AudioCallbackDriver( this, CurrentDriver(), mSampleRate, primaryOutputChannelCount, inputChannelCount, PrimaryOutputDeviceID(), inputDevice, inputPreference, inputProcessingParams); SwitchAtNextIteration(driver); } return; } // Check if this graph should switch to a different number of output channels. // Generally, a driver switch is explicitly made by an event (e.g., setting // the AudioDestinationNode channelCount), but if an HTMLMediaElement is // directly playing back via another HTMLMediaElement, the number of channels // of the media determines how many channels to output, and it can change // dynamically. if (primaryOutputChannelCount != audioCallbackDriver->OutputChannelCount()) { AudioCallbackDriver* driver = new AudioCallbackDriver( this, CurrentDriver(), mSampleRate, primaryOutputChannelCount, inputChannelCount, PrimaryOutputDeviceID(), inputDevice, inputPreference, inputProcessingParams); SwitchAtNextIteration(driver); } if (native) { audioCallbackDriver->SetRequestedInputProcessingParams( inputProcessingParams); } } void MediaTrackGraphImpl::UpdateTrackOrder() { if (!mTrackOrderDirty) { return; } mTrackOrderDirty = false; // The algorithm for finding cycles is based on Tim Leslie's iterative // implementation [1][2] of Pearce's variant [3] of Tarjan's strongly // connected components (SCC) algorithm. There are variations (a) to // distinguish whether tracks in SCCs of size 1 are in a cycle and (b) to // re-run the algorithm over SCCs with breaks at DelayNodes. // // [1] http://www.timl.id.au/?p=327 // [2] // https://github.com/scipy/scipy/blob/e2c502fca/scipy/sparse/csgraph/_traversal.pyx#L582 // [3] http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.102.1707 // // There are two stacks. One for the depth-first search (DFS), mozilla::LinkedList dfsStack; // and another for tracks popped from the DFS stack, but still being // considered as part of SCCs involving tracks on the stack. mozilla::LinkedList sccStack; // An index into mTracks for the next track found with no unsatisfied // upstream dependencies. uint32_t orderedTrackCount = 0; for (uint32_t i = 0; i < mTracks.Length(); ++i) { MediaTrack* t = mTracks[i]; ProcessedMediaTrack* pt = t->AsProcessedTrack(); if (pt) { // The dfsStack initially contains a list of all processed tracks in // unchanged order. dfsStack.insertBack(t); pt->mCycleMarker = NOT_VISITED; } else { // SourceMediaTracks have no inputs and so can be ordered now. mTracks[orderedTrackCount] = t; ++orderedTrackCount; } } // mNextStackMarker corresponds to "index" in Tarjan's algorithm. It is a // counter to label mCycleMarker on the next visited track in the DFS // uniquely in the set of visited tracks that are still being considered. // // In this implementation, the counter descends so that the values are // strictly greater than the values that mCycleMarker takes when the track // has been ordered (0 or IN_MUTED_CYCLE). // // Each new track labelled, as the DFS searches upstream, receives a value // less than those used for all other tracks being considered. uint32_t nextStackMarker = NOT_VISITED - 1; // Reset list of DelayNodes in cycles stored at the tail of mTracks. mFirstCycleBreaker = mTracks.Length(); // Rearrange dfsStack order as required to DFS upstream and pop tracks // in processing order to place in mTracks. while (auto pt = static_cast(dfsStack.getFirst())) { const auto& inputs = pt->mInputs; MOZ_ASSERT(pt->AsProcessedTrack()); if (pt->mCycleMarker == NOT_VISITED) { // Record the position on the visited stack, so that any searches // finding this track again know how much of the stack is in the cycle. pt->mCycleMarker = nextStackMarker; --nextStackMarker; // Not-visited input tracks should be processed first. // SourceMediaTracks have already been ordered. for (uint32_t i = inputs.Length(); i--;) { if (inputs[i]->GetSource()->IsSuspended()) { continue; } auto input = inputs[i]->GetSource()->AsProcessedTrack(); if (input && input->mCycleMarker == NOT_VISITED) { // It can be that this track has an input which is from a suspended // AudioContext. if (input->isInList()) { input->remove(); dfsStack.insertFront(input); } } } continue; } // Returning from DFS. Pop from dfsStack. pt->remove(); // cycleStackMarker keeps track of the highest marker value on any // upstream track, if any, found receiving input, directly or indirectly, // from the visited stack (and so from |ps|, making a cycle). In a // variation from Tarjan's SCC algorithm, this does not include |ps| // unless it is part of the cycle. uint32_t cycleStackMarker = 0; for (uint32_t i = inputs.Length(); i--;) { if (inputs[i]->GetSource()->IsSuspended()) { continue; } auto input = inputs[i]->GetSource()->AsProcessedTrack(); if (input) { cycleStackMarker = std::max(cycleStackMarker, input->mCycleMarker); } } if (cycleStackMarker <= IN_MUTED_CYCLE) { // All inputs have been ordered and their stack markers have been removed. // This track is not part of a cycle. It can be processed next. pt->mCycleMarker = 0; mTracks[orderedTrackCount] = pt; ++orderedTrackCount; continue; } // A cycle has been found. Record this track for ordering when all // tracks in this SCC have been popped from the DFS stack. sccStack.insertFront(pt); if (cycleStackMarker > pt->mCycleMarker) { // Cycles have been found that involve tracks that remain on the stack. // Leave mCycleMarker indicating the most downstream (last) track on // the stack known to be part of this SCC. In this way, any searches on // other paths that find |ps| will know (without having to traverse from // this track again) that they are part of this SCC (i.e. part of an // intersecting cycle). pt->mCycleMarker = cycleStackMarker; continue; } // |pit| is the root of an SCC involving no other tracks on dfsStack, the // complete SCC has been recorded, and tracks in this SCC are part of at // least one cycle. MOZ_ASSERT(cycleStackMarker == pt->mCycleMarker); // If there are DelayNodes in this SCC, then they may break the cycles. bool haveDelayNode = false; auto next = sccStack.getFirst(); // Tracks in this SCC are identified by mCycleMarker <= cycleStackMarker. // (There may be other tracks later in sccStack from other incompletely // searched SCCs, involving tracks still on dfsStack.) // // DelayNodes in cycles must behave differently from those not in cycles, // so all DelayNodes in the SCC must be identified. while (next && static_cast(next)->mCycleMarker <= cycleStackMarker) { auto nt = next->AsAudioNodeTrack(); // Get next before perhaps removing from list below. next = next->getNext(); if (nt && nt->Engine()->AsDelayNodeEngine()) { haveDelayNode = true; // DelayNodes break cycles by producing their output in a // preprocessing phase; they do not need to be ordered before their // consumers. Order them at the tail of mTracks so that they can be // handled specially. Do so now, so that DFS ignores them. nt->remove(); nt->mCycleMarker = 0; --mFirstCycleBreaker; mTracks[mFirstCycleBreaker] = nt; } } auto after_scc = next; while ((next = sccStack.getFirst()) != after_scc) { next->remove(); auto removed = static_cast(next); if (haveDelayNode) { // Return tracks to the DFS stack again (to order and detect cycles // without delayNodes). Any of these tracks that are still inputs // for tracks on the visited stack must be returned to the front of // the stack to be ordered before their dependents. We know that none // of these tracks need input from tracks on the visited stack, so // they can all be searched and ordered before the current stack head // is popped. removed->mCycleMarker = NOT_VISITED; dfsStack.insertFront(removed); } else { // Tracks in cycles without any DelayNodes must be muted, and so do // not need input and can be ordered now. They must be ordered before // their consumers so that their muted output is available. removed->mCycleMarker = IN_MUTED_CYCLE; mTracks[orderedTrackCount] = removed; ++orderedTrackCount; } } } MOZ_ASSERT(orderedTrackCount == mFirstCycleBreaker); } TrackTime MediaTrackGraphImpl::PlayAudio(const TrackAndVolume& aOutput, GraphTime aPlayedTime, uint32_t aOutputChannelCount) { MOZ_ASSERT(OnGraphThread()); MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode"); TrackTime ticksWritten = 0; ticksWritten = 0; MediaTrack* track = aOutput.mTrack; AudioSegment* audio = track->GetData(); AudioSegment output; TrackTime offset = track->GraphTimeToTrackTime(aPlayedTime); // We don't update Track->mTracksStartTime here to account for time spent // blocked. Instead, we'll update it in UpdateCurrentTimeForTracks after // the blocked period has completed. But we do need to make sure we play // from the right offsets in the track buffer, even if we've already // written silence for some amount of blocked time after the current time. GraphTime t = aPlayedTime; while (t < mStateComputedTime) { bool blocked = t >= track->mStartBlocking; GraphTime end = blocked ? mStateComputedTime : track->mStartBlocking; NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!"); // Check how many ticks of sound we can provide if we are blocked some // time in the middle of this cycle. TrackTime toWrite = end - t; if (blocked) { output.InsertNullDataAtStart(toWrite); ticksWritten += toWrite; LOG(LogLevel::Verbose, ("%p: MediaTrack %p writing %" PRId64 " blocking-silence samples for " "%f to %f (%" PRId64 " to %" PRId64 ")", this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end), offset, offset + toWrite)); } else { TrackTime endTicksNeeded = offset + toWrite; TrackTime endTicksAvailable = audio->GetDuration(); if (endTicksNeeded <= endTicksAvailable) { LOG(LogLevel::Verbose, ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f " "(samples %" PRId64 " to %" PRId64 ")", this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end), offset, endTicksNeeded)); output.AppendSlice(*audio, offset, endTicksNeeded); ticksWritten += toWrite; offset = endTicksNeeded; } else { // MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not // ended."); If we are at the end of the track, maybe write the // remaining samples, and pad with/output silence. if (endTicksNeeded > endTicksAvailable && offset < endTicksAvailable) { output.AppendSlice(*audio, offset, endTicksAvailable); LOG(LogLevel::Verbose, ("%p: MediaTrack %p writing %" PRId64 " samples for %f to %f " "(samples %" PRId64 " to %" PRId64 ")", this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end), offset, endTicksNeeded)); uint32_t available = endTicksAvailable - offset; ticksWritten += available; toWrite -= available; offset = endTicksAvailable; } output.AppendNullData(toWrite); LOG(LogLevel::Verbose, ("%p MediaTrack %p writing %" PRId64 " padding slsamples for %f to " "%f (samples %" PRId64 " to %" PRId64 ")", this, track, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end), offset, endTicksNeeded)); ticksWritten += toWrite; } output.ApplyVolume(mGlobalVolume * aOutput.mVolume); } t = end; output.Mix(mMixer, aOutputChannelCount, mSampleRate); } return ticksWritten; } DeviceInputTrack* MediaTrackGraph::GetDeviceInputTrackMainThread( CubebUtils::AudioDeviceID aID) { MOZ_ASSERT(NS_IsMainThread()); auto* impl = static_cast(this); return impl->mDeviceInputTrackManagerMainThread.GetDeviceInputTrack(aID); } NativeInputTrack* MediaTrackGraph::GetNativeInputTrackMainThread() { MOZ_ASSERT(NS_IsMainThread()); auto* impl = static_cast(this); return impl->mDeviceInputTrackManagerMainThread.GetNativeInputTrack(); } void MediaTrackGraphImpl::OpenAudioInputImpl(DeviceInputTrack* aTrack) { MOZ_ASSERT(OnGraphThread()); LOG(LogLevel::Debug, ("%p OpenAudioInputImpl: device %p", this, aTrack->mDeviceId)); mDeviceInputTrackManagerGraphThread.Add(aTrack); if (aTrack->AsNativeInputTrack()) { // Switch Drivers since we're adding input (to input-only or full-duplex) AudioCallbackDriver* driver = new AudioCallbackDriver( this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(), AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(), aTrack->mDeviceId, AudioInputDevicePreference(aTrack->mDeviceId), aTrack->RequestedProcessingParams()); LOG(LogLevel::Debug, ("%p OpenAudioInputImpl: starting new AudioCallbackDriver(input) %p", this, driver)); SwitchAtNextIteration(driver); } else { NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack(); MOZ_ASSERT(nonNative); // Start non-native input right away. nonNative->StartAudio(MakeRefPtr( MakeRefPtr(nonNative), nonNative->GenerateSourceId(), nonNative->mDeviceId, AudioInputChannelCount(nonNative->mDeviceId), AudioInputDevicePreference(nonNative->mDeviceId) == AudioInputType::Voice, nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate())); } } void MediaTrackGraphImpl::OpenAudioInput(DeviceInputTrack* aTrack) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aTrack); LOG(LogLevel::Debug, ("%p OpenInput: DeviceInputTrack %p for device %p", this, aTrack, aTrack->mDeviceId)); class Message : public ControlMessage { public: Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack) : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {} void Run() override { TRACE("MTG::OpenAudioInputImpl ControlMessage"); mGraph->OpenAudioInputImpl(mInputTrack); } MediaTrackGraphImpl* mGraph; DeviceInputTrack* mInputTrack; }; mDeviceInputTrackManagerMainThread.Add(aTrack); this->AppendMessage(MakeUnique(this, aTrack)); } void MediaTrackGraphImpl::CloseAudioInputImpl(DeviceInputTrack* aTrack) { MOZ_ASSERT(OnGraphThread()); LOG(LogLevel::Debug, ("%p CloseAudioInputImpl: device %p", this, aTrack->mDeviceId)); if (NonNativeInputTrack* nonNative = aTrack->AsNonNativeInputTrack()) { nonNative->StopAudio(); mDeviceInputTrackManagerGraphThread.Remove(aTrack); return; } MOZ_ASSERT(aTrack->AsNativeInputTrack()); mDeviceInputTrackManagerGraphThread.Remove(aTrack); // Switch Drivers since we're adding or removing an input (to nothing/system // or output only) bool audioTrackPresent = AudioTrackPresent(); GraphDriver* driver; if (audioTrackPresent) { // We still have audio output LOG(LogLevel::Debug, ("%p: CloseInput: output present (AudioCallback)", this)); driver = new AudioCallbackDriver( this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(), AudioInputChannelCount(aTrack->mDeviceId), PrimaryOutputDeviceID(), nullptr, AudioInputDevicePreference(aTrack->mDeviceId), aTrack->RequestedProcessingParams()); SwitchAtNextIteration(driver); } else if (CurrentDriver()->AsAudioCallbackDriver()) { LOG(LogLevel::Debug, ("%p: CloseInput: no output present (SystemClockCallback)", this)); driver = new SystemClockDriver(this, CurrentDriver(), mSampleRate); SwitchAtNextIteration(driver); } // else SystemClockDriver->SystemClockDriver, no switch } void MediaTrackGraphImpl::UnregisterAllAudioOutputs(MediaTrack* aTrack) { MOZ_ASSERT(OnGraphThreadOrNotRunning()); mOutputDevices.RemoveElementsBy([&](OutputDeviceEntry& aDeviceRef) { aDeviceRef.mTrackOutputs.RemoveElement(aTrack); // mReceiver is null for the primary output device, which is retained for // AudioCallbackDriver output even when no tracks have audio outputs. return aDeviceRef.mTrackOutputs.IsEmpty() && aDeviceRef.mReceiver; }); } void MediaTrackGraphImpl::CloseAudioInput(DeviceInputTrack* aTrack) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aTrack); LOG(LogLevel::Debug, ("%p CloseInput: DeviceInputTrack %p for device %p", this, aTrack, aTrack->mDeviceId)); class Message : public ControlMessage { public: Message(MediaTrackGraphImpl* aGraph, DeviceInputTrack* aInputTrack) : ControlMessage(nullptr), mGraph(aGraph), mInputTrack(aInputTrack) {} void Run() override { TRACE("MTG::CloseAudioInputImpl ControlMessage"); mGraph->CloseAudioInputImpl(mInputTrack); } MediaTrackGraphImpl* mGraph; DeviceInputTrack* mInputTrack; }; // DeviceInputTrack is still alive (in mTracks) even we remove it here, since // aTrack->Destroy() is called after this. See DeviceInputTrack::CloseAudio // for more details. mDeviceInputTrackManagerMainThread.Remove(aTrack); this->AppendMessage(MakeUnique(this, aTrack)); if (aTrack->AsNativeInputTrack()) { LOG(LogLevel::Debug, ("%p Native input device %p is closed!", this, aTrack->mDeviceId)); SetNewNativeInput(); } } // All AudioInput listeners get the same speaker data (at least for now). void MediaTrackGraphImpl::NotifyOutputData(const AudioChunk& aChunk) { if (!mDeviceInputTrackManagerGraphThread.GetNativeInputTrack()) { return; } #if defined(MOZ_WEBRTC) for (const auto& track : mTracks) { if (const auto& t = track->AsAudioProcessingTrack()) { t->NotifyOutputData(this, aChunk); } } #endif } void MediaTrackGraphImpl::NotifyInputStopped() { NativeInputTrack* native = mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); if (!native) { return; } native->NotifyInputStopped(this); } void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames, TrackRate aRate, uint32_t aChannels, uint32_t aAlreadyBuffered) { // Either we have an audio input device, or we just removed the audio input // this iteration, and we're switching back to an output-only driver next // iteration. NativeInputTrack* native = mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); MOZ_ASSERT(native || Switching()); if (!native) { return; } native->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels, aAlreadyBuffered); } void MediaTrackGraphImpl::NotifySetRequestedInputProcessingParamsResult( AudioCallbackDriver* aDriver, cubeb_input_processing_params aRequestedParams, Result&& aResult) { MOZ_ASSERT(NS_IsMainThread()); NativeInputTrack* native = mDeviceInputTrackManagerMainThread.GetNativeInputTrack(); if (!native) { return; } QueueControlMessageWithNoShutdown([this, self = RefPtr(this), driver = RefPtr(aDriver), aRequestedParams, result = std::move(aResult)]() mutable { NativeInputTrack* native = mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); if (!native) { return; } if (driver != mDriver) { return; } native->NotifySetRequestedProcessingParamsResult(this, aRequestedParams, result); }); } void MediaTrackGraphImpl::DeviceChangedImpl() { MOZ_ASSERT(OnGraphThread()); NativeInputTrack* native = mDeviceInputTrackManagerGraphThread.GetNativeInputTrack(); if (!native) { return; } native->DeviceChanged(this); } void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount) { MOZ_ASSERT(OnGraphThread()); mMaxOutputChannelCount = aMaxChannelCount; } void MediaTrackGraphImpl::DeviceChanged() { // This is safe to be called from any thread: this message comes from an // underlying platform API, and we don't have much guarantees. If it is not // called from the main thread (and it probably will rarely be), it will post // itself to the main thread, and the actual device change message will be ran // and acted upon on the graph thread. if (!NS_IsMainThread()) { RefPtr runnable = WrapRunnable( RefPtr(this), &MediaTrackGraphImpl::DeviceChanged); mMainThread->Dispatch(runnable.forget()); return; } class Message : public ControlMessage { public: explicit Message(MediaTrackGraph* aGraph) : ControlMessage(nullptr), mGraphImpl(static_cast(aGraph)) {} void Run() override { TRACE("MTG::DeviceChangeImpl ControlMessage"); mGraphImpl->DeviceChangedImpl(); } // We know that this is valid, because the graph can't shutdown if it has // messages. MediaTrackGraphImpl* mGraphImpl; }; if (mMainThreadTrackCount == 0 && mMainThreadPortCount == 0) { // This is a special case where the origin of this event cannot control the // lifetime of the graph, because the graph is controling the lifetime of // the AudioCallbackDriver where the event originated. // We know the graph is soon going away, so there's no need to notify about // this device change. return; } // Reset the latency, it will get fetched again next time it's queried. MOZ_ASSERT(NS_IsMainThread()); mAudioOutputLatency = 0.0; // Dispatch to the bg thread to do the (potentially expensive) query of the // maximum channel count, and then dispatch back to the main thread, then to // the graph, with the new info. RefPtr self = this; NS_DispatchBackgroundTask(NS_NewRunnableFunction( "MaxChannelCountUpdateOnBgThread", [self{std::move(self)}]() { uint32_t maxChannelCount = CubebUtils::MaxNumberOfChannels(); self->Dispatch(NS_NewRunnableFunction( "MaxChannelCountUpdateToMainThread", [self{self}, maxChannelCount]() { class MessageToGraph : public ControlMessage { public: explicit MessageToGraph(MediaTrackGraph* aGraph, uint32_t aMaxChannelCount) : ControlMessage(nullptr), mGraphImpl(static_cast(aGraph)), mMaxChannelCount(aMaxChannelCount) {} void Run() override { TRACE("MTG::SetMaxOutputChannelCount ControlMessage") mGraphImpl->SetMaxOutputChannelCount(mMaxChannelCount); } MediaTrackGraphImpl* mGraphImpl; uint32_t mMaxChannelCount; }; self->AppendMessage( MakeUnique(self, maxChannelCount)); })); })); AppendMessage(MakeUnique(this)); } static const char* GetAudioInputTypeString(const AudioInputType& aType) { return aType == AudioInputType::Voice ? "Voice" : "Unknown"; } void MediaTrackGraph::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) { MOZ_ASSERT(OnGraphThread()); auto* impl = static_cast(this); impl->ReevaluateInputDevice(aID); } void MediaTrackGraphImpl::ReevaluateInputDevice(CubebUtils::AudioDeviceID aID) { MOZ_ASSERT(OnGraphThread()); LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: device %p", this, aID)); DeviceInputTrack* track = mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID); if (!track) { LOG(LogLevel::Debug, ("%p: No DeviceInputTrack for this device. Ignore", this)); return; } bool needToSwitch = false; if (NonNativeInputTrack* nonNative = track->AsNonNativeInputTrack()) { if (nonNative->NumberOfChannels() != AudioInputChannelCount(aID)) { LOG(LogLevel::Debug, ("%p: %u-channel non-native input device %p (track %p) is " "re-configured to %d-channel", this, nonNative->NumberOfChannels(), aID, track, AudioInputChannelCount(aID))); needToSwitch = true; } if (nonNative->DevicePreference() != AudioInputDevicePreference(aID)) { LOG(LogLevel::Debug, ("%p: %s-type non-native input device %p (track %p) is re-configured " "to %s-type", this, GetAudioInputTypeString(nonNative->DevicePreference()), aID, track, GetAudioInputTypeString(AudioInputDevicePreference(aID)))); needToSwitch = true; } if (needToSwitch) { nonNative->StopAudio(); nonNative->StartAudio(MakeRefPtr( MakeRefPtr(nonNative), nonNative->GenerateSourceId(), aID, AudioInputChannelCount(aID), AudioInputDevicePreference(aID) == AudioInputType::Voice, nonNative->mPrincipalHandle, nonNative->mSampleRate, GraphRate())); } return; } MOZ_ASSERT(track->AsNativeInputTrack()); if (AudioCallbackDriver* audioCallbackDriver = CurrentDriver()->AsAudioCallbackDriver()) { if (audioCallbackDriver->InputChannelCount() != AudioInputChannelCount(aID)) { LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: %u-channel AudioCallbackDriver %p is " "re-configured to %d-channel", this, audioCallbackDriver->InputChannelCount(), audioCallbackDriver, AudioInputChannelCount(aID))); needToSwitch = true; } if (audioCallbackDriver->InputDevicePreference() != AudioInputDevicePreference(aID)) { LOG(LogLevel::Debug, ("%p: ReevaluateInputDevice: %s-type AudioCallbackDriver %p is " "re-configured to %s-type", this, GetAudioInputTypeString( audioCallbackDriver->InputDevicePreference()), audioCallbackDriver, GetAudioInputTypeString(AudioInputDevicePreference(aID)))); needToSwitch = true; } } else if (Switching() && NextDriver()->AsAudioCallbackDriver()) { // We're already in the process of switching to a audio callback driver, // which will happen at the next iteration. // However, maybe it's not the correct number of channels. Re-query the // correct channel amount at this time. needToSwitch = true; } if (needToSwitch) { AudioCallbackDriver* newDriver = new AudioCallbackDriver( this, CurrentDriver(), mSampleRate, PrimaryOutputChannelCount(), AudioInputChannelCount(aID), PrimaryOutputDeviceID(), aID, AudioInputDevicePreference(aID), track->RequestedProcessingParams()); SwitchAtNextIteration(newDriver); } } bool MediaTrackGraphImpl::OnGraphThreadOrNotRunning() const { // either we're on the right thread (and calling CurrentDriver() is safe), // or we're going to fail the assert anyway, so don't cross-check // via CurrentDriver(). return mGraphDriverRunning ? OnGraphThread() : NS_IsMainThread(); } bool MediaTrackGraphImpl::OnGraphThread() const { // we're on the right thread (and calling mDriver is safe), MOZ_ASSERT(mDriver); if (mGraphRunner && mGraphRunner->OnThread()) { return true; } return mDriver->OnThread(); } bool MediaTrackGraphImpl::Destroyed() const { MOZ_ASSERT(NS_IsMainThread()); return !mSelfRef; } bool MediaTrackGraphImpl::ShouldUpdateMainThread() { MOZ_ASSERT(OnGraphThreadOrNotRunning()); if (mRealtime) { return true; } TimeStamp now = TimeStamp::Now(); // For offline graphs, update now if it has been long enough since the last // update, or if it has reached the end. if ((now - mLastMainThreadUpdate).ToMilliseconds() > CurrentDriver()->IterationDuration() || mStateComputedTime >= mEndTime) { mLastMainThreadUpdate = now; return true; } return false; } void MediaTrackGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) { MOZ_ASSERT(OnGraphThreadOrNotRunning()); mMonitor.AssertCurrentThreadOwns(); // We don't want to frequently update the main thread about timing update // when we are not running in realtime. if (aFinalUpdate || ShouldUpdateMainThread()) { // Strip updates that will be obsoleted below, so as to keep the length of // mTrackUpdates sane. size_t keptUpdateCount = 0; for (size_t i = 0; i < mTrackUpdates.Length(); ++i) { MediaTrack* track = mTrackUpdates[i].mTrack; // RemoveTrackGraphThread() clears mTrack in updates for // tracks that are removed from the graph. MOZ_ASSERT(!track || track->GraphImpl() == this); if (!track || track->MainThreadNeedsUpdates()) { // Discard this update as it has either been cleared when the track // was destroyed or there will be a newer update below. continue; } if (keptUpdateCount != i) { mTrackUpdates[keptUpdateCount] = std::move(mTrackUpdates[i]); MOZ_ASSERT(!mTrackUpdates[i].mTrack); } ++keptUpdateCount; } mTrackUpdates.TruncateLength(keptUpdateCount); mTrackUpdates.SetCapacity(mTrackUpdates.Length() + mTracks.Length() + mSuspendedTracks.Length()); for (MediaTrack* track : AllTracks()) { if (!track->MainThreadNeedsUpdates()) { continue; } TrackUpdate* update = mTrackUpdates.AppendElement(); update->mTrack = track; // No blocking to worry about here, since we've passed // UpdateCurrentTimeForTracks. update->mNextMainThreadCurrentTime = track->GraphTimeToTrackTime(mProcessedTime); update->mNextMainThreadEnded = track->mNotifiedEnded; } mNextMainThreadGraphTime = mProcessedTime; if (!mPendingUpdateRunnables.IsEmpty()) { mUpdateRunnables.AppendElements(std::move(mPendingUpdateRunnables)); } } // If this is the final update, then a stable state event will soon be // posted just before this thread finishes, and so there is no need to also // post here. if (!aFinalUpdate && // Don't send the message to the main thread if it's not going to have // any work to do. !(mUpdateRunnables.IsEmpty() && mTrackUpdates.IsEmpty())) { EnsureStableStateEventPosted(); } } GraphTime MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(GraphTime aTime) { if (aTime % WEBAUDIO_BLOCK_SIZE == 0) { return aTime; } return RoundUpToNextAudioBlock(aTime); } GraphTime MediaTrackGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime) { uint64_t block = aTime >> WEBAUDIO_BLOCK_SIZE_BITS; uint64_t nextBlock = block + 1; GraphTime nextTime = nextBlock << WEBAUDIO_BLOCK_SIZE_BITS; return nextTime; } void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock( uint32_t aTrackIndex, TrackRate aSampleRate) { MOZ_ASSERT(OnGraphThread()); MOZ_ASSERT(aTrackIndex <= mFirstCycleBreaker, "Cycle breaker is not AudioNodeTrack?"); while (mProcessedTime < mStateComputedTime) { // Microtask checkpoints are in between render quanta. nsAutoMicroTask mt; GraphTime next = RoundUpToNextAudioBlock(mProcessedTime); for (uint32_t i = mFirstCycleBreaker; i < mTracks.Length(); ++i) { auto nt = static_cast(mTracks[i]); MOZ_ASSERT(nt->AsAudioNodeTrack()); nt->ProduceOutputBeforeInput(mProcessedTime); } for (uint32_t i = aTrackIndex; i < mTracks.Length(); ++i) { ProcessedMediaTrack* pt = mTracks[i]->AsProcessedTrack(); if (pt) { pt->ProcessInput( mProcessedTime, next, (next == mStateComputedTime) ? ProcessedMediaTrack::ALLOW_END : 0); } } mProcessedTime = next; } NS_ASSERTION(mProcessedTime == mStateComputedTime, "Something went wrong with rounding to block boundaries"); } void MediaTrackGraphImpl::RunMessageAfterProcessing( UniquePtr aMessage) { MOZ_ASSERT(OnGraphThread()); if (mFrontMessageQueue.IsEmpty()) { mFrontMessageQueue.AppendElement(); } // Only one block is used for messages from the graph thread. MOZ_ASSERT(mFrontMessageQueue.Length() == 1); mFrontMessageQueue[0].mMessages.AppendElement(std::move(aMessage)); } void MediaTrackGraphImpl::RunMessagesInQueue() { TRACE("MTG::RunMessagesInQueue"); MOZ_ASSERT(OnGraphThread()); // Calculate independent action times for each batch of messages (each // batch corresponding to an event loop task). This isolates the performance // of different scripts to some extent. for (uint32_t i = 0; i < mFrontMessageQueue.Length(); ++i) { nsTArray>& messages = mFrontMessageQueue[i].mMessages; for (uint32_t j = 0; j < messages.Length(); ++j) { TRACE("ControlMessage::Run"); messages[j]->Run(); } } mFrontMessageQueue.Clear(); } void MediaTrackGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) { TRACE("MTG::UpdateGraph"); MOZ_ASSERT(OnGraphThread()); MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime); // The next state computed time can be the same as the previous: it // means the driver would have been blocking indefinitly, but the graph has // been woken up right after having been to sleep. MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime); CheckDriver(); UpdateTrackOrder(); // Always do another iteration if there are tracks waiting to resume. bool ensureNextIteration = !mPendingResumeOperations.IsEmpty(); for (MediaTrack* track : mTracks) { if (SourceMediaTrack* is = track->AsSourceTrack()) { ensureNextIteration |= is->PullNewData(aEndBlockingDecisions); is->ExtractPendingInput(mStateComputedTime, aEndBlockingDecisions); } if (track->mEnded) { // The track's not suspended, and since it's ended, underruns won't // stop it playing out. So there's no blocking other than what we impose // here. GraphTime endTime = track->GetEnd() + track->mStartTime; if (endTime <= mStateComputedTime) { LOG(LogLevel::Verbose, ("%p: MediaTrack %p is blocked due to being ended", this, track)); track->mStartBlocking = mStateComputedTime; } else { LOG(LogLevel::Verbose, ("%p: MediaTrack %p has ended, but is not blocked yet (current " "time %f, end at %f)", this, track, MediaTimeToSeconds(mStateComputedTime), MediaTimeToSeconds(endTime))); // Data can't be added to a ended track, so underruns are irrelevant. MOZ_ASSERT(endTime <= aEndBlockingDecisions); track->mStartBlocking = endTime; } } else { track->mStartBlocking = WillUnderrun(track, aEndBlockingDecisions); #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED if (SourceMediaTrack* s = track->AsSourceTrack()) { if (s->Ended()) { continue; } { MutexAutoLock lock(s->mMutex); if (!s->mUpdateTrack->mPullingEnabled) { // The invariant that data must be provided is only enforced when // pulling. continue; } } if (track->GetEnd() < track->GraphTimeToTrackTime(aEndBlockingDecisions)) { LOG(LogLevel::Error, ("%p: SourceMediaTrack %p (%s) is live and pulled, " "but wasn't fed " "enough data. TrackListeners=%zu. Track-end=%f, " "Iteration-end=%f", this, track, (track->mType == MediaSegment::AUDIO ? "audio" : "video"), track->mTrackListeners.Length(), MediaTimeToSeconds(track->GetEnd()), MediaTimeToSeconds( track->GraphTimeToTrackTime(aEndBlockingDecisions)))); MOZ_DIAGNOSTIC_ASSERT(false, "A non-ended SourceMediaTrack wasn't fed " "enough data by NotifyPull"); } } #endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */ } } for (MediaTrack* track : mSuspendedTracks) { track->mStartBlocking = mStateComputedTime; } // If the loop is woken up so soon that IterationEnd() barely advances or // if an offline graph is not currently rendering, we end up having // aEndBlockingDecisions == mStateComputedTime. // Since the process interval [mStateComputedTime, aEndBlockingDecision) is // empty, Process() will not find any unblocked track and so will not // ensure another iteration. If the graph should be rendering, then ensure // another iteration to render. if (ensureNextIteration || (aEndBlockingDecisions == mStateComputedTime && mStateComputedTime < mEndTime)) { EnsureNextIteration(); } } void MediaTrackGraphImpl::SelectOutputDeviceForAEC() { MOZ_ASSERT(OnGraphThread()); size_t currentDeviceIndex = mOutputDevices.IndexOf(mOutputDeviceForAEC); if (currentDeviceIndex == mOutputDevices.NoIndex) { // Outputs for this device have been removed. // Fall back to the primary output device. LOG(LogLevel::Info, ("%p: No remaining outputs to device %p. " "Switch to primary output device %p for AEC", this, mOutputDeviceForAEC, PrimaryOutputDeviceID())); mOutputDeviceForAEC = PrimaryOutputDeviceID(); currentDeviceIndex = 0; MOZ_ASSERT(mOutputDevices[0].mDeviceID == mOutputDeviceForAEC); } if (mOutputDevices.Length() == 1) { // No other output devices so there is no choice. return; } // The output is considered silent intentionally only if the whole duration // (often more than just this processing interval) of audio data in the // MediaSegment is null so as to reduce switching between output devices // should there be short durations of silence. auto HasNonNullAudio = [](const TrackAndVolume& aTV) { return aTV.mVolume != 0 && !aTV.mTrack->IsSuspended() && !aTV.mTrack->GetData()->IsNull(); }; // Keep using the same output device stream if it has non-null data, // so as to stay with a stream having ongoing audio. If the output stream // is switched, the echo cancellation algorithm can take some time to adjust // to the change in delay, so there is less value in switching back and // forth between output devices for very short sounds. for (const auto& output : mOutputDevices[currentDeviceIndex].mTrackOutputs) { if (HasNonNullAudio(output)) { return; } } // The current output device is silent. Use another if it has non-null data. for (const auto& outputDeviceEntry : mOutputDevices) { for (const auto& output : outputDeviceEntry.mTrackOutputs) { if (HasNonNullAudio(output)) { // Switch to this device. LOG(LogLevel::Info, ("%p: Switch output device for AEC from silent %p to non-null %p", this, mOutputDeviceForAEC, outputDeviceEntry.mDeviceID)); mOutputDeviceForAEC = outputDeviceEntry.mDeviceID; return; } } } // Null data for all outputs. Keep using the same device. } void MediaTrackGraphImpl::Process(MixerCallbackReceiver* aMixerReceiver) { TRACE("MTG::Process"); MOZ_ASSERT(OnGraphThread()); // Play track contents. bool allBlockedForever = true; // True when we've done ProcessInput for all processed tracks. bool doneAllProducing = false; const GraphTime oldProcessedTime = mProcessedTime; // Figure out what each track wants to do for (uint32_t i = 0; i < mTracks.Length(); ++i) { MediaTrack* track = mTracks[i]; if (!doneAllProducing) { ProcessedMediaTrack* pt = track->AsProcessedTrack(); if (pt) { AudioNodeTrack* n = track->AsAudioNodeTrack(); if (n) { #ifdef DEBUG // Verify that the sampling rate for all of the following tracks is // the same for (uint32_t j = i + 1; j < mTracks.Length(); ++j) { AudioNodeTrack* nextTrack = mTracks[j]->AsAudioNodeTrack(); if (nextTrack) { MOZ_ASSERT(n->mSampleRate == nextTrack->mSampleRate, "All AudioNodeTracks in the graph must have the same " "sampling rate"); } } #endif // Since an AudioNodeTrack is present, go ahead and // produce audio block by block for all the rest of the tracks. ProduceDataForTracksBlockByBlock(i, n->mSampleRate); doneAllProducing = true; } else { pt->ProcessInput(mProcessedTime, mStateComputedTime, ProcessedMediaTrack::ALLOW_END); // Assert that a live track produced enough data MOZ_ASSERT_IF(!track->mEnded, track->GetEnd() >= GraphTimeToTrackTimeWithBlocking( track, mStateComputedTime)); } } } if (track->mStartBlocking > oldProcessedTime) { allBlockedForever = false; } } mProcessedTime = mStateComputedTime; SelectOutputDeviceForAEC(); for (const auto& outputDeviceEntry : mOutputDevices) { uint32_t outputChannelCount; if (!outputDeviceEntry.mReceiver) { // primary output if (!aMixerReceiver) { // Running off a system clock driver. No need to mix output. continue; } MOZ_ASSERT(CurrentDriver()->AsAudioCallbackDriver(), "Driver must be AudioCallbackDriver if aMixerReceiver"); // Use the number of channel the driver expects: this is the number of // channel that can be output by the underlying system level audio stream. outputChannelCount = CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount(); } else { outputChannelCount = AudioOutputChannelCount(outputDeviceEntry); } MOZ_ASSERT(mRealtime, "If there's an output device, this graph must be realtime"); mMixer.StartMixing(); // This is the number of frames that are written to the output buffer, for // this iteration. TrackTime ticksPlayed = 0; for (const auto& t : outputDeviceEntry.mTrackOutputs) { TrackTime ticksPlayedForThisTrack = PlayAudio(t, oldProcessedTime, outputChannelCount); if (ticksPlayed == 0) { ticksPlayed = ticksPlayedForThisTrack; } else { MOZ_ASSERT( !ticksPlayedForThisTrack || ticksPlayedForThisTrack == ticksPlayed, "Each track should have the same number of frames."); } } if (ticksPlayed == 0) { // Nothing was played, so the mixer doesn't know how many frames were // processed. We still tell it so AudioCallbackDriver knows how much has // been processed. (bug 1406027) mMixer.Mix(nullptr, outputChannelCount, mStateComputedTime - oldProcessedTime, mSampleRate); } AudioChunk* outputChunk = mMixer.MixedChunk(); if (outputDeviceEntry.mDeviceID == mOutputDeviceForAEC) { // Callback any observers for the AEC speaker data. Note that one // (maybe) of these will be full-duplex, the others will get their input // data off separate cubeb callbacks. NotifyOutputData(*outputChunk); } if (!outputDeviceEntry.mReceiver) { // primary output aMixerReceiver->MixerCallback(outputChunk, mSampleRate); } else { outputDeviceEntry.mReceiver->EnqueueAudio(*outputChunk); } } if (!allBlockedForever) { EnsureNextIteration(); } } bool MediaTrackGraphImpl::UpdateMainThreadState() { MOZ_ASSERT(OnGraphThread()); if (mForceShutDownReceived) { for (MediaTrack* track : AllTracks()) { track->OnGraphThreadDone(); } } { MonitorAutoLock lock(mMonitor); bool finalUpdate = mForceShutDownReceived || (IsEmpty() && mBackMessageQueue.IsEmpty()); PrepareUpdatesToMainThreadState(finalUpdate); if (!finalUpdate) { SwapMessageQueues(); return true; } // The JSContext will not be used again. // Clear main thread access while under monitor. mJSContext = nullptr; } dom::WorkletThread::DeleteCycleCollectedJSContext(); // Enter shutdown mode when this iteration is completed. // No need to Destroy tracks here. The main-thread owner of each // track is responsible for calling Destroy on them. return false; } auto MediaTrackGraphImpl::OneIteration(GraphTime aStateTime, GraphTime aIterationEnd, MixerCallbackReceiver* aMixerReceiver) -> IterationResult { if (mGraphRunner) { return mGraphRunner->OneIteration(aStateTime, aIterationEnd, aMixerReceiver); } return OneIterationImpl(aStateTime, aIterationEnd, aMixerReceiver); } auto MediaTrackGraphImpl::OneIterationImpl( GraphTime aStateTime, GraphTime aIterationEnd, MixerCallbackReceiver* aMixerReceiver) -> IterationResult { TRACE("MTG::OneIterationImpl"); mIterationEndTime = aIterationEnd; if (SoftRealTimeLimitReached()) { TRACE("MTG::Demoting real-time thread!"); DemoteThreadFromRealTime(); } // Changes to LIFECYCLE_RUNNING occur before starting or reviving the graph // thread, and so the monitor need not be held to check mLifecycleState. // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline // graphs that have not started. // While changes occur on mainthread, this assert confirms that // this code shouldn't run if mainthread might be changing the state (to // > LIFECYCLE_RUNNING) // Ignore mutex warning: static during execution of the graph MOZ_PUSH_IGNORE_THREAD_SAFETY MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING); MOZ_POP_THREAD_SAFETY MOZ_ASSERT(OnGraphThread()); WebCore::DenormalDisabler disabler; // Process graph message from the main thread for this iteration. RunMessagesInQueue(); // Process MessagePort events. // These require a single thread, which has an nsThread with an event queue. if (mGraphRunner || !mRealtime) { TRACE("MTG::MessagePort events"); NS_ProcessPendingEvents(nullptr); } GraphTime stateTime = std::min(aStateTime, GraphTime(mEndTime)); UpdateGraph(stateTime); mStateComputedTime = stateTime; GraphTime oldProcessedTime = mProcessedTime; Process(aMixerReceiver); MOZ_ASSERT(mProcessedTime == stateTime); UpdateCurrentTimeForTracks(oldProcessedTime); ProcessChunkMetadata(oldProcessedTime); // Process graph messages queued from RunMessageAfterProcessing() on this // thread during the iteration. RunMessagesInQueue(); if (!UpdateMainThreadState()) { if (Switching()) { // We'll never get to do this switch. Clear mNextDriver to break the // ref-cycle graph->nextDriver->currentDriver->graph. SwitchAtNextIteration(nullptr); } return IterationResult::CreateStop( NewRunnableMethod("MediaTrackGraphImpl::SignalMainThreadCleanup", this, &MediaTrackGraphImpl::SignalMainThreadCleanup)); } if (Switching()) { RefPtr nextDriver = std::move(mNextDriver); return IterationResult::CreateSwitchDriver( nextDriver, NewRunnableMethod>( "MediaTrackGraphImpl::SetCurrentDriver", this, &MediaTrackGraphImpl::SetCurrentDriver, nextDriver)); } return IterationResult::CreateStillProcessing(); } void MediaTrackGraphImpl::ApplyTrackUpdate(TrackUpdate* aUpdate) { MOZ_ASSERT(NS_IsMainThread()); mMonitor.AssertCurrentThreadOwns(); MediaTrack* track = aUpdate->mTrack; if (!track) return; track->mMainThreadCurrentTime = aUpdate->mNextMainThreadCurrentTime; track->mMainThreadEnded = aUpdate->mNextMainThreadEnded; if (track->ShouldNotifyTrackEnded()) { track->NotifyMainThreadListeners(); } } void MediaTrackGraphImpl::ForceShutDown() { MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread"); LOG(LogLevel::Debug, ("%p: MediaTrackGraph::ForceShutdown", this)); if (mShutdownBlocker) { // Avoid waiting forever for a graph to shut down // synchronously. Reports are that some 3rd-party audio drivers // occasionally hang in shutdown (both for us and Chrome). NS_NewTimerWithCallback( getter_AddRefs(mShutdownTimer), this, MediaTrackGraph::AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT, nsITimer::TYPE_ONE_SHOT); } class Message final : public ControlMessage { public: explicit Message(MediaTrackGraphImpl* aGraph) : ControlMessage(nullptr), mGraph(aGraph) {} void Run() override { TRACE("MTG::ForceShutdown ControlMessage"); mGraph->mForceShutDownReceived = true; } // The graph owns this message. MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph; }; if (mMainThreadTrackCount > 0 || mMainThreadPortCount > 0) { // If both the track and port counts are zero, the regular shutdown // sequence will progress shortly to shutdown threads and destroy the graph. AppendMessage(MakeUnique(this)); InterruptJS(); } } NS_IMETHODIMP MediaTrackGraphImpl::Notify(nsITimer* aTimer) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(!mShutdownBlocker, "MediaTrackGraph took too long to shut down!"); // Sigh, graph took too long to shut down. Stop blocking system // shutdown and hope all is well. RemoveShutdownBlocker(); return NS_OK; } static nsCString GetDocumentTitle(uint64_t aWindowID) { MOZ_ASSERT(NS_IsMainThread()); nsCString title; auto* win = nsGlobalWindowInner::GetInnerWindowWithId(aWindowID); if (!win) { return title; } Document* doc = win->GetExtantDoc(); if (!doc) { return title; } nsAutoString titleUTF16; doc->GetTitle(titleUTF16); CopyUTF16toUTF8(titleUTF16, title); return title; } NS_IMETHODIMP MediaTrackGraphImpl::Observe(nsISupports* aSubject, const char* aTopic, const char16_t* aData) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(strcmp(aTopic, "document-title-changed") == 0); nsCString streamName = GetDocumentTitle(mWindowID); LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get())); if (streamName.IsEmpty()) { return NS_OK; } QueueControlMessageWithNoShutdown( [self = RefPtr{this}, this, streamName = std::move(streamName)] { CurrentDriver()->SetStreamName(streamName); }); return NS_OK; } bool MediaTrackGraphImpl::AddShutdownBlocker() { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(!mShutdownBlocker); class Blocker : public media::ShutdownBlocker { const RefPtr mGraph; public: Blocker(MediaTrackGraphImpl* aGraph, const nsString& aName) : media::ShutdownBlocker(aName), mGraph(aGraph) {} NS_IMETHOD BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override { mGraph->ForceShutDown(); return NS_OK; } }; nsCOMPtr barrier = media::GetShutdownBarrier(); if (!barrier) { // We're already shutting down, we won't be able to add a blocker, bail. LOG(LogLevel::Error, ("%p: Couldn't get shutdown barrier, won't add shutdown blocker", this)); return false; } // Blocker names must be distinct. nsString blockerName; blockerName.AppendPrintf("MediaTrackGraph %p shutdown", this); mShutdownBlocker = MakeAndAddRef(this, blockerName); nsresult rv = barrier->AddBlocker(mShutdownBlocker, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__, u"MediaTrackGraph shutdown"_ns); MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); return true; } void MediaTrackGraphImpl::RemoveShutdownBlocker() { if (!mShutdownBlocker) { return; } media::MustGetShutdownBarrier()->RemoveBlocker(mShutdownBlocker); mShutdownBlocker = nullptr; } NS_IMETHODIMP MediaTrackGraphImpl::GetName(nsACString& aName) { aName.AssignLiteral("MediaTrackGraphImpl"); return NS_OK; } namespace { class MediaTrackGraphShutDownRunnable : public Runnable { public: explicit MediaTrackGraphShutDownRunnable(MediaTrackGraphImpl* aGraph) : Runnable("MediaTrackGraphShutDownRunnable"), mGraph(aGraph) {} // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. // See bug 1535398. MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override { TRACE("MTG::MediaTrackGraphShutDownRunnable runnable"); MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(!mGraph->mGraphDriverRunning && mGraph->mDriver, "We should know the graph thread control loop isn't running!"); LOG(LogLevel::Debug, ("%p: Shutting down graph", mGraph.get())); // We've asserted the graph isn't running. Use mDriver instead of // CurrentDriver to avoid thread-safety checks #if 0 // AudioCallbackDrivers are released asynchronously anyways // XXX a better test would be have setting mGraphDriverRunning make sure // any current callback has finished and block future ones -- or just // handle it all in Shutdown()! if (mGraph->mDriver->AsAudioCallbackDriver()) { MOZ_ASSERT(!mGraph->mDriver->AsAudioCallbackDriver()->InCallback()); } #endif for (MediaTrackGraphImpl::PendingResumeOperation& op : mGraph->mPendingResumeOperations) { op.Abort(); } if (mGraph->mGraphRunner) { RefPtr(mGraph->mGraphRunner)->Shutdown(); } RefPtr(mGraph->mDriver)->Shutdown(); // Release the driver now so that an AudioCallbackDriver will release its // SharedThreadPool reference. Each SharedThreadPool reference must be // released before SharedThreadPool::SpinUntilEmpty() runs on // xpcom-shutdown-threads. Don't wait for GC/CC to release references to // objects owning tracks, or for expiration of mGraph->mShutdownTimer, // which won't otherwise release its reference on the graph until // nsTimerImpl::Shutdown(), which runs after xpcom-shutdown-threads. mGraph->SetCurrentDriver(nullptr); // Safe to access these without the monitor since the graph isn't running. // We may be one of several graphs. Drop ticket to eventually unblock // shutdown. if (mGraph->mShutdownTimer && !mGraph->mShutdownBlocker) { MOZ_ASSERT( false, "AudioCallbackDriver took too long to shut down and we let shutdown" " continue - freezing and leaking"); // The timer fired, so we may be deeper in shutdown now. Block any // further teardown and just leak, for safety. return NS_OK; } // mGraph's thread is not running so it's OK to do whatever here for (MediaTrack* track : mGraph->AllTracks()) { // Clean up all MediaSegments since we cannot release Images too // late during shutdown. Also notify listeners that they were removed // so they can clean up any gfx resources. track->RemoveAllResourcesAndListenersImpl(); } #ifdef DEBUG { MonitorAutoLock lock(mGraph->mMonitor); MOZ_ASSERT(mGraph->mUpdateRunnables.IsEmpty()); } #endif mGraph->mPendingUpdateRunnables.Clear(); mGraph->RemoveShutdownBlocker(); // We can't block past the final LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION // stage, since completion of that stage requires all tracks to be freed, // which requires shutdown to proceed. if (mGraph->IsEmpty()) { // mGraph is no longer needed, so delete it. mGraph->Destroy(); } else { // The graph is not empty. We must be in a forced shutdown. // Some later AppendMessage will detect that the graph has // been emptied, and delete it. NS_ASSERTION(mGraph->mForceShutDownReceived, "Not in forced shutdown?"); mGraph->LifecycleStateRef() = MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION; } return NS_OK; } private: RefPtr mGraph; }; class MediaTrackGraphStableStateRunnable : public Runnable { public: explicit MediaTrackGraphStableStateRunnable(MediaTrackGraphImpl* aGraph, bool aSourceIsMTG) : Runnable("MediaTrackGraphStableStateRunnable"), mGraph(aGraph), mSourceIsMTG(aSourceIsMTG) {} NS_IMETHOD Run() override { TRACE("MTG::MediaTrackGraphStableStateRunnable ControlMessage"); if (mGraph) { mGraph->RunInStableState(mSourceIsMTG); } return NS_OK; } private: RefPtr mGraph; bool mSourceIsMTG; }; /* * Control messages forwarded from main thread to graph manager thread */ class CreateMessage : public ControlMessage { public: explicit CreateMessage(MediaTrack* aTrack) : ControlMessage(aTrack) {} void Run() override { TRACE("MTG::AddTrackGraphThread ControlMessage"); mTrack->GraphImpl()->AddTrackGraphThread(mTrack); } void RunDuringShutdown() override { // Make sure to run this message during shutdown too, to make sure // that we balance the number of tracks registered with the graph // as they're destroyed during shutdown. Run(); } }; } // namespace void MediaTrackGraphImpl::RunInStableState(bool aSourceIsMTG) { MOZ_ASSERT(NS_IsMainThread(), "Must be called on main thread"); nsTArray> runnables; // When we're doing a forced shutdown, pending control messages may be // run on the main thread via RunDuringShutdown. Those messages must // run without the graph monitor being held. So, we collect them here. nsTArray> controlMessagesToRunDuringShutdown; { MonitorAutoLock lock(mMonitor); if (aSourceIsMTG) { MOZ_ASSERT(mPostedRunInStableStateEvent); mPostedRunInStableStateEvent = false; } // This should be kept in sync with the LifecycleState enum in // MediaTrackGraphImpl.h const char* LifecycleState_str[] = { "LIFECYCLE_THREAD_NOT_STARTED", "LIFECYCLE_RUNNING", "LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP", "LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN", "LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION"}; if (LifecycleStateRef() != LIFECYCLE_RUNNING) { LOG(LogLevel::Debug, ("%p: Running stable state callback. Current state: %s", this, LifecycleState_str[LifecycleStateRef()])); } runnables = std::move(mUpdateRunnables); for (uint32_t i = 0; i < mTrackUpdates.Length(); ++i) { TrackUpdate* update = &mTrackUpdates[i]; if (update->mTrack) { ApplyTrackUpdate(update); } } mTrackUpdates.Clear(); mMainThreadGraphTime = mNextMainThreadGraphTime; if (mCurrentTaskMessageQueue.IsEmpty()) { if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP && IsEmpty()) { // Complete shutdown. First, ensure that this graph is no longer used. // A new graph graph will be created if one is needed. // Asynchronously clean up old graph. We don't want to do this // synchronously because it spins the event loop waiting for threads // to shut down, and we don't want to do that in a stable state handler. LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN; LOG(LogLevel::Debug, ("%p: Sending MediaTrackGraphShutDownRunnable", this)); nsCOMPtr event = new MediaTrackGraphShutDownRunnable(this); mMainThread->Dispatch(event.forget()); } } else { if (LifecycleStateRef() <= LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) { MessageBlock* block = mBackMessageQueue.AppendElement(); block->mMessages = std::move(mCurrentTaskMessageQueue); EnsureNextIteration(); } // If this MediaTrackGraph has entered regular (non-forced) shutdown it // is not able to process any more messages. Those messages being added to // the graph in the first place is an error. MOZ_DIAGNOSTIC_ASSERT(LifecycleStateRef() < LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP || mForceShutDownReceived); } if (LifecycleStateRef() == LIFECYCLE_THREAD_NOT_STARTED) { // Start the driver now. We couldn't start it earlier because the graph // might exit immediately on finding it has no tracks. The first message // for a new graph must create a track. Ensure that his message runs on // the first iteration. MOZ_ASSERT(MessagesQueued()); SwapMessageQueues(); LOG(LogLevel::Debug, ("%p: Starting a graph with a %s", this, CurrentDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver")); LifecycleStateRef() = LIFECYCLE_RUNNING; mGraphDriverRunning = true; RefPtr driver = CurrentDriver(); driver->Start(); // It's not safe to Shutdown() a thread from StableState, and // releasing this may shutdown a SystemClockDriver thread. // Proxy the release to outside of StableState. NS_ReleaseOnMainThread("MediaTrackGraphImpl::CurrentDriver", driver.forget(), true); // always proxy } if (LifecycleStateRef() == LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP && mForceShutDownReceived) { // Defer calls to RunDuringShutdown() to happen while mMonitor is not // held. for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) { MessageBlock& mb = mBackMessageQueue[i]; controlMessagesToRunDuringShutdown.AppendElements( std::move(mb.mMessages)); } mBackMessageQueue.Clear(); MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty()); // Stop MediaTrackGraph threads. LifecycleStateRef() = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN; nsCOMPtr event = new MediaTrackGraphShutDownRunnable(this); mMainThread->Dispatch(event.forget()); } mGraphDriverRunning = LifecycleStateRef() == LIFECYCLE_RUNNING; } // Make sure we get a new current time in the next event loop task if (!aSourceIsMTG) { MOZ_ASSERT(mPostedRunInStableState); mPostedRunInStableState = false; } for (uint32_t i = 0; i < controlMessagesToRunDuringShutdown.Length(); ++i) { controlMessagesToRunDuringShutdown[i]->RunDuringShutdown(); } #ifdef DEBUG mCanRunMessagesSynchronously = !mGraphDriverRunning && LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN; #endif for (uint32_t i = 0; i < runnables.Length(); ++i) { runnables[i]->Run(); } } void MediaTrackGraphImpl::EnsureRunInStableState() { MOZ_ASSERT(NS_IsMainThread(), "main thread only"); if (mPostedRunInStableState) return; mPostedRunInStableState = true; nsCOMPtr event = new MediaTrackGraphStableStateRunnable(this, false); nsContentUtils::RunInStableState(event.forget()); } void MediaTrackGraphImpl::EnsureStableStateEventPosted() { MOZ_ASSERT(OnGraphThread()); mMonitor.AssertCurrentThreadOwns(); if (mPostedRunInStableStateEvent) return; mPostedRunInStableStateEvent = true; nsCOMPtr event = new MediaTrackGraphStableStateRunnable(this, true); mMainThread->Dispatch(event.forget()); } void MediaTrackGraphImpl::SignalMainThreadCleanup() { MOZ_ASSERT(mDriver->OnThread()); MonitorAutoLock lock(mMonitor); // LIFECYCLE_THREAD_NOT_STARTED is possible when shutting down offline // graphs that have not started. MOZ_DIAGNOSTIC_ASSERT(mLifecycleState <= LIFECYCLE_RUNNING); LOG(LogLevel::Debug, ("%p: MediaTrackGraph waiting for main thread cleanup", this)); LifecycleStateRef() = MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP; EnsureStableStateEventPosted(); } void MediaTrackGraphImpl::AppendMessage( UniquePtr aMessage) { MOZ_ASSERT(NS_IsMainThread(), "main thread only"); MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0 || mMainThreadPortCount > 0); if (!mGraphDriverRunning && LifecycleStateRef() > LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP) { // The graph control loop is not running and main thread cleanup has // happened. From now on we can't append messages to // mCurrentTaskMessageQueue, because that will never be processed again, so // just RunDuringShutdown this message. This should only happen during // forced shutdown, or after a non-realtime graph has finished processing. #ifdef DEBUG MOZ_ASSERT(mCanRunMessagesSynchronously); mCanRunMessagesSynchronously = false; #endif aMessage->RunDuringShutdown(); #ifdef DEBUG mCanRunMessagesSynchronously = true; #endif if (IsEmpty() && LifecycleStateRef() >= LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION) { Destroy(); } return; } mCurrentTaskMessageQueue.AppendElement(std::move(aMessage)); EnsureRunInStableState(); } void MediaTrackGraphImpl::Dispatch(already_AddRefed&& aRunnable) { mMainThread->Dispatch(std::move(aRunnable)); } MediaTrack::MediaTrack(TrackRate aSampleRate, MediaSegment::Type aType, MediaSegment* aSegment) : mSampleRate(aSampleRate), mType(aType), mSegment(aSegment), mStartTime(0), mForgottenTime(0), mEnded(false), mNotifiedEnded(false), mDisabledMode(DisabledTrackMode::ENABLED), mStartBlocking(GRAPH_TIME_MAX), mSuspendedCount(0), mMainThreadCurrentTime(0), mMainThreadEnded(false), mEndedNotificationSent(false), mMainThreadDestroyed(false), mGraph(nullptr) { MOZ_COUNT_CTOR(MediaTrack); MOZ_ASSERT_IF(mSegment, mSegment->GetType() == aType); } MediaTrack::~MediaTrack() { MOZ_COUNT_DTOR(MediaTrack); NS_ASSERTION(mMainThreadDestroyed, "Should have been destroyed already"); NS_ASSERTION(mMainThreadListeners.IsEmpty(), "All main thread listeners should have been removed"); } size_t MediaTrack::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { size_t amount = 0; // Not owned: // - mGraph - Not reported here // - mConsumers - elements // Future: // - mLastPlayedVideoFrame // - mTrackListeners - elements amount += mTrackListeners.ShallowSizeOfExcludingThis(aMallocSizeOf); amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf); amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf); return amount; } size_t MediaTrack::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } void MediaTrack::IncrementSuspendCount() { ++mSuspendedCount; if (mSuspendedCount != 1 || !mGraph) { MOZ_ASSERT(mGraph || mConsumers.IsEmpty()); return; } AssertOnGraphThreadOrNotRunning(); auto* graph = GraphImpl(); for (uint32_t i = 0; i < mConsumers.Length(); ++i) { mConsumers[i]->Suspended(); } MOZ_ASSERT(graph->mTracks.Contains(this)); graph->mTracks.RemoveElement(this); graph->mSuspendedTracks.AppendElement(this); graph->SetTrackOrderDirty(); } void MediaTrack::DecrementSuspendCount() { MOZ_ASSERT(mSuspendedCount > 0, "Suspend count underrun"); --mSuspendedCount; if (mSuspendedCount != 0 || !mGraph) { MOZ_ASSERT(mGraph || mConsumers.IsEmpty()); return; } AssertOnGraphThreadOrNotRunning(); auto* graph = GraphImpl(); for (uint32_t i = 0; i < mConsumers.Length(); ++i) { mConsumers[i]->Resumed(); } MOZ_ASSERT(graph->mSuspendedTracks.Contains(this)); graph->mSuspendedTracks.RemoveElement(this); graph->mTracks.AppendElement(this); graph->SetTrackOrderDirty(); } void ProcessedMediaTrack::DecrementSuspendCount() { mCycleMarker = NOT_VISITED; MediaTrack::DecrementSuspendCount(); } MediaTrackGraphImpl* MediaTrack::GraphImpl() { return static_cast(mGraph); } const MediaTrackGraphImpl* MediaTrack::GraphImpl() const { return static_cast(mGraph); } void MediaTrack::SetGraphImpl(MediaTrackGraphImpl* aGraph) { MOZ_ASSERT(!mGraph, "Should only be called once"); MOZ_ASSERT(mSampleRate == aGraph->GraphRate()); mGraph = aGraph; } void MediaTrack::SetGraphImpl(MediaTrackGraph* aGraph) { MediaTrackGraphImpl* graph = static_cast(aGraph); SetGraphImpl(graph); } TrackTime MediaTrack::GraphTimeToTrackTime(GraphTime aTime) const { NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime || aTime <= mStartBlocking, "Incorrectly ignoring blocking!"); return aTime - mStartTime; } GraphTime MediaTrack::TrackTimeToGraphTime(TrackTime aTime) const { NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime || aTime + mStartTime <= mStartBlocking, "Incorrectly ignoring blocking!"); return aTime + mStartTime; } TrackTime MediaTrack::GraphTimeToTrackTimeWithBlocking(GraphTime aTime) const { return GraphImpl()->GraphTimeToTrackTimeWithBlocking(this, aTime); } void MediaTrack::RemoveAllResourcesAndListenersImpl() { GraphImpl()->AssertOnGraphThreadOrNotRunning(); for (auto& l : mTrackListeners.Clone()) { l->NotifyRemoved(Graph()); } mTrackListeners.Clear(); RemoveAllDirectListenersImpl(); if (mSegment) { mSegment->Clear(); } } void MediaTrack::DestroyImpl() { for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) { mConsumers[i]->Disconnect(); } if (mSegment) { mSegment->Clear(); } mGraph = nullptr; } void MediaTrack::Destroy() { // Keep this track alive until we leave this method RefPtr kungFuDeathGrip = this; // Keep a reference to the graph, since Message might RunDuringShutdown() // synchronously and make GraphImpl() invalid. RefPtr graph = GraphImpl(); QueueControlOrShutdownMessage( [self = RefPtr{this}, this](IsInShutdown aInShutdown) { if (aInShutdown == IsInShutdown::No) { OnGraphThreadDone(); } TRACE("MediaTrack::Destroy ControlMessage"); RemoveAllResourcesAndListenersImpl(); auto* graph = GraphImpl(); DestroyImpl(); graph->RemoveTrackGraphThread(this); }); graph->RemoveTrack(this); // Message::RunDuringShutdown may have removed this track from the graph, // but our kungFuDeathGrip above will have kept this track alive if // necessary. mMainThreadDestroyed = true; } TrackTime MediaTrack::GetEnd() const { return mSegment ? mSegment->GetDuration() : 0; } void MediaTrack::AddAudioOutput(void* aKey, const AudioDeviceInfo* aSink) { MOZ_ASSERT(NS_IsMainThread()); AudioDeviceID deviceID = nullptr; TrackRate preferredSampleRate = 0; if (aSink) { deviceID = aSink->DeviceID(); preferredSampleRate = static_cast(aSink->DefaultRate()); } AddAudioOutput(aKey, deviceID, preferredSampleRate); } void MediaTrack::AddAudioOutput(void* aKey, CubebUtils::AudioDeviceID aDeviceID, TrackRate aPreferredSampleRate) { MOZ_ASSERT(NS_IsMainThread()); if (mMainThreadDestroyed) { return; } LOG(LogLevel::Info, ("MediaTrack %p adding AudioOutput", this)); GraphImpl()->RegisterAudioOutput(this, aKey, aDeviceID, aPreferredSampleRate); } void MediaTrackGraphImpl::SetAudioOutputVolume(MediaTrack* aTrack, void* aKey, float aVolume) { MOZ_ASSERT(NS_IsMainThread()); for (auto& params : mAudioOutputParams) { if (params.mKey == aKey && aTrack == params.mTrack) { params.mVolume = aVolume; UpdateAudioOutput(aTrack, params.mDeviceID); return; } } MOZ_CRASH("Audio output key not found when setting the volume."); } void MediaTrack::SetAudioOutputVolume(void* aKey, float aVolume) { if (mMainThreadDestroyed) { return; } GraphImpl()->SetAudioOutputVolume(this, aKey, aVolume); } void MediaTrack::RemoveAudioOutput(void* aKey) { MOZ_ASSERT(NS_IsMainThread()); if (mMainThreadDestroyed) { return; } LOG(LogLevel::Info, ("MediaTrack %p removing AudioOutput", this)); GraphImpl()->UnregisterAudioOutput(this, aKey); } void MediaTrackGraphImpl::RegisterAudioOutput( MediaTrack* aTrack, void* aKey, CubebUtils::AudioDeviceID aDeviceID, TrackRate aPreferredSampleRate) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(!mAudioOutputParams.Contains(TrackAndKey{aTrack, aKey})); IncrementOutputDeviceRefCnt(aDeviceID, aPreferredSampleRate); mAudioOutputParams.EmplaceBack( TrackKeyDeviceAndVolume{aTrack, aKey, aDeviceID, 1.f}); UpdateAudioOutput(aTrack, aDeviceID); } void MediaTrackGraphImpl::UnregisterAudioOutput(MediaTrack* aTrack, void* aKey) { MOZ_ASSERT(NS_IsMainThread()); size_t index = mAudioOutputParams.IndexOf(TrackAndKey{aTrack, aKey}); MOZ_ASSERT(index != mAudioOutputParams.NoIndex); AudioDeviceID deviceID = mAudioOutputParams[index].mDeviceID; mAudioOutputParams.UnorderedRemoveElementAt(index); UpdateAudioOutput(aTrack, deviceID); DecrementOutputDeviceRefCnt(deviceID); } void MediaTrackGraphImpl::UpdateAudioOutput(MediaTrack* aTrack, AudioDeviceID aDeviceID) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(!aTrack->IsDestroyed()); float volume = 0.f; bool found = false; for (const auto& params : mAudioOutputParams) { if (params.mTrack == aTrack && params.mDeviceID == aDeviceID) { volume += params.mVolume; found = true; } } QueueControlMessageWithNoShutdown( // track has a strong reference to this. [track = RefPtr{aTrack}, aDeviceID, volume, found] { TRACE("MediaTrack::UpdateAudioOutput ControlMessage"); MediaTrackGraphImpl* graph = track->GraphImpl(); auto& outputDevicesRef = graph->mOutputDevices; size_t deviceIndex = outputDevicesRef.IndexOf(aDeviceID); MOZ_ASSERT(deviceIndex != outputDevicesRef.NoIndex); auto& deviceOutputsRef = outputDevicesRef[deviceIndex].mTrackOutputs; if (found) { for (auto& outputRef : deviceOutputsRef) { if (outputRef.mTrack == track) { outputRef.mVolume = volume; return; } } deviceOutputsRef.EmplaceBack(TrackAndVolume{track, volume}); } else { DebugOnly removed = deviceOutputsRef.RemoveElement(track); MOZ_ASSERT(removed); // mOutputDevices[0] is retained for AudioCallbackDriver output even // when no tracks have audio outputs. if (deviceIndex != 0 && deviceOutputsRef.IsEmpty()) { // The device is no longer in use. outputDevicesRef.UnorderedRemoveElementAt(deviceIndex); } } }); } void MediaTrackGraphImpl::IncrementOutputDeviceRefCnt( AudioDeviceID aDeviceID, TrackRate aPreferredSampleRate) { MOZ_ASSERT(NS_IsMainThread()); for (auto& elementRef : mOutputDeviceRefCnts) { if (elementRef.mDeviceID == aDeviceID) { ++elementRef.mRefCnt; return; } } MOZ_ASSERT(aDeviceID != mPrimaryOutputDeviceID, "mOutputDeviceRefCnts should always have the primary device"); // Need to add an output device. // Output via another graph for this device. // This sample rate is not exposed to content. TrackRate sampleRate = aPreferredSampleRate != 0 ? aPreferredSampleRate : static_cast(CubebUtils::PreferredSampleRate( /*aShouldResistFingerprinting*/ false)); MediaTrackGraph* newGraph = MediaTrackGraphImpl::GetInstance( MediaTrackGraph::AUDIO_THREAD_DRIVER, mWindowID, sampleRate, aDeviceID, GetMainThreadSerialEventTarget()); // CreateCrossGraphReceiver wants the sample rate of this graph. RefPtr receiver = newGraph->CreateCrossGraphReceiver(mSampleRate); receiver->AddAudioOutput(nullptr, aDeviceID, sampleRate); mOutputDeviceRefCnts.EmplaceBack( DeviceReceiverAndCount{aDeviceID, receiver, 1}); QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aDeviceID, receiver = std::move(receiver)]() mutable { TRACE("MediaTrackGraph add output device ControlMessage"); MOZ_ASSERT(!mOutputDevices.Contains(aDeviceID)); mOutputDevices.EmplaceBack( OutputDeviceEntry{aDeviceID, std::move(receiver)}); }); } void MediaTrackGraphImpl::DecrementOutputDeviceRefCnt(AudioDeviceID aDeviceID) { MOZ_ASSERT(NS_IsMainThread()); size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID); MOZ_ASSERT(index != mOutputDeviceRefCnts.NoIndex); // mOutputDeviceRefCnts[0] is retained for consistency with // mOutputDevices[0], which is retained for AudioCallbackDriver output even // when no tracks have audio outputs. if (--mOutputDeviceRefCnts[index].mRefCnt == 0 && index != 0) { mOutputDeviceRefCnts[index].mReceiver->Destroy(); mOutputDeviceRefCnts.UnorderedRemoveElementAt(index); } } void MediaTrack::Suspend() { // This can happen if this method has been called asynchronously, and the // track has been destroyed since then. if (mMainThreadDestroyed) { return; } QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] { TRACE("MediaTrack::IncrementSuspendCount ControlMessage"); IncrementSuspendCount(); }); } void MediaTrack::Resume() { // This can happen if this method has been called asynchronously, and the // track has been destroyed since then. if (mMainThreadDestroyed) { return; } QueueControlMessageWithNoShutdown([self = RefPtr{this}, this] { TRACE("MediaTrack::DecrementSuspendCount ControlMessage"); DecrementSuspendCount(); }); } void MediaTrack::AddListenerImpl( already_AddRefed aListener) { RefPtr l(aListener); mTrackListeners.AppendElement(std::move(l)); PrincipalHandle lastPrincipalHandle = mSegment->GetLastPrincipalHandle(); mTrackListeners.LastElement()->NotifyPrincipalHandleChanged( Graph(), lastPrincipalHandle); if (mNotifiedEnded) { mTrackListeners.LastElement()->NotifyEnded(Graph()); } if (CombinedDisabledMode() == DisabledTrackMode::SILENCE_BLACK) { mTrackListeners.LastElement()->NotifyEnabledStateChanged(Graph(), false); } } void MediaTrack::AddListener(MediaTrackListener* aListener) { MOZ_ASSERT(mSegment, "Segment-less tracks do not support listeners"); if (mMainThreadDestroyed) { return; } QueueControlMessageWithNoShutdown( [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable { TRACE("MediaTrack::AddListenerImpl ControlMessage"); AddListenerImpl(listener.forget()); }); } void MediaTrack::RemoveListenerImpl(MediaTrackListener* aListener) { for (size_t i = 0; i < mTrackListeners.Length(); ++i) { if (mTrackListeners[i] == aListener) { mTrackListeners[i]->NotifyRemoved(Graph()); mTrackListeners.RemoveElementAt(i); return; } } } RefPtr MediaTrack::RemoveListener( MediaTrackListener* aListener) { MozPromiseHolder promiseHolder; RefPtr p = promiseHolder.Ensure(__func__); if (mMainThreadDestroyed) { promiseHolder.Reject(NS_ERROR_FAILURE, __func__); return p; } QueueControlOrShutdownMessage( [self = RefPtr{this}, this, listener = RefPtr{aListener}, promiseHolder = std::move(promiseHolder)](IsInShutdown) mutable { TRACE("MediaTrack::RemoveListenerImpl ControlMessage"); // During shutdown we still want the listener's NotifyRemoved to be // called, since not doing that might block shutdown of other modules. RemoveListenerImpl(listener); promiseHolder.Resolve(true, __func__); }); return p; } void MediaTrack::AddDirectListenerImpl( already_AddRefed aListener) { AssertOnGraphThread(); // Base implementation, for tracks that don't support direct track listeners. RefPtr listener = aListener; listener->NotifyDirectListenerInstalled( DirectMediaTrackListener::InstallationResult::TRACK_NOT_SUPPORTED); } void MediaTrack::AddDirectListener(DirectMediaTrackListener* aListener) { if (mMainThreadDestroyed) { return; } QueueControlMessageWithNoShutdown( [self = RefPtr{this}, this, listener = RefPtr{aListener}]() mutable { TRACE("MediaTrack::AddDirectListenerImpl ControlMessage"); AddDirectListenerImpl(listener.forget()); }); } void MediaTrack::RemoveDirectListenerImpl(DirectMediaTrackListener* aListener) { // Base implementation, the listener was never added so nothing to do. } void MediaTrack::RemoveDirectListener(DirectMediaTrackListener* aListener) { if (mMainThreadDestroyed) { return; } QueueControlOrShutdownMessage( [self = RefPtr{this}, this, listener = RefPtr{aListener}](IsInShutdown) { TRACE("MediaTrack::RemoveDirectListenerImpl ControlMessage"); // During shutdown we still want the listener's // NotifyDirectListenerUninstalled to be called, since not doing that // might block shutdown of other modules. RemoveDirectListenerImpl(listener); }); } void MediaTrack::RunAfterPendingUpdates( already_AddRefed aRunnable) { MOZ_ASSERT(NS_IsMainThread()); if (mMainThreadDestroyed) { return; } QueueControlOrShutdownMessage( [self = RefPtr{this}, this, runnable = nsCOMPtr{aRunnable}](IsInShutdown aInShutdown) mutable { TRACE("MediaTrack::DispatchToMainThreadStableState ControlMessage"); if (aInShutdown == IsInShutdown::No) { Graph()->DispatchToMainThreadStableState(runnable.forget()); } else { // Don't run mRunnable now as it may call AppendMessage() which would // assume that there are no remaining // controlMessagesToRunDuringShutdown. MOZ_ASSERT(NS_IsMainThread()); GraphImpl()->Dispatch(runnable.forget()); } }); } void MediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) { AssertOnGraphThread(); MOZ_DIAGNOSTIC_ASSERT( aMode == DisabledTrackMode::ENABLED || mDisabledMode == DisabledTrackMode::ENABLED, "Changing disabled track mode for a track is not allowed"); DisabledTrackMode oldMode = CombinedDisabledMode(); mDisabledMode = aMode; NotifyIfDisabledModeChangedFrom(oldMode); } void MediaTrack::SetDisabledTrackMode(DisabledTrackMode aMode) { if (mMainThreadDestroyed) { return; } QueueControlMessageWithNoShutdown([self = RefPtr{this}, this, aMode]() { TRACE("MediaTrack::SetDisabledTrackModeImpl ControlMessage"); SetDisabledTrackModeImpl(aMode); }); } void MediaTrack::ApplyTrackDisabling(MediaSegment* aSegment, MediaSegment* aRawSegment) { AssertOnGraphThread(); mozilla::ApplyTrackDisabling(mDisabledMode, aSegment, aRawSegment); } void MediaTrack::AddMainThreadListener( MainThreadMediaTrackListener* aListener) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aListener); MOZ_ASSERT(!mMainThreadListeners.Contains(aListener)); mMainThreadListeners.AppendElement(aListener); // If it is not yet time to send the notification, then exit here. if (!mEndedNotificationSent) { return; } class NotifyRunnable final : public Runnable { public: explicit NotifyRunnable(MediaTrack* aTrack) : Runnable("MediaTrack::NotifyRunnable"), mTrack(aTrack) {} NS_IMETHOD Run() override { TRACE("MediaTrack::NotifyMainThreadListeners Runnable"); MOZ_ASSERT(NS_IsMainThread()); mTrack->NotifyMainThreadListeners(); return NS_OK; } private: ~NotifyRunnable() = default; RefPtr mTrack; }; nsCOMPtr runnable = new NotifyRunnable(this); GraphImpl()->Dispatch(runnable.forget()); } void MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime) { mStartTime += aBlockedTime; if (!mSegment) { // No data to be forgotten. return; } TrackTime time = aCurrentTime - mStartTime; // Only prune if there is a reasonable chunk (50ms) to forget, so we don't // spend too much time pruning segments. const TrackTime minChunkSize = mSampleRate * 50 / 1000; if (time < mForgottenTime + minChunkSize) { return; } mForgottenTime = std::min(GetEnd() - 1, time); mSegment->ForgetUpTo(mForgottenTime); } void MediaTrack::NotifyIfDisabledModeChangedFrom(DisabledTrackMode aOldMode) { DisabledTrackMode mode = CombinedDisabledMode(); if (aOldMode == mode) { return; } for (const auto& listener : mTrackListeners) { listener->NotifyEnabledStateChanged( Graph(), mode != DisabledTrackMode::SILENCE_BLACK); } for (const auto& c : mConsumers) { if (c->GetDestination()) { c->GetDestination()->OnInputDisabledModeChanged(mode); } } } void MediaTrack::QueueMessage(UniquePtr aMessage) { MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); MOZ_RELEASE_ASSERT(!IsDestroyed()); GraphImpl()->AppendMessage(std::move(aMessage)); } void MediaTrack::RunMessageAfterProcessing( UniquePtr aMessage) { AssertOnGraphThread(); GraphImpl()->RunMessageAfterProcessing(std::move(aMessage)); } SourceMediaTrack::SourceMediaTrack(MediaSegment::Type aType, TrackRate aSampleRate) : MediaTrack(aSampleRate, aType, aType == MediaSegment::AUDIO ? static_cast(new AudioSegment()) : static_cast(new VideoSegment())), mMutex("mozilla::media::SourceMediaTrack") { mUpdateTrack = MakeUnique(); mUpdateTrack->mInputRate = aSampleRate; mUpdateTrack->mResamplerChannelCount = 0; mUpdateTrack->mData = UniquePtr(mSegment->CreateEmptyClone()); mUpdateTrack->mEnded = false; mUpdateTrack->mPullingEnabled = false; mUpdateTrack->mGraphThreadDone = false; } void SourceMediaTrack::DestroyImpl() { GraphImpl()->AssertOnGraphThreadOrNotRunning(); for (int32_t i = mConsumers.Length() - 1; i >= 0; --i) { // Disconnect before we come under mMutex's lock since it can call back // through RemoveDirectListenerImpl() and deadlock. mConsumers[i]->Disconnect(); } // Hold mMutex while mGraph is reset so that other threads holding mMutex // can null-check know that the graph will not destroyed. MutexAutoLock lock(mMutex); mUpdateTrack = nullptr; MediaTrack::DestroyImpl(); } void SourceMediaTrack::SetPullingEnabled(bool aEnabled) { class Message : public ControlMessage { public: Message(SourceMediaTrack* aTrack, bool aEnabled) : ControlMessage(nullptr), mTrack(aTrack), mEnabled(aEnabled) {} void Run() override { TRACE("SourceMediaTrack::SetPullingEnabled ControlMessage"); MutexAutoLock lock(mTrack->mMutex); if (!mTrack->mUpdateTrack) { // We can't enable pulling for a track that has ended. We ignore // this if we're disabling pulling, since shutdown sequences are // complex. If there's truly an issue we'll have issues enabling anyway. MOZ_ASSERT_IF(mEnabled, mTrack->mEnded); return; } MOZ_ASSERT(mTrack->mType == MediaSegment::AUDIO, "Pulling is not allowed for video"); mTrack->mUpdateTrack->mPullingEnabled = mEnabled; } SourceMediaTrack* mTrack; bool mEnabled; }; GraphImpl()->AppendMessage(MakeUnique(this, aEnabled)); } bool SourceMediaTrack::PullNewData(GraphTime aDesiredUpToTime) { TRACE_COMMENT("SourceMediaTrack::PullNewData", "%p", this); TrackTime t; TrackTime current; { if (mEnded) { return false; } MutexAutoLock lock(mMutex); if (mUpdateTrack->mEnded) { return false; } if (!mUpdateTrack->mPullingEnabled) { return false; } // Compute how much track time we'll need assuming we don't block // the track at all. t = GraphTimeToTrackTime(aDesiredUpToTime); current = GetEnd() + mUpdateTrack->mData->GetDuration(); } if (t <= current) { return false; } LOG(LogLevel::Verbose, ("%p: Calling NotifyPull track=%p t=%f current end=%f", GraphImpl(), this, GraphImpl()->MediaTimeToSeconds(t), GraphImpl()->MediaTimeToSeconds(current))); for (auto& l : mTrackListeners) { l->NotifyPull(Graph(), current, t); } return true; } /** * This moves chunks from aIn to aOut. For audio this is simple. For video * we carry durations over if present, or extend up to aDesiredUpToTime if not. * * We also handle "resetters" from captured media elements. This type of source * pushes future frames into the track, and should it need to remove some, e.g., * because of a seek or pause, it tells us by letting time go backwards. Without * this, tracks would be live for too long after a seek or pause. */ static void MoveToSegment(SourceMediaTrack* aTrack, MediaSegment* aIn, MediaSegment* aOut, TrackTime aCurrentTime, TrackTime aDesiredUpToTime) MOZ_REQUIRES(aTrack->GetMutex()) { MOZ_ASSERT(aIn->GetType() == aOut->GetType()); MOZ_ASSERT(aOut->GetDuration() >= aCurrentTime); MOZ_ASSERT(aDesiredUpToTime >= aCurrentTime); if (aIn->GetType() == MediaSegment::AUDIO) { AudioSegment* in = static_cast(aIn); AudioSegment* out = static_cast(aOut); TrackTime desiredDurationToMove = aDesiredUpToTime - aCurrentTime; TrackTime end = std::min(in->GetDuration(), desiredDurationToMove); out->AppendSlice(*in, 0, end); in->RemoveLeading(end); aTrack->GetMutex().AssertCurrentThreadOwns(); out->ApplyVolume(aTrack->GetVolumeLocked()); } else { VideoSegment* in = static_cast(aIn); VideoSegment* out = static_cast(aOut); for (VideoSegment::ConstChunkIterator c(*in); !c.IsEnded(); c.Next()) { MOZ_ASSERT(!c->mTimeStamp.IsNull()); VideoChunk* last = out->GetLastChunk(); if (!last || last->mTimeStamp.IsNull()) { // This is the first frame, or the last frame pushed to `out` has been // all consumed. Just append and we deal with its duration later. out->AppendFrame(do_AddRef(c->mFrame.GetImage()), c->mFrame.GetIntrinsicSize(), c->mFrame.GetPrincipalHandle(), c->mFrame.GetForceBlack(), c->mTimeStamp); if (c->GetDuration() > 0) { out->ExtendLastFrameBy(c->GetDuration()); } continue; } // We now know when this frame starts, aka when the last frame ends. if (c->mTimeStamp < last->mTimeStamp) { // Time is going backwards. This is a resetting frame from // DecodedStream. Clear everything up to currentTime. out->Clear(); out->AppendNullData(aCurrentTime); } // Append the current frame (will have duration 0). out->AppendFrame(do_AddRef(c->mFrame.GetImage()), c->mFrame.GetIntrinsicSize(), c->mFrame.GetPrincipalHandle(), c->mFrame.GetForceBlack(), c->mTimeStamp); if (c->GetDuration() > 0) { out->ExtendLastFrameBy(c->GetDuration()); } } if (out->GetDuration() < aDesiredUpToTime) { out->ExtendLastFrameBy(aDesiredUpToTime - out->GetDuration()); } in->Clear(); MOZ_ASSERT(aIn->GetDuration() == 0, "aIn must be consumed"); } } void SourceMediaTrack::ExtractPendingInput(GraphTime aCurrentTime, GraphTime aDesiredUpToTime) { MutexAutoLock lock(mMutex); if (!mUpdateTrack) { MOZ_ASSERT(mEnded); return; } TrackTime trackCurrentTime = GraphTimeToTrackTime(aCurrentTime); ApplyTrackDisabling(mUpdateTrack->mData.get()); if (!mUpdateTrack->mData->IsEmpty()) { for (const auto& l : mTrackListeners) { l->NotifyQueuedChanges(GraphImpl(), GetEnd(), *mUpdateTrack->mData); } } TrackTime trackDesiredUpToTime = GraphTimeToTrackTime(aDesiredUpToTime); TrackTime endTime = trackDesiredUpToTime; if (mUpdateTrack->mEnded) { endTime = std::min(trackDesiredUpToTime, GetEnd() + mUpdateTrack->mData->GetDuration()); } LOG(LogLevel::Verbose, ("%p: SourceMediaTrack %p advancing end from %" PRId64 " to %" PRId64, GraphImpl(), this, int64_t(trackCurrentTime), int64_t(endTime))); MoveToSegment(this, mUpdateTrack->mData.get(), mSegment.get(), trackCurrentTime, endTime); if (mUpdateTrack->mEnded && GetEnd() < trackDesiredUpToTime) { mEnded = true; mUpdateTrack = nullptr; } } void SourceMediaTrack::ResampleAudioToGraphSampleRate(MediaSegment* aSegment) { mMutex.AssertCurrentThreadOwns(); if (aSegment->GetType() != MediaSegment::AUDIO || mUpdateTrack->mInputRate == GraphImpl()->GraphRate()) { return; } AudioSegment* segment = static_cast(aSegment); segment->ResampleChunks(mUpdateTrack->mResampler, &mUpdateTrack->mResamplerChannelCount, mUpdateTrack->mInputRate, GraphImpl()->GraphRate()); } void SourceMediaTrack::AdvanceTimeVaryingValuesToCurrentTime( GraphTime aCurrentTime, GraphTime aBlockedTime) { MutexAutoLock lock(mMutex); MediaTrack::AdvanceTimeVaryingValuesToCurrentTime(aCurrentTime, aBlockedTime); } void SourceMediaTrack::SetAppendDataSourceRate(TrackRate aRate) { MutexAutoLock lock(mMutex); if (!mUpdateTrack) { return; } MOZ_DIAGNOSTIC_ASSERT(mSegment->GetType() == MediaSegment::AUDIO); // Set the new input rate and reset the resampler. mUpdateTrack->mInputRate = aRate; mUpdateTrack->mResampler.own(nullptr); mUpdateTrack->mResamplerChannelCount = 0; } TrackTime SourceMediaTrack::AppendData(MediaSegment* aSegment, MediaSegment* aRawSegment) { MutexAutoLock lock(mMutex); MOZ_DIAGNOSTIC_ASSERT(aSegment->GetType() == mType); TrackTime appended = 0; if (!mUpdateTrack || mUpdateTrack->mEnded || mUpdateTrack->mGraphThreadDone) { aSegment->Clear(); return appended; } // Data goes into mData, and on the next iteration of the MTG moves // into the track's segment after NotifyQueuedTrackChanges(). This adds // 0-10ms of delay before data gets to direct listeners. // Indirect listeners (via subsequent TrackUnion nodes) are synced to // playout time, and so can be delayed by buffering. // Apply track disabling before notifying any consumers directly // or inserting into the graph mozilla::ApplyTrackDisabling(mDirectDisabledMode, aSegment, aRawSegment); ResampleAudioToGraphSampleRate(aSegment); // Must notify first, since AppendFrom() will empty out aSegment NotifyDirectConsumers(aRawSegment ? aRawSegment : aSegment); appended = aSegment->GetDuration(); mUpdateTrack->mData->AppendFrom(aSegment); // note: aSegment is now dead { auto graph = GraphImpl(); MonitorAutoLock lock(graph->GetMonitor()); if (graph->CurrentDriver()) { // graph has not completed forced shutdown graph->EnsureNextIteration(); } } return appended; } TrackTime SourceMediaTrack::ClearFutureData() { MutexAutoLock lock(mMutex); auto graph = GraphImpl(); if (!mUpdateTrack || !graph) { return 0; } TrackTime duration = mUpdateTrack->mData->GetDuration(); mUpdateTrack->mData->Clear(); return duration; } void SourceMediaTrack::NotifyDirectConsumers(MediaSegment* aSegment) { mMutex.AssertCurrentThreadOwns(); for (const auto& l : mDirectTrackListeners) { TrackTime offset = 0; // FIX! need a separate TrackTime.... or the end of // the internal buffer l->NotifyRealtimeTrackDataAndApplyTrackDisabling(Graph(), offset, *aSegment); } } void SourceMediaTrack::AddDirectListenerImpl( already_AddRefed aListener) { AssertOnGraphThread(); MutexAutoLock lock(mMutex); RefPtr listener = aListener; LOG(LogLevel::Debug, ("%p: Adding direct track listener %p to source track %p", GraphImpl(), listener.get(), this)); MOZ_ASSERT(mType == MediaSegment::VIDEO); for (const auto& l : mDirectTrackListeners) { if (l == listener) { listener->NotifyDirectListenerInstalled( DirectMediaTrackListener::InstallationResult::ALREADY_EXISTS); return; } } mDirectTrackListeners.AppendElement(listener); LOG(LogLevel::Debug, ("%p: Added direct track listener %p", GraphImpl(), listener.get())); listener->NotifyDirectListenerInstalled( DirectMediaTrackListener::InstallationResult::SUCCESS); if (mDisabledMode != DisabledTrackMode::ENABLED) { listener->IncreaseDisabled(mDisabledMode); } if (mEnded) { return; } // Pass buffered data to the listener VideoSegment bufferedData; size_t videoFrames = 0; VideoSegment& segment = *GetData(); for (VideoSegment::ConstChunkIterator iter(segment); !iter.IsEnded(); iter.Next()) { if (iter->mTimeStamp.IsNull()) { // No timestamp means this is only for the graph's internal book-keeping, // denoting a late start of the track. continue; } ++videoFrames; bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()), iter->mFrame.GetIntrinsicSize(), iter->mFrame.GetPrincipalHandle(), iter->mFrame.GetForceBlack(), iter->mTimeStamp); } VideoSegment& video = static_cast(*mUpdateTrack->mData); for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded(); iter.Next()) { ++videoFrames; MOZ_ASSERT(!iter->mTimeStamp.IsNull()); bufferedData.AppendFrame(do_AddRef(iter->mFrame.GetImage()), iter->mFrame.GetIntrinsicSize(), iter->mFrame.GetPrincipalHandle(), iter->mFrame.GetForceBlack(), iter->mTimeStamp); } LOG(LogLevel::Info, ("%p: Notifying direct listener %p of %zu video frames and duration " "%" PRId64, GraphImpl(), listener.get(), videoFrames, bufferedData.GetDuration())); listener->NotifyRealtimeTrackData(Graph(), 0, bufferedData); } void SourceMediaTrack::RemoveDirectListenerImpl( DirectMediaTrackListener* aListener) { mGraph->AssertOnGraphThreadOrNotRunning(); MutexAutoLock lock(mMutex); for (int32_t i = mDirectTrackListeners.Length() - 1; i >= 0; --i) { const RefPtr& l = mDirectTrackListeners[i]; if (l == aListener) { if (mDisabledMode != DisabledTrackMode::ENABLED) { aListener->DecreaseDisabled(mDisabledMode); } aListener->NotifyDirectListenerUninstalled(); mDirectTrackListeners.RemoveElementAt(i); } } } void SourceMediaTrack::End() { MutexAutoLock lock(mMutex); if (!mUpdateTrack) { // Already ended return; } mUpdateTrack->mEnded = true; if (auto graph = GraphImpl()) { MonitorAutoLock lock(graph->GetMonitor()); if (graph->CurrentDriver()) { // graph has not completed forced shutdown graph->EnsureNextIteration(); } } } void SourceMediaTrack::SetDisabledTrackModeImpl(DisabledTrackMode aMode) { AssertOnGraphThread(); { MutexAutoLock lock(mMutex); const DisabledTrackMode oldMode = mDirectDisabledMode; const bool oldEnabled = oldMode == DisabledTrackMode::ENABLED; const bool enabled = aMode == DisabledTrackMode::ENABLED; mDirectDisabledMode = aMode; for (const auto& l : mDirectTrackListeners) { if (!oldEnabled && enabled) { LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting " "direct listener enabled", GraphImpl(), this)); l->DecreaseDisabled(oldMode); } else if (oldEnabled && !enabled) { LOG(LogLevel::Debug, ("%p: SourceMediaTrack %p setting " "direct listener disabled", GraphImpl(), this)); l->IncreaseDisabled(aMode); } } } MediaTrack::SetDisabledTrackModeImpl(aMode); } uint32_t SourceMediaTrack::NumberOfChannels() const { AudioSegment* audio = GetData(); MOZ_DIAGNOSTIC_ASSERT(audio); if (!audio) { return 0; } return audio->MaxChannelCount(); } void SourceMediaTrack::RemoveAllDirectListenersImpl() { GraphImpl()->AssertOnGraphThreadOrNotRunning(); MutexAutoLock lock(mMutex); for (auto& l : mDirectTrackListeners.Clone()) { l->NotifyDirectListenerUninstalled(); } mDirectTrackListeners.Clear(); } void SourceMediaTrack::SetVolume(float aVolume) { MutexAutoLock lock(mMutex); mVolume = aVolume; } float SourceMediaTrack::GetVolumeLocked() { mMutex.AssertCurrentThreadOwns(); return mVolume; } SourceMediaTrack::~SourceMediaTrack() = default; void MediaInputPort::Init() { mGraph->AssertOnGraphThreadOrNotRunning(); LOG(LogLevel::Debug, ("%p: Adding MediaInputPort %p (from %p to %p)", mGraph, this, mSource, mDest)); // Only connect the port if it wasn't disconnected on allocation. if (mSource) { mSource->AddConsumer(this); mDest->AddInput(this); } // mPortCount decremented via MediaInputPort::Destroy's message ++mGraph->mPortCount; } void MediaInputPort::Disconnect() { mGraph->AssertOnGraphThreadOrNotRunning(); NS_ASSERTION(!mSource == !mDest, "mSource and mDest must either both be null or both non-null"); if (!mSource) { return; } mSource->RemoveConsumer(this); mDest->RemoveInput(this); mSource = nullptr; mDest = nullptr; mGraph->SetTrackOrderDirty(); } MediaTrack* MediaInputPort::GetSource() const { mGraph->AssertOnGraphThreadOrNotRunning(); return mSource; } ProcessedMediaTrack* MediaInputPort::GetDestination() const { mGraph->AssertOnGraphThreadOrNotRunning(); return mDest; } MediaInputPort::InputInterval MediaInputPort::GetNextInputInterval( MediaInputPort const* aPort, GraphTime aTime) { InputInterval result = {GRAPH_TIME_MAX, GRAPH_TIME_MAX, false}; if (!aPort) { result.mStart = aTime; result.mInputIsBlocked = true; return result; } aPort->mGraph->AssertOnGraphThreadOrNotRunning(); if (aTime >= aPort->mDest->mStartBlocking) { return result; } result.mStart = aTime; result.mEnd = aPort->mDest->mStartBlocking; result.mInputIsBlocked = aTime >= aPort->mSource->mStartBlocking; if (!result.mInputIsBlocked) { result.mEnd = std::min(result.mEnd, aPort->mSource->mStartBlocking); } return result; } void MediaInputPort::Suspended() { mGraph->AssertOnGraphThreadOrNotRunning(); mDest->InputSuspended(this); } void MediaInputPort::Resumed() { mGraph->AssertOnGraphThreadOrNotRunning(); mDest->InputResumed(this); } void MediaInputPort::Destroy() { class Message : public ControlMessage { public: explicit Message(MediaInputPort* aPort) : ControlMessage(nullptr), mPort(aPort) {} void Run() override { TRACE("MediaInputPort::Destroy ControlMessage"); mPort->Disconnect(); --mPort->GraphImpl()->mPortCount; mPort->SetGraphImpl(nullptr); NS_RELEASE(mPort); } void RunDuringShutdown() override { Run(); } MediaInputPort* mPort; }; // Keep a reference to the graph, since Message might RunDuringShutdown() // synchronously and make GraphImpl() invalid. RefPtr graph = mGraph; graph->AppendMessage(MakeUnique(this)); --graph->mMainThreadPortCount; } MediaTrackGraphImpl* MediaInputPort::GraphImpl() const { mGraph->AssertOnGraphThreadOrNotRunning(); return mGraph; } MediaTrackGraph* MediaInputPort::Graph() const { mGraph->AssertOnGraphThreadOrNotRunning(); return mGraph; } void MediaInputPort::SetGraphImpl(MediaTrackGraphImpl* aGraph) { MOZ_ASSERT(!mGraph || !aGraph, "Should only be set once"); DebugOnly graph = mGraph ? mGraph : aGraph; MOZ_ASSERT(graph->OnGraphThreadOrNotRunning()); mGraph = aGraph; } already_AddRefed ProcessedMediaTrack::AllocateInputPort( MediaTrack* aTrack, uint16_t aInputNumber, uint16_t aOutputNumber) { // This method creates two references to the MediaInputPort: one for // the main thread, and one for the MediaTrackGraph. class Message : public ControlMessage { public: explicit Message(MediaInputPort* aPort) : ControlMessage(aPort->mDest), mPort(aPort) {} void Run() override { TRACE("ProcessedMediaTrack::AllocateInputPort ControlMessage"); mPort->Init(); // The graph holds its reference implicitly mPort->GraphImpl()->SetTrackOrderDirty(); Unused << mPort.forget(); } void RunDuringShutdown() override { Run(); } RefPtr mPort; }; MOZ_DIAGNOSTIC_ASSERT(aTrack->mType == mType); RefPtr port; if (aTrack->IsDestroyed()) { // Create a port that's disconnected, which is what it'd be after its source // track is Destroy()ed normally. Disconnect() is idempotent so destroying // this later is fine. port = new MediaInputPort(GraphImpl(), nullptr, nullptr, aInputNumber, aOutputNumber); } else { MOZ_ASSERT(aTrack->GraphImpl() == GraphImpl()); port = new MediaInputPort(GraphImpl(), aTrack, this, aInputNumber, aOutputNumber); } ++GraphImpl()->mMainThreadPortCount; GraphImpl()->AppendMessage(MakeUnique(port)); return port.forget(); } void ProcessedMediaTrack::QueueSetAutoend(bool aAutoend) { class Message : public ControlMessage { public: Message(ProcessedMediaTrack* aTrack, bool aAutoend) : ControlMessage(aTrack), mAutoend(aAutoend) {} void Run() override { TRACE("ProcessedMediaTrack::SetAutoendImpl ControlMessage"); static_cast(mTrack)->SetAutoendImpl(mAutoend); } bool mAutoend; }; if (mMainThreadDestroyed) { return; } GraphImpl()->AppendMessage(MakeUnique(this, aAutoend)); } void ProcessedMediaTrack::DestroyImpl() { for (int32_t i = mInputs.Length() - 1; i >= 0; --i) { mInputs[i]->Disconnect(); } for (int32_t i = mSuspendedInputs.Length() - 1; i >= 0; --i) { mSuspendedInputs[i]->Disconnect(); } MediaTrack::DestroyImpl(); // The track order is only important if there are connections, in which // case MediaInputPort::Disconnect() called SetTrackOrderDirty(). // MediaTrackGraphImpl::RemoveTrackGraphThread() will also call // SetTrackOrderDirty(), for other reasons. } MediaTrackGraphImpl::MediaTrackGraphImpl(uint64_t aWindowID, TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID, nsISerialEventTarget* aMainThread) : MediaTrackGraph(aSampleRate, aPrimaryOutputDeviceID), mWindowID(aWindowID), mFirstCycleBreaker(0) // An offline graph is not initially processing. , mPortCount(0), mMonitor("MediaTrackGraphImpl"), mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED), mPostedRunInStableStateEvent(false), mGraphDriverRunning(false), mPostedRunInStableState(false), mTrackOrderDirty(false), mMainThread(aMainThread), mGlobalVolume(CubebUtils::GetVolumeScale()) #ifdef DEBUG , mCanRunMessagesSynchronously(false) #endif , mMainThreadGraphTime(0, "MediaTrackGraphImpl::mMainThreadGraphTime"), mAudioOutputLatency(0.0), mMaxOutputChannelCount(std::min(8u, CubebUtils::MaxNumberOfChannels())) { } void MediaTrackGraphImpl::Init(GraphDriverType aDriverRequested, GraphRunType aRunTypeRequested, uint32_t aChannelCount) { mSelfRef = this; mEndTime = aDriverRequested == OFFLINE_THREAD_DRIVER ? 0 : GRAPH_TIME_MAX; mRealtime = aDriverRequested != OFFLINE_THREAD_DRIVER; // The primary output device always exists because an AudioCallbackDriver // may exist, and want to be fed data, even when no tracks have audio // outputs. mOutputDeviceRefCnts.EmplaceBack( DeviceReceiverAndCount{mPrimaryOutputDeviceID, nullptr, 0}); mOutputDevices.EmplaceBack(OutputDeviceEntry{mPrimaryOutputDeviceID}); bool failedToGetShutdownBlocker = false; if (!IsNonRealtime()) { failedToGetShutdownBlocker = !AddShutdownBlocker(); } mGraphRunner = aRunTypeRequested == SINGLE_THREAD ? GraphRunner::Create(this) : already_AddRefed(nullptr); if ((aRunTypeRequested == SINGLE_THREAD && !mGraphRunner) || failedToGetShutdownBlocker) { MonitorAutoLock lock(mMonitor); // At least one of the following happened // - Failed to create thread. // - Failed to install a shutdown blocker when one is needed. // Because we have a fail state, jump to last phase of the lifecycle. mLifecycleState = LIFECYCLE_WAITING_FOR_TRACK_DESTRUCTION; RemoveShutdownBlocker(); // No-op if blocker wasn't added. #ifdef DEBUG mCanRunMessagesSynchronously = true; #endif return; } if (mRealtime) { if (aDriverRequested == AUDIO_THREAD_DRIVER) { // Always start with zero input channels, and no particular preferences // for the input channel. mDriver = new AudioCallbackDriver( this, nullptr, mSampleRate, aChannelCount, 0, PrimaryOutputDeviceID(), nullptr, AudioInputType::Unknown, CUBEB_INPUT_PROCESSING_PARAM_NONE); } else { mDriver = new SystemClockDriver(this, nullptr, mSampleRate); } nsCString streamName = GetDocumentTitle(mWindowID); LOG(LogLevel::Debug, ("%p: document title: %s", this, streamName.get())); mDriver->SetStreamName(streamName); } else { mDriver = new OfflineClockDriver(this, mSampleRate, MEDIA_GRAPH_TARGET_PERIOD_MS); } mLastMainThreadUpdate = TimeStamp::Now(); RegisterWeakAsyncMemoryReporter(this); } #ifdef DEBUG bool MediaTrackGraphImpl::InDriverIteration(const GraphDriver* aDriver) const { return aDriver->OnThread() || (mGraphRunner && mGraphRunner->InDriverIteration(aDriver)); } #endif void MediaTrackGraphImpl::Destroy() { // First unregister from memory reporting. UnregisterWeakMemoryReporter(this); // Clear the self reference which will destroy this instance if all // associated GraphDrivers are destroyed. mSelfRef = nullptr; } // Internal method has a Window ID parameter so that TestAudioTrackGraph // GTests can create a graph without a window. /* static */ MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstanceIfExists( uint64_t aWindowID, TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID) { MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); MOZ_ASSERT(aSampleRate > 0); GraphHashSet::Ptr p = Graphs()->lookup({aWindowID, aSampleRate, aPrimaryOutputDeviceID}); return p ? *p : nullptr; } // Public method has an nsPIDOMWindowInner* parameter to ensure that the // window is a real inner Window, not a WindowProxy. /* static */ MediaTrackGraph* MediaTrackGraph::GetInstanceIfExists( nsPIDOMWindowInner* aWindow, TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID) { TrackRate sampleRate = aSampleRate ? aSampleRate : CubebUtils::PreferredSampleRate( aWindow->AsGlobal()->ShouldResistFingerprinting( RFPTarget::AudioSampleRate)); return MediaTrackGraphImpl::GetInstanceIfExists( aWindow->WindowID(), sampleRate, aPrimaryOutputDeviceID); } /* static */ MediaTrackGraphImpl* MediaTrackGraphImpl::GetInstance( GraphDriverType aGraphDriverRequested, uint64_t aWindowID, TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID, nsISerialEventTarget* aMainThread) { MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); MOZ_ASSERT(aSampleRate > 0); MOZ_ASSERT(aGraphDriverRequested != OFFLINE_THREAD_DRIVER, "Use CreateNonRealtimeInstance() for offline graphs"); GraphHashSet* graphs = Graphs(); GraphHashSet::AddPtr addPtr = graphs->lookupForAdd({aWindowID, aSampleRate, aPrimaryOutputDeviceID}); if (addPtr) { // graph already exists return *addPtr; } GraphRunType runType = DIRECT_DRIVER; if (Preferences::GetBool("media.audiograph.single_thread.enabled", true)) { runType = SINGLE_THREAD; } // In a real time graph, the number of output channels is determined by // the underlying number of channel of the default audio output device, and // capped to 8. uint32_t channelCount = std::min(8, CubebUtils::MaxNumberOfChannels()); MediaTrackGraphImpl* graph = new MediaTrackGraphImpl( aWindowID, aSampleRate, aPrimaryOutputDeviceID, aMainThread); graph->Init(aGraphDriverRequested, runType, channelCount); MOZ_ALWAYS_TRUE(graphs->add(addPtr, graph)); LOG(LogLevel::Debug, ("Starting up MediaTrackGraph %p for window 0x%" PRIx64, graph, aWindowID)); return graph; } /* static */ MediaTrackGraph* MediaTrackGraph::GetInstance( GraphDriverType aGraphDriverRequested, nsPIDOMWindowInner* aWindow, TrackRate aSampleRate, AudioDeviceID aPrimaryOutputDeviceID) { TrackRate sampleRate = aSampleRate ? aSampleRate : CubebUtils::PreferredSampleRate( aWindow->AsGlobal()->ShouldResistFingerprinting( RFPTarget::AudioSampleRate)); return MediaTrackGraphImpl::GetInstance( aGraphDriverRequested, aWindow->WindowID(), sampleRate, aPrimaryOutputDeviceID, GetMainThreadSerialEventTarget()); } MediaTrackGraph* MediaTrackGraphImpl::CreateNonRealtimeInstance( TrackRate aSampleRate) { MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); nsISerialEventTarget* mainThread = GetMainThreadSerialEventTarget(); // Offline graphs have 0 output channel count: they write the output to a // buffer, not an audio output track. MediaTrackGraphImpl* graph = new MediaTrackGraphImpl( 0, aSampleRate, DEFAULT_OUTPUT_DEVICE, mainThread); graph->Init(OFFLINE_THREAD_DRIVER, DIRECT_DRIVER, 0); LOG(LogLevel::Debug, ("Starting up Offline MediaTrackGraph %p", graph)); return graph; } MediaTrackGraph* MediaTrackGraph::CreateNonRealtimeInstance( TrackRate aSampleRate) { return MediaTrackGraphImpl::CreateNonRealtimeInstance(aSampleRate); } void MediaTrackGraph::ForceShutDown() { MOZ_ASSERT(NS_IsMainThread(), "Main thread only"); MediaTrackGraphImpl* graph = static_cast(this); graph->ForceShutDown(); } NS_IMPL_ISUPPORTS(MediaTrackGraphImpl, nsIMemoryReporter, nsIObserver, nsIThreadObserver, nsITimerCallback, nsINamed) NS_IMETHODIMP MediaTrackGraphImpl::CollectReports(nsIHandleReportCallback* aHandleReport, nsISupports* aData, bool aAnonymize) { MOZ_ASSERT(NS_IsMainThread()); if (mMainThreadTrackCount == 0) { // No tracks to report. FinishCollectReports(aHandleReport, aData, nsTArray()); return NS_OK; } class Message final : public ControlMessage { public: Message(MediaTrackGraphImpl* aGraph, nsIHandleReportCallback* aHandleReport, nsISupports* aHandlerData) : ControlMessage(nullptr), mGraph(aGraph), mHandleReport(aHandleReport), mHandlerData(aHandlerData) {} void Run() override { TRACE("MTG::CollectSizesForMemoryReport ControlMessage"); mGraph->CollectSizesForMemoryReport(mHandleReport.forget(), mHandlerData.forget()); } void RunDuringShutdown() override { // Run this message during shutdown too, so that endReports is called. Run(); } MediaTrackGraphImpl* mGraph; // nsMemoryReporterManager keeps the callback and data alive only if it // does not time out. nsCOMPtr mHandleReport; nsCOMPtr mHandlerData; }; AppendMessage(MakeUnique(this, aHandleReport, aData)); return NS_OK; } void MediaTrackGraphImpl::CollectSizesForMemoryReport( already_AddRefed aHandleReport, already_AddRefed aHandlerData) { class FinishCollectRunnable final : public Runnable { public: explicit FinishCollectRunnable( already_AddRefed aHandleReport, already_AddRefed aHandlerData) : mozilla::Runnable("FinishCollectRunnable"), mHandleReport(aHandleReport), mHandlerData(aHandlerData) {} NS_IMETHOD Run() override { TRACE("MTG::FinishCollectReports ControlMessage"); MediaTrackGraphImpl::FinishCollectReports(mHandleReport, mHandlerData, std::move(mAudioTrackSizes)); return NS_OK; } nsTArray mAudioTrackSizes; private: ~FinishCollectRunnable() = default; // Avoiding nsCOMPtr because NSCAP_ASSERT_NO_QUERY_NEEDED in its // constructor modifies the ref-count, which cannot be done off main // thread. RefPtr mHandleReport; RefPtr mHandlerData; }; RefPtr runnable = new FinishCollectRunnable( std::move(aHandleReport), std::move(aHandlerData)); auto audioTrackSizes = &runnable->mAudioTrackSizes; for (MediaTrack* t : AllTracks()) { AudioNodeTrack* track = t->AsAudioNodeTrack(); if (track) { AudioNodeSizes* usage = audioTrackSizes->AppendElement(); track->SizeOfAudioNodesIncludingThis(MallocSizeOf, *usage); } } mMainThread->Dispatch(runnable.forget()); } void MediaTrackGraphImpl::FinishCollectReports( nsIHandleReportCallback* aHandleReport, nsISupports* aData, const nsTArray& aAudioTrackSizes) { MOZ_ASSERT(NS_IsMainThread()); nsCOMPtr manager = do_GetService("@mozilla.org/memory-reporter-manager;1"); if (!manager) return; #define REPORT(_path, _amount, _desc) \ aHandleReport->Callback(""_ns, _path, KIND_HEAP, UNITS_BYTES, _amount, \ nsLiteralCString(_desc), aData); for (size_t i = 0; i < aAudioTrackSizes.Length(); i++) { const AudioNodeSizes& usage = aAudioTrackSizes[i]; const char* const nodeType = usage.mNodeType ? usage.mNodeType : ""; nsPrintfCString enginePath("explicit/webaudio/audio-node/%s/engine-objects", nodeType); REPORT(enginePath, usage.mEngine, "Memory used by AudioNode engine objects (Web Audio)."); nsPrintfCString trackPath("explicit/webaudio/audio-node/%s/track-objects", nodeType); REPORT(trackPath, usage.mTrack, "Memory used by AudioNode track objects (Web Audio)."); } size_t hrtfLoaders = WebCore::HRTFDatabaseLoader::sizeOfLoaders(MallocSizeOf); if (hrtfLoaders) { REPORT(nsLiteralCString( "explicit/webaudio/audio-node/PannerNode/hrtf-databases"), hrtfLoaders, "Memory used by PannerNode databases (Web Audio)."); } #undef REPORT manager->EndReport(); } SourceMediaTrack* MediaTrackGraph::CreateSourceTrack(MediaSegment::Type aType) { SourceMediaTrack* track = new SourceMediaTrack(aType, GraphRate()); AddTrack(track); return track; } ProcessedMediaTrack* MediaTrackGraph::CreateForwardedInputTrack( MediaSegment::Type aType) { ForwardedInputTrack* track = new ForwardedInputTrack(GraphRate(), aType); AddTrack(track); return track; } AudioCaptureTrack* MediaTrackGraph::CreateAudioCaptureTrack() { AudioCaptureTrack* track = new AudioCaptureTrack(GraphRate()); AddTrack(track); return track; } CrossGraphTransmitter* MediaTrackGraph::CreateCrossGraphTransmitter( CrossGraphReceiver* aReceiver) { CrossGraphTransmitter* track = new CrossGraphTransmitter(GraphRate(), aReceiver); AddTrack(track); return track; } CrossGraphReceiver* MediaTrackGraph::CreateCrossGraphReceiver( TrackRate aTransmitterRate) { CrossGraphReceiver* track = new CrossGraphReceiver(GraphRate(), aTransmitterRate); AddTrack(track); return track; } void MediaTrackGraph::AddTrack(MediaTrack* aTrack) { MediaTrackGraphImpl* graph = static_cast(this); MOZ_ASSERT(NS_IsMainThread()); #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED if (graph->mRealtime) { GraphHashSet::Ptr p = Graphs()->lookup(*graph); MOZ_DIAGNOSTIC_ASSERT(p, "Graph must not be shutting down"); } #endif if (graph->mMainThreadTrackCount == 0) { nsCOMPtr observerService = mozilla::services::GetObserverService(); if (observerService) { observerService->AddObserver(graph, "document-title-changed", false); } } NS_ADDREF(aTrack); aTrack->SetGraphImpl(graph); ++graph->mMainThreadTrackCount; graph->AppendMessage(MakeUnique(aTrack)); } void MediaTrackGraphImpl::RemoveTrack(MediaTrack* aTrack) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(mMainThreadTrackCount > 0); mAudioOutputParams.RemoveElementsBy( [&](const TrackKeyDeviceAndVolume& aElement) { if (aElement.mTrack != aTrack) { return false; }; DecrementOutputDeviceRefCnt(aElement.mDeviceID); return true; }); if (--mMainThreadTrackCount == 0) { LOG(LogLevel::Info, ("MediaTrackGraph %p, last track %p removed from " "main thread. Graph will shut down.", this, aTrack)); if (mRealtime) { // Find the graph in the hash table and remove it. GraphHashSet* graphs = Graphs(); GraphHashSet::Ptr p = graphs->lookup(*this); MOZ_ASSERT(*p == this); graphs->remove(p); nsCOMPtr observerService = mozilla::services::GetObserverService(); if (observerService) { observerService->RemoveObserver(this, "document-title-changed"); } } // The graph thread will shut itself down soon, but won't be able to do // that if JS continues to run. InterruptJS(); } } auto MediaTrackGraphImpl::NotifyWhenDeviceStarted(AudioDeviceID aDeviceID) -> RefPtr { MOZ_ASSERT(NS_IsMainThread()); size_t index = mOutputDeviceRefCnts.IndexOf(aDeviceID); if (index == decltype(mOutputDeviceRefCnts)::NoIndex) { return GraphStartedPromise::CreateAndReject(NS_ERROR_INVALID_ARG, __func__); } MozPromiseHolder h; RefPtr p = h.Ensure(__func__); if (CrossGraphReceiver* receiver = mOutputDeviceRefCnts[index].mReceiver) { receiver->GraphImpl()->NotifyWhenPrimaryDeviceStarted(std::move(h)); return p; } // aSink corresponds to the primary audio output device of this graph. NotifyWhenPrimaryDeviceStarted(std::move(h)); return p; } void MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted( MozPromiseHolder&& aHolder) { MOZ_ASSERT(NS_IsMainThread()); if (mOutputDeviceRefCnts[0].mRefCnt == 0) { // There are no track outputs that require the device, so the creator of // this promise no longer needs to know when the graph is running. Don't // keep the graph alive with another message. aHolder.Reject(NS_ERROR_NOT_AVAILABLE, __func__); return; } QueueControlOrShutdownMessage( [self = RefPtr{this}, this, holder = std::move(aHolder)](IsInShutdown aInShutdown) mutable { if (aInShutdown == IsInShutdown::Yes) { holder.Reject(NS_ERROR_ILLEGAL_DURING_SHUTDOWN, __func__); return; } TRACE("MTG::NotifyWhenPrimaryDeviceStarted ControlMessage"); // This runs on the graph thread, so when this runs, and the current // driver is an AudioCallbackDriver, we know the audio hardware is // started. If not, we are going to switch soon, keep reposting this // ControlMessage. if (CurrentDriver()->AsAudioCallbackDriver() && CurrentDriver()->ThreadRunning() && !CurrentDriver()->AsAudioCallbackDriver()->OnFallback()) { // Avoid Resolve's locking on the graph thread by doing it on main. Dispatch(NS_NewRunnableFunction( "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted::Resolver", [holder = std::move(holder)]() mutable { holder.Resolve(true, __func__); })); } else { DispatchToMainThreadStableState( NewRunnableMethod< StoreCopyPassByRRef>>( "MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted", this, &MediaTrackGraphImpl::NotifyWhenPrimaryDeviceStarted, std::move(holder))); } }); } class AudioContextOperationControlMessage : public ControlMessage { using AudioContextOperationPromise = MediaTrackGraph::AudioContextOperationPromise; public: AudioContextOperationControlMessage( MediaTrack* aDestinationTrack, nsTArray> aTracks, AudioContextOperation aOperation, MozPromiseHolder&& aHolder) : ControlMessage(aDestinationTrack), mTracks(std::move(aTracks)), mAudioContextOperation(aOperation), mHolder(std::move(aHolder)) {} void Run() override { TRACE_COMMENT("MTG::ApplyAudioContextOperationImpl ControlMessage", kAudioContextOptionsStrings[static_cast( mAudioContextOperation)]); mTrack->GraphImpl()->ApplyAudioContextOperationImpl(this); } void RunDuringShutdown() override { MOZ_ASSERT(mAudioContextOperation == AudioContextOperation::Close, "We should be reviving the graph?"); mHolder.Reject(false, __func__); } nsTArray> mTracks; AudioContextOperation mAudioContextOperation; MozPromiseHolder mHolder; }; void MediaTrackGraphImpl::ApplyAudioContextOperationImpl( AudioContextOperationControlMessage* aMessage) { MOZ_ASSERT(OnGraphThread()); // Initialize state to zero. This silences a GCC warning about uninitialized // values, because although the switch below initializes state for all valid // enum values, the actual value could be any integer that fits in the enum. AudioContextState state{0}; switch (aMessage->mAudioContextOperation) { // Suspend and Close operations may be performed immediately because no // specific kind of GraphDriver is required. CheckDriver() will schedule // a change to a SystemCallbackDriver if all tracks are suspended. case AudioContextOperation::Suspend: state = AudioContextState::Suspended; break; case AudioContextOperation::Close: state = AudioContextState::Closed; break; case AudioContextOperation::Resume: // Resume operations require an AudioCallbackDriver. CheckDriver() will // schedule an AudioCallbackDriver if necessary and process pending // operations if and when an AudioCallbackDriver is running. mPendingResumeOperations.EmplaceBack(aMessage); return; } // First resolve any pending Resume promises for the same AudioContext so as // to resolve its associated promises in the same order as they were // created. These Resume operations are considered complete and immediately // canceled by the Suspend or Close. MediaTrack* destinationTrack = aMessage->GetTrack(); bool shrinking = false; auto moveDest = mPendingResumeOperations.begin(); for (PendingResumeOperation& op : mPendingResumeOperations) { if (op.DestinationTrack() == destinationTrack) { op.Apply(this); shrinking = true; continue; } if (shrinking) { // Fill-in gaps in the array. *moveDest = std::move(op); } ++moveDest; } mPendingResumeOperations.TruncateLength(moveDest - mPendingResumeOperations.begin()); for (MediaTrack* track : aMessage->mTracks) { track->IncrementSuspendCount(); } // Resolve after main thread state is up to date with completed processing. DispatchToMainThreadStableState(NS_NewRunnableFunction( "MediaTrackGraphImpl::ApplyAudioContextOperationImpl", [holder = std::move(aMessage->mHolder), state]() mutable { holder.Resolve(state, __func__); })); } MediaTrackGraphImpl::PendingResumeOperation::PendingResumeOperation( AudioContextOperationControlMessage* aMessage) : mDestinationTrack(aMessage->GetTrack()), mTracks(std::move(aMessage->mTracks)), mHolder(std::move(aMessage->mHolder)) { MOZ_ASSERT(aMessage->mAudioContextOperation == AudioContextOperation::Resume); } void MediaTrackGraphImpl::PendingResumeOperation::Apply( MediaTrackGraphImpl* aGraph) { MOZ_ASSERT(aGraph->OnGraphThread()); for (MediaTrack* track : mTracks) { track->DecrementSuspendCount(); } // The graph is provided through the parameter so that it is available even // when the track is destroyed. aGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction( "PendingResumeOperation::Apply", [holder = std::move(mHolder)]() mutable { holder.Resolve(AudioContextState::Running, __func__); })); } void MediaTrackGraphImpl::PendingResumeOperation::Abort() { // The graph is shutting down before the operation completed. MOZ_ASSERT(!mDestinationTrack->GraphImpl() || mDestinationTrack->GraphImpl()->LifecycleStateRef() == MediaTrackGraphImpl::LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN); mHolder.Reject(false, __func__); } auto MediaTrackGraph::ApplyAudioContextOperation( MediaTrack* aDestinationTrack, nsTArray> aTracks, AudioContextOperation aOperation) -> RefPtr { MozPromiseHolder holder; RefPtr p = holder.Ensure(__func__); MediaTrackGraphImpl* graphImpl = static_cast(this); graphImpl->AppendMessage(MakeUnique( aDestinationTrack, std::move(aTracks), aOperation, std::move(holder))); return p; } uint32_t MediaTrackGraphImpl::PrimaryOutputChannelCount() const { MOZ_ASSERT(!mOutputDevices[0].mReceiver); return AudioOutputChannelCount(mOutputDevices[0]); } uint32_t MediaTrackGraphImpl::AudioOutputChannelCount( const OutputDeviceEntry& aDevice) const { MOZ_ASSERT(OnGraphThread()); // The audio output channel count for a graph is the maximum of the output // channel count of all the tracks with outputs to this device, or the max // audio output channel count the machine can do, whichever is smaller. uint32_t channelCount = 0; for (const auto& output : aDevice.mTrackOutputs) { channelCount = std::max(channelCount, output.mTrack->NumberOfChannels()); } channelCount = std::min(channelCount, mMaxOutputChannelCount); if (channelCount) { return channelCount; } else { // null aDevice.mReceiver indicates the primary graph output device. if (!aDevice.mReceiver && CurrentDriver()->AsAudioCallbackDriver()) { return CurrentDriver()->AsAudioCallbackDriver()->OutputChannelCount(); } return 2; } } double MediaTrackGraph::AudioOutputLatency() { return static_cast(this)->AudioOutputLatency(); } double MediaTrackGraphImpl::AudioOutputLatency() { MOZ_ASSERT(NS_IsMainThread()); if (mAudioOutputLatency != 0.0) { return mAudioOutputLatency; } MonitorAutoLock lock(mMonitor); if (CurrentDriver()->AsAudioCallbackDriver()) { mAudioOutputLatency = CurrentDriver() ->AsAudioCallbackDriver() ->AudioOutputLatency() .ToSeconds(); } else { // Failure mode: return 0.0 if running on a normal thread. mAudioOutputLatency = 0.0; } return mAudioOutputLatency; } bool MediaTrackGraph::OutputForAECMightDrift() { return static_cast(this)->OutputForAECMightDrift(); } bool MediaTrackGraph::IsNonRealtime() const { return !static_cast(this)->mRealtime; } void MediaTrackGraph::StartNonRealtimeProcessing(uint32_t aTicksToProcess) { MOZ_ASSERT(NS_IsMainThread(), "main thread only"); MediaTrackGraphImpl* graph = static_cast(this); NS_ASSERTION(!graph->mRealtime, "non-realtime only"); class Message : public ControlMessage { public: explicit Message(MediaTrackGraphImpl* aGraph, uint32_t aTicksToProcess) : ControlMessage(nullptr), mGraph(aGraph), mTicksToProcess(aTicksToProcess) {} void Run() override { TRACE("MTG::StartNonRealtimeProcessing ControlMessage"); MOZ_ASSERT(mGraph->mEndTime == 0, "StartNonRealtimeProcessing should be called only once"); mGraph->mEndTime = mGraph->RoundUpToEndOfAudioBlock( mGraph->mStateComputedTime + mTicksToProcess); } // The graph owns this message. MediaTrackGraphImpl* MOZ_NON_OWNING_REF mGraph; uint32_t mTicksToProcess; }; graph->AppendMessage(MakeUnique(graph, aTicksToProcess)); } void MediaTrackGraphImpl::InterruptJS() { MonitorAutoLock lock(mMonitor); mInterruptJSCalled = true; if (mJSContext) { JS_RequestInterruptCallback(mJSContext); } } static bool InterruptCallback(JSContext* aCx) { // Interrupt future calls also. JS_RequestInterruptCallback(aCx); // Stop execution. return false; } void MediaTrackGraph::NotifyJSContext(JSContext* aCx) { MOZ_ASSERT(OnGraphThread()); MOZ_ASSERT(aCx); auto* impl = static_cast(this); MonitorAutoLock lock(impl->mMonitor); if (impl->mJSContext) { MOZ_ASSERT(impl->mJSContext == aCx); return; } JS_AddInterruptCallback(aCx, InterruptCallback); impl->mJSContext = aCx; if (impl->mInterruptJSCalled) { JS_RequestInterruptCallback(aCx); } } void ProcessedMediaTrack::AddInput(MediaInputPort* aPort) { MediaTrack* t = aPort->GetSource(); if (!t->IsSuspended()) { mInputs.AppendElement(aPort); } else { mSuspendedInputs.AppendElement(aPort); } GraphImpl()->SetTrackOrderDirty(); } void ProcessedMediaTrack::InputSuspended(MediaInputPort* aPort) { GraphImpl()->AssertOnGraphThreadOrNotRunning(); mInputs.RemoveElement(aPort); mSuspendedInputs.AppendElement(aPort); GraphImpl()->SetTrackOrderDirty(); } void ProcessedMediaTrack::InputResumed(MediaInputPort* aPort) { GraphImpl()->AssertOnGraphThreadOrNotRunning(); mSuspendedInputs.RemoveElement(aPort); mInputs.AppendElement(aPort); GraphImpl()->SetTrackOrderDirty(); } void MediaTrackGraphImpl::SwitchAtNextIteration(GraphDriver* aNextDriver) { MOZ_ASSERT(OnGraphThread()); LOG(LogLevel::Debug, ("%p: Switching to new driver: %p", this, aNextDriver)); if (GraphDriver* nextDriver = NextDriver()) { if (nextDriver != CurrentDriver()) { LOG(LogLevel::Debug, ("%p: Discarding previous next driver: %p", this, nextDriver)); } } mNextDriver = aNextDriver; } void MediaTrackGraph::RegisterCaptureTrackForWindow( uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) { MOZ_ASSERT(NS_IsMainThread()); MediaTrackGraphImpl* graphImpl = static_cast(this); graphImpl->RegisterCaptureTrackForWindow(aWindowId, aCaptureTrack); } void MediaTrackGraphImpl::RegisterCaptureTrackForWindow( uint64_t aWindowId, ProcessedMediaTrack* aCaptureTrack) { MOZ_ASSERT(NS_IsMainThread()); WindowAndTrack winAndTrack; winAndTrack.mWindowId = aWindowId; winAndTrack.mCaptureTrackSink = aCaptureTrack; mWindowCaptureTracks.AppendElement(winAndTrack); } void MediaTrackGraph::UnregisterCaptureTrackForWindow(uint64_t aWindowId) { MOZ_ASSERT(NS_IsMainThread()); MediaTrackGraphImpl* graphImpl = static_cast(this); graphImpl->UnregisterCaptureTrackForWindow(aWindowId); } void MediaTrackGraphImpl::UnregisterCaptureTrackForWindow(uint64_t aWindowId) { MOZ_ASSERT(NS_IsMainThread()); mWindowCaptureTracks.RemoveElementsBy( [aWindowId](const auto& track) { return track.mWindowId == aWindowId; }); } already_AddRefed MediaTrackGraph::ConnectToCaptureTrack( uint64_t aWindowId, MediaTrack* aMediaTrack) { return aMediaTrack->GraphImpl()->ConnectToCaptureTrack(aWindowId, aMediaTrack); } already_AddRefed MediaTrackGraphImpl::ConnectToCaptureTrack( uint64_t aWindowId, MediaTrack* aMediaTrack) { MOZ_ASSERT(NS_IsMainThread()); for (uint32_t i = 0; i < mWindowCaptureTracks.Length(); i++) { if (mWindowCaptureTracks[i].mWindowId == aWindowId) { ProcessedMediaTrack* sink = mWindowCaptureTracks[i].mCaptureTrackSink; return sink->AllocateInputPort(aMediaTrack); } } return nullptr; } void MediaTrackGraph::DispatchToMainThreadStableState( already_AddRefed aRunnable) { AssertOnGraphThreadOrNotRunning(); static_cast(this) ->mPendingUpdateRunnables.AppendElement(std::move(aRunnable)); } Watchable& MediaTrackGraphImpl::CurrentTime() { MOZ_ASSERT(NS_IsMainThread()); return mMainThreadGraphTime; } GraphTime MediaTrackGraph::ProcessedTime() const { AssertOnGraphThreadOrNotRunning(); return static_cast(this)->mProcessedTime; } void* MediaTrackGraph::CurrentDriver() const { AssertOnGraphThreadOrNotRunning(); return static_cast(this)->mDriver; } uint32_t MediaTrackGraphImpl::AudioInputChannelCount( CubebUtils::AudioDeviceID aID) { MOZ_ASSERT(OnGraphThreadOrNotRunning()); DeviceInputTrack* t = mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID); return t ? t->MaxRequestedInputChannels() : 0; } AudioInputType MediaTrackGraphImpl::AudioInputDevicePreference( CubebUtils::AudioDeviceID aID) { MOZ_ASSERT(OnGraphThreadOrNotRunning()); DeviceInputTrack* t = mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID); return t && t->HasVoiceInput() ? AudioInputType::Voice : AudioInputType::Unknown; } cubeb_input_processing_params MediaTrackGraphImpl::RequestedAudioInputProcessingParams( CubebUtils::AudioDeviceID aID) { MOZ_ASSERT(OnGraphThreadOrNotRunning()); DeviceInputTrack* t = mDeviceInputTrackManagerGraphThread.GetDeviceInputTrack(aID); return t ? t->RequestedProcessingParams() : CUBEB_INPUT_PROCESSING_PARAM_NONE; } void MediaTrackGraphImpl::SetNewNativeInput() { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(!mDeviceInputTrackManagerMainThread.GetNativeInputTrack()); LOG(LogLevel::Debug, ("%p SetNewNativeInput", this)); NonNativeInputTrack* track = mDeviceInputTrackManagerMainThread.GetFirstNonNativeInputTrack(); if (!track) { LOG(LogLevel::Debug, ("%p No other devices opened. Do nothing", this)); return; } const CubebUtils::AudioDeviceID deviceId = track->mDeviceId; const PrincipalHandle principal = track->mPrincipalHandle; LOG(LogLevel::Debug, ("%p Select device %p as the new native input device", this, deviceId)); struct TrackListener { DeviceInputConsumerTrack* track; // Keep its reference so it won't be dropped when after // DisconnectDeviceInput(). RefPtr listener; }; nsTArray pairs; for (const auto& t : track->GetConsumerTracks()) { pairs.AppendElement( TrackListener{t.get(), t->GetAudioDataListener().get()}); } for (TrackListener& pair : pairs) { pair.track->DisconnectDeviceInput(); } for (TrackListener& pair : pairs) { pair.track->ConnectDeviceInput(deviceId, pair.listener.get(), principal); LOG(LogLevel::Debug, ("%p: Reinitialize AudioProcessingTrack %p for device %p", this, pair.track, deviceId)); } LOG(LogLevel::Debug, ("%p Native input device is set to device %p now", this, deviceId)); MOZ_ASSERT(mDeviceInputTrackManagerMainThread.GetNativeInputTrack()); } // nsIThreadObserver methods NS_IMETHODIMP MediaTrackGraphImpl::OnDispatchedEvent() { MonitorAutoLock lock(mMonitor); EnsureNextIteration(); return NS_OK; } NS_IMETHODIMP MediaTrackGraphImpl::OnProcessNextEvent(nsIThreadInternal*, bool) { return NS_OK; } NS_IMETHODIMP MediaTrackGraphImpl::AfterProcessNextEvent(nsIThreadInternal*, bool) { return NS_OK; } } // namespace mozilla