diff options
Diffstat (limited to '')
| -rw-r--r-- | dom/media/webaudio/ScriptProcessorNode.cpp | 549 |
1 files changed, 549 insertions, 0 deletions
diff --git a/dom/media/webaudio/ScriptProcessorNode.cpp b/dom/media/webaudio/ScriptProcessorNode.cpp new file mode 100644 index 0000000000..aafbec4202 --- /dev/null +++ b/dom/media/webaudio/ScriptProcessorNode.cpp @@ -0,0 +1,549 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ScriptProcessorNode.h" +#include "mozilla/dom/ScriptProcessorNodeBinding.h" +#include "AudioBuffer.h" +#include "AudioDestinationNode.h" +#include "AudioNodeEngine.h" +#include "AudioNodeTrack.h" +#include "AudioProcessingEvent.h" +#include "WebAudioUtils.h" +#include "mozilla/dom/ScriptSettings.h" +#include "mozilla/Mutex.h" +#include "mozilla/PodOperations.h" +#include <deque> +#include "Tracing.h" + +namespace mozilla::dom { + +// The maximum latency, in seconds, that we can live with before dropping +// buffers. +static const float MAX_LATENCY_S = 0.5; + +// This class manages a queue of output buffers shared between +// the main thread and the Media Track Graph thread. +class SharedBuffers final { + private: + class OutputQueue final { + public: + explicit OutputQueue(const char* aName) : mMutex(aName) {} + + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const + MOZ_REQUIRES(mMutex) { + mMutex.AssertCurrentThreadOwns(); + + size_t amount = 0; + for (size_t i = 0; i < mBufferList.size(); i++) { + amount += mBufferList[i].SizeOfExcludingThis(aMallocSizeOf, false); + } + + return amount; + } + + Mutex& Lock() const MOZ_RETURN_CAPABILITY(mMutex) { + return const_cast<OutputQueue*>(this)->mMutex; + } + + size_t ReadyToConsume() const MOZ_REQUIRES(mMutex) { + // Accessed on both main thread and media graph thread. + mMutex.AssertCurrentThreadOwns(); + return mBufferList.size(); + } + + // Produce one buffer + AudioChunk& Produce() MOZ_REQUIRES(mMutex) { + mMutex.AssertCurrentThreadOwns(); + MOZ_ASSERT(NS_IsMainThread()); + mBufferList.push_back(AudioChunk()); + return mBufferList.back(); + } + + // Consumes one buffer. + AudioChunk Consume() MOZ_REQUIRES(mMutex) { + mMutex.AssertCurrentThreadOwns(); + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_ASSERT(ReadyToConsume() > 0); + AudioChunk front = mBufferList.front(); + mBufferList.pop_front(); + return front; + } + + // Empties the buffer queue. + void Clear() MOZ_REQUIRES(mMutex) { + mMutex.AssertCurrentThreadOwns(); + mBufferList.clear(); + } + + private: + typedef std::deque<AudioChunk> BufferList; + + // Synchronizes access to mBufferList. Note that it's the responsibility + // of the callers to perform the required locking, and we assert that every + // time we access mBufferList. + Mutex mMutex MOZ_UNANNOTATED; + // The list representing the queue. + BufferList mBufferList; + }; + + public: + explicit SharedBuffers(float aSampleRate) + : mOutputQueue("SharedBuffers::outputQueue"), + mDelaySoFar(TRACK_TIME_MAX), + mSampleRate(aSampleRate), + mLatency(0.0), + mDroppingBuffers(false) {} + + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { + size_t amount = aMallocSizeOf(this); + + { + MutexAutoLock lock(mOutputQueue.Lock()); + amount += mOutputQueue.SizeOfExcludingThis(aMallocSizeOf); + } + + return amount; + } + + // main thread + + // NotifyNodeIsConnected() may be called even when the state has not + // changed. + void NotifyNodeIsConnected(bool aIsConnected) { + MOZ_ASSERT(NS_IsMainThread()); + if (!aIsConnected) { + // Reset main thread state for FinishProducingOutputBuffer(). + mLatency = 0.0f; + mLastEventTime = TimeStamp(); + mDroppingBuffers = false; + // Don't flush the output buffer here because the graph thread may be + // using it now. The graph thread will flush when it knows it is + // disconnected. + } + mNodeIsConnected = aIsConnected; + } + + void FinishProducingOutputBuffer(const AudioChunk& aBuffer) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!mNodeIsConnected) { + // The output buffer is not used, and mLastEventTime will not be + // initialized until the node is re-connected. + return; + } + + TimeStamp now = TimeStamp::Now(); + + if (mLastEventTime.IsNull()) { + mLastEventTime = now; + } else { + // When main thread blocking has built up enough so + // |mLatency > MAX_LATENCY_S|, frame dropping starts. It continues until + // the output buffer is completely empty, at which point the accumulated + // latency is also reset to 0. + // It could happen that the output queue becomes empty before the input + // node has fully caught up. In this case there will be events where + // |(now - mLastEventTime)| is very short, making mLatency negative. + // As this happens and the size of |mLatency| becomes greater than + // MAX_LATENCY_S, frame dropping starts again to maintain an as short + // output queue as possible. + float latency = (now - mLastEventTime).ToSeconds(); + float bufferDuration = aBuffer.mDuration / mSampleRate; + mLatency += latency - bufferDuration; + mLastEventTime = now; + if (fabs(mLatency) > MAX_LATENCY_S) { + mDroppingBuffers = true; + } + } + + MutexAutoLock lock(mOutputQueue.Lock()); + if (mDroppingBuffers) { + if (mOutputQueue.ReadyToConsume()) { + return; + } + mDroppingBuffers = false; + mLatency = 0; + } + + for (uint32_t offset = 0; offset < aBuffer.mDuration; + offset += WEBAUDIO_BLOCK_SIZE) { + AudioChunk& chunk = mOutputQueue.Produce(); + chunk = aBuffer; + chunk.SliceTo(offset, offset + WEBAUDIO_BLOCK_SIZE); + } + } + + // graph thread + + AudioChunk GetOutputBuffer() { + MOZ_ASSERT(!NS_IsMainThread()); + AudioChunk buffer; + + { + MutexAutoLock lock(mOutputQueue.Lock()); + if (mOutputQueue.ReadyToConsume() > 0) { + if (mDelaySoFar == TRACK_TIME_MAX) { + mDelaySoFar = 0; + } + buffer = mOutputQueue.Consume(); + } else { + // If we're out of buffers to consume, just output silence + buffer.SetNull(WEBAUDIO_BLOCK_SIZE); + if (mDelaySoFar != TRACK_TIME_MAX) { + // Remember the delay that we just hit + mDelaySoFar += WEBAUDIO_BLOCK_SIZE; + } + } + } + + return buffer; + } + + TrackTime DelaySoFar() const { + MOZ_ASSERT(!NS_IsMainThread()); + return mDelaySoFar == TRACK_TIME_MAX ? 0 : mDelaySoFar; + } + + void Flush() { + MOZ_ASSERT(!NS_IsMainThread()); + mDelaySoFar = TRACK_TIME_MAX; + { + MutexAutoLock lock(mOutputQueue.Lock()); + mOutputQueue.Clear(); + } + } + + private: + OutputQueue mOutputQueue; + // How much delay we've seen so far. This measures the amount of delay + // caused by the main thread lagging behind in producing output buffers. + // TRACK_TIME_MAX means that we have not received our first buffer yet. + // Graph thread only. + TrackTime mDelaySoFar; + // The samplerate of the context. + const float mSampleRate; + // The remaining members are main thread only. + // This is the latency caused by the buffering. If this grows too high, we + // will drop buffers until it is acceptable. + float mLatency; + // This is the time at which we last produced a buffer, to detect if the main + // thread has been blocked. + TimeStamp mLastEventTime; + // True if we should be dropping buffers. + bool mDroppingBuffers; + // True iff the AudioNode has at least one input or output connected. + bool mNodeIsConnected; +}; + +class ScriptProcessorNodeEngine final : public AudioNodeEngine { + public: + ScriptProcessorNodeEngine(ScriptProcessorNode* aNode, + AudioDestinationNode* aDestination, + uint32_t aBufferSize, + uint32_t aNumberOfInputChannels) + : AudioNodeEngine(aNode), + mDestination(aDestination->Track()), + mSharedBuffers(new SharedBuffers(mDestination->mSampleRate)), + mBufferSize(aBufferSize), + mInputChannelCount(aNumberOfInputChannels), + mInputWriteIndex(0) {} + + SharedBuffers* GetSharedBuffers() const { return mSharedBuffers.get(); } + + enum { + IS_CONNECTED, + }; + + void SetInt32Parameter(uint32_t aIndex, int32_t aParam) override { + switch (aIndex) { + case IS_CONNECTED: + mIsConnected = aParam; + break; + default: + NS_ERROR("Bad Int32Parameter"); + } // End index switch. + } + + void ProcessBlock(AudioNodeTrack* aTrack, GraphTime aFrom, + const AudioBlock& aInput, AudioBlock* aOutput, + bool* aFinished) override { + TRACE("ScriptProcessorNodeEngine::ProcessBlock"); + + // This node is not connected to anything. Per spec, we don't fire the + // onaudioprocess event. We also want to clear out the input and output + // buffer queue, and output a null buffer. + if (!mIsConnected) { + aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); + mSharedBuffers->Flush(); + mInputWriteIndex = 0; + return; + } + + // The input buffer is allocated lazily when non-null input is received. + if (!aInput.IsNull() && !mInputBuffer) { + mInputBuffer = ThreadSharedFloatArrayBufferList::Create( + mInputChannelCount, mBufferSize, fallible); + if (mInputBuffer && mInputWriteIndex) { + // Zero leading for null chunks that were skipped. + for (uint32_t i = 0; i < mInputChannelCount; ++i) { + float* channelData = mInputBuffer->GetDataForWrite(i); + PodZero(channelData, mInputWriteIndex); + } + } + } + + // First, record our input buffer, if its allocation succeeded. + uint32_t inputChannelCount = mInputBuffer ? mInputBuffer->GetChannels() : 0; + for (uint32_t i = 0; i < inputChannelCount; ++i) { + float* writeData = mInputBuffer->GetDataForWrite(i) + mInputWriteIndex; + if (aInput.IsNull()) { + PodZero(writeData, aInput.GetDuration()); + } else { + MOZ_ASSERT(aInput.GetDuration() == WEBAUDIO_BLOCK_SIZE, "sanity check"); + MOZ_ASSERT(aInput.ChannelCount() == inputChannelCount); + AudioBlockCopyChannelWithScale( + static_cast<const float*>(aInput.mChannelData[i]), aInput.mVolume, + writeData); + } + } + mInputWriteIndex += aInput.GetDuration(); + + // Now, see if we have data to output + // Note that we need to do this before sending the buffer to the main + // thread so that our delay time is updated. + *aOutput = mSharedBuffers->GetOutputBuffer(); + + if (mInputWriteIndex >= mBufferSize) { + SendBuffersToMainThread(aTrack, aFrom); + mInputWriteIndex -= mBufferSize; + } + } + + bool IsActive() const override { + // Could return false when !mIsConnected after all output chunks produced + // by main thread events calling + // SharedBuffers::FinishProducingOutputBuffer() have been processed. + return true; + } + + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { + // Not owned: + // - mDestination (probably) + size_t amount = AudioNodeEngine::SizeOfExcludingThis(aMallocSizeOf); + amount += mSharedBuffers->SizeOfIncludingThis(aMallocSizeOf); + if (mInputBuffer) { + amount += mInputBuffer->SizeOfIncludingThis(aMallocSizeOf); + } + + return amount; + } + + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override { + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); + } + + private: + void SendBuffersToMainThread(AudioNodeTrack* aTrack, GraphTime aFrom) { + MOZ_ASSERT(!NS_IsMainThread()); + + // we now have a full input buffer ready to be sent to the main thread. + TrackTime playbackTick = mDestination->GraphTimeToTrackTime(aFrom); + // Add the duration of the current sample + playbackTick += WEBAUDIO_BLOCK_SIZE; + // Add the delay caused by the main thread + playbackTick += mSharedBuffers->DelaySoFar(); + // Compute the playback time in the coordinate system of the destination + double playbackTime = mDestination->TrackTimeToSeconds(playbackTick); + + class Command final : public Runnable { + public: + Command(AudioNodeTrack* aTrack, + already_AddRefed<ThreadSharedFloatArrayBufferList> aInputBuffer, + double aPlaybackTime) + : mozilla::Runnable("Command"), + mTrack(aTrack), + mInputBuffer(aInputBuffer), + mPlaybackTime(aPlaybackTime) {} + + NS_IMETHOD Run() override { + auto engine = static_cast<ScriptProcessorNodeEngine*>(mTrack->Engine()); + AudioChunk output; + output.SetNull(engine->mBufferSize); + { + auto node = + static_cast<ScriptProcessorNode*>(engine->NodeMainThread()); + if (!node) { + return NS_OK; + } + + if (node->HasListenersFor(nsGkAtoms::onaudioprocess)) { + DispatchAudioProcessEvent(node, &output); + } + // The node may have been destroyed during event dispatch. + } + + // Append it to our output buffer queue + engine->GetSharedBuffers()->FinishProducingOutputBuffer(output); + + return NS_OK; + } + + // Sets up |output| iff buffers are set in event handlers. + void DispatchAudioProcessEvent(ScriptProcessorNode* aNode, + AudioChunk* aOutput) { + AudioContext* context = aNode->Context(); + if (!context) { + return; + } + + AutoJSAPI jsapi; + if (NS_WARN_IF(!jsapi.Init(aNode->GetOwner()))) { + return; + } + JSContext* cx = jsapi.cx(); + uint32_t inputChannelCount = aNode->ChannelCount(); + + // Create the input buffer + RefPtr<AudioBuffer> inputBuffer; + if (mInputBuffer) { + ErrorResult rv; + inputBuffer = AudioBuffer::Create( + context->GetOwner(), inputChannelCount, aNode->BufferSize(), + context->SampleRate(), mInputBuffer.forget(), rv); + if (rv.Failed()) { + rv.SuppressException(); + return; + } + } + + // Ask content to produce data in the output buffer + // Note that we always avoid creating the output buffer here, and we try + // to avoid creating the input buffer as well. The AudioProcessingEvent + // class knows how to lazily create them if needed once the script tries + // to access them. Otherwise, we may be able to get away without + // creating them! + RefPtr<AudioProcessingEvent> event = + new AudioProcessingEvent(aNode, nullptr, nullptr); + event->InitEvent(inputBuffer, inputChannelCount, mPlaybackTime); + aNode->DispatchTrustedEvent(event); + + // Steal the output buffers if they have been set. + // Don't create a buffer if it hasn't been used to return output; + // FinishProducingOutputBuffer() will optimize output = null. + // GetThreadSharedChannelsForRate() may also return null after OOM. + if (event->HasOutputBuffer()) { + ErrorResult rv; + AudioBuffer* buffer = event->GetOutputBuffer(rv); + // HasOutputBuffer() returning true means that GetOutputBuffer() + // will not fail. + MOZ_ASSERT(!rv.Failed()); + *aOutput = buffer->GetThreadSharedChannelsForRate(cx); + MOZ_ASSERT(aOutput->IsNull() || + aOutput->mBufferFormat == AUDIO_FORMAT_FLOAT32, + "AudioBuffers initialized from JS have float data"); + } + } + + private: + RefPtr<AudioNodeTrack> mTrack; + RefPtr<ThreadSharedFloatArrayBufferList> mInputBuffer; + double mPlaybackTime; + }; + + RefPtr<Command> command = + new Command(aTrack, mInputBuffer.forget(), playbackTime); + mAbstractMainThread->Dispatch(command.forget()); + } + + friend class ScriptProcessorNode; + + RefPtr<AudioNodeTrack> mDestination; + UniquePtr<SharedBuffers> mSharedBuffers; + RefPtr<ThreadSharedFloatArrayBufferList> mInputBuffer; + const uint32_t mBufferSize; + const uint32_t mInputChannelCount; + // The write index into the current input buffer + uint32_t mInputWriteIndex; + bool mIsConnected = false; +}; + +ScriptProcessorNode::ScriptProcessorNode(AudioContext* aContext, + uint32_t aBufferSize, + uint32_t aNumberOfInputChannels, + uint32_t aNumberOfOutputChannels) + : AudioNode(aContext, aNumberOfInputChannels, + mozilla::dom::ChannelCountMode::Explicit, + mozilla::dom::ChannelInterpretation::Speakers), + mBufferSize(aBufferSize ? aBufferSize + : // respect what the web developer requested + 4096) // choose our own buffer size -- 4KB for now + , + mNumberOfOutputChannels(aNumberOfOutputChannels) { + MOZ_ASSERT(BufferSize() % WEBAUDIO_BLOCK_SIZE == 0, "Invalid buffer size"); + ScriptProcessorNodeEngine* engine = new ScriptProcessorNodeEngine( + this, aContext->Destination(), BufferSize(), aNumberOfInputChannels); + mTrack = AudioNodeTrack::Create( + aContext, engine, AudioNodeTrack::NO_TRACK_FLAGS, aContext->Graph()); +} + +ScriptProcessorNode::~ScriptProcessorNode() = default; + +size_t ScriptProcessorNode::SizeOfExcludingThis( + MallocSizeOf aMallocSizeOf) const { + size_t amount = AudioNode::SizeOfExcludingThis(aMallocSizeOf); + return amount; +} + +size_t ScriptProcessorNode::SizeOfIncludingThis( + MallocSizeOf aMallocSizeOf) const { + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); +} + +void ScriptProcessorNode::EventListenerAdded(nsAtom* aType) { + AudioNode::EventListenerAdded(aType); + if (aType == nsGkAtoms::onaudioprocess) { + UpdateConnectedStatus(); + } +} + +void ScriptProcessorNode::EventListenerRemoved(nsAtom* aType) { + AudioNode::EventListenerRemoved(aType); + if (aType == nsGkAtoms::onaudioprocess && mTrack) { + UpdateConnectedStatus(); + } +} + +JSObject* ScriptProcessorNode::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return ScriptProcessorNode_Binding::Wrap(aCx, this, aGivenProto); +} + +void ScriptProcessorNode::UpdateConnectedStatus() { + bool isConnected = + mHasPhantomInput || !(OutputNodes().IsEmpty() && + OutputParams().IsEmpty() && InputNodes().IsEmpty()); + + // Events are queued even when there is no listener because a listener + // may be added while events are in the queue. + SendInt32ParameterToTrack(ScriptProcessorNodeEngine::IS_CONNECTED, + isConnected); + + if (isConnected && HasListenersFor(nsGkAtoms::onaudioprocess)) { + MarkActive(); + } else { + MarkInactive(); + } + + // MarkInactive above might have released this node, check if it has a track. + if (!mTrack) { + return; + } + + auto engine = static_cast<ScriptProcessorNodeEngine*>(mTrack->Engine()); + engine->GetSharedBuffers()->NotifyNodeIsConnected(isConnected); +} + +} // namespace mozilla::dom |