diff options
Diffstat (limited to 'dom/media/AudioSegment.h')
| -rw-r--r-- | dom/media/AudioSegment.h | 539 |
1 files changed, 539 insertions, 0 deletions
diff --git a/dom/media/AudioSegment.h b/dom/media/AudioSegment.h new file mode 100644 index 0000000000..a13db6fec6 --- /dev/null +++ b/dom/media/AudioSegment.h @@ -0,0 +1,539 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_AUDIOSEGMENT_H_ +#define MOZILLA_AUDIOSEGMENT_H_ + +#include <speex/speex_resampler.h> +#include "MediaTrackGraph.h" +#include "MediaSegment.h" +#include "AudioSampleFormat.h" +#include "AudioChannelFormat.h" +#include "SharedBuffer.h" +#include "WebAudioUtils.h" +#include "mozilla/ScopeExit.h" +#include "nsAutoRef.h" +#ifdef MOZILLA_INTERNAL_API +# include "mozilla/TimeStamp.h" +#endif +#include <float.h> + +namespace mozilla { +struct AudioChunk; +class AudioSegment; +} // namespace mozilla +MOZ_DECLARE_RELOCATE_USING_MOVE_CONSTRUCTOR(mozilla::AudioChunk) + +/** + * This allows compilation of nsTArray<AudioSegment> and + * AutoTArray<AudioSegment> since without it, static analysis fails on the + * mChunks member being a non-memmovable AutoTArray. + * + * Note that AudioSegment(const AudioSegment&) is deleted, so this should + * never come into effect. + */ +MOZ_DECLARE_RELOCATE_USING_MOVE_CONSTRUCTOR(mozilla::AudioSegment) + +namespace mozilla { + +template <typename T> +class SharedChannelArrayBuffer : public ThreadSharedObject { + public: + explicit SharedChannelArrayBuffer(nsTArray<nsTArray<T> >&& aBuffers) + : mBuffers(std::move(aBuffers)) {} + + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { + size_t amount = 0; + amount += mBuffers.ShallowSizeOfExcludingThis(aMallocSizeOf); + for (size_t i = 0; i < mBuffers.Length(); i++) { + amount += mBuffers[i].ShallowSizeOfExcludingThis(aMallocSizeOf); + } + + return amount; + } + + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override { + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); + } + + nsTArray<nsTArray<T> > mBuffers; +}; + +class AudioMixer; + +/** + * For auto-arrays etc, guess this as the common number of channels. + */ +const int GUESS_AUDIO_CHANNELS = 2; + +// We ensure that the graph advances in steps that are multiples of the Web +// Audio block size +const uint32_t WEBAUDIO_BLOCK_SIZE_BITS = 7; +const uint32_t WEBAUDIO_BLOCK_SIZE = 1 << WEBAUDIO_BLOCK_SIZE_BITS; + +template <typename SrcT, typename DestT> +static void InterleaveAndConvertBuffer(const SrcT* const* aSourceChannels, + uint32_t aLength, float aVolume, + uint32_t aChannels, DestT* aOutput) { + DestT* output = aOutput; + for (size_t i = 0; i < aLength; ++i) { + for (size_t channel = 0; channel < aChannels; ++channel) { + float v = AudioSampleToFloat(aSourceChannels[channel][i]) * aVolume; + *output = FloatToAudioSample<DestT>(v); + ++output; + } + } +} + +template <typename SrcT, typename DestT> +static void DeinterleaveAndConvertBuffer(const SrcT* aSourceBuffer, + uint32_t aFrames, uint32_t aChannels, + DestT** aOutput) { + for (size_t i = 0; i < aChannels; i++) { + size_t interleavedIndex = i; + for (size_t j = 0; j < aFrames; j++) { + ConvertAudioSample(aSourceBuffer[interleavedIndex], aOutput[i][j]); + interleavedIndex += aChannels; + } + } +} + +class SilentChannel { + public: + static const int AUDIO_PROCESSING_FRAMES = 640; /* > 10ms of 48KHz audio */ + static const uint8_t + gZeroChannel[MAX_AUDIO_SAMPLE_SIZE * AUDIO_PROCESSING_FRAMES]; + // We take advantage of the fact that zero in float and zero in int have the + // same all-zeros bit layout. + template <typename T> + static const T* ZeroChannel(); +}; + +/** + * Given an array of input channels (aChannelData), downmix to aOutputChannels, + * interleave the channel data. A total of aOutputChannels*aDuration + * interleaved samples will be copied to a channel buffer in aOutput. + */ +template <typename SrcT, typename DestT> +void DownmixAndInterleave(const nsTArray<const SrcT*>& aChannelData, + int32_t aDuration, float aVolume, + uint32_t aOutputChannels, DestT* aOutput) { + if (aChannelData.Length() == aOutputChannels) { + InterleaveAndConvertBuffer(aChannelData.Elements(), aDuration, aVolume, + aOutputChannels, aOutput); + } else { + AutoTArray<SrcT*, GUESS_AUDIO_CHANNELS> outputChannelData; + AutoTArray<SrcT, + SilentChannel::AUDIO_PROCESSING_FRAMES * GUESS_AUDIO_CHANNELS> + outputBuffers; + outputChannelData.SetLength(aOutputChannels); + outputBuffers.SetLength(aDuration * aOutputChannels); + for (uint32_t i = 0; i < aOutputChannels; i++) { + outputChannelData[i] = outputBuffers.Elements() + aDuration * i; + } + AudioChannelsDownMix(aChannelData, outputChannelData.Elements(), + aOutputChannels, aDuration); + InterleaveAndConvertBuffer(outputChannelData.Elements(), aDuration, aVolume, + aOutputChannels, aOutput); + } +} + +/** + * An AudioChunk represents a multi-channel buffer of audio samples. + * It references an underlying ThreadSharedObject which manages the lifetime + * of the buffer. An AudioChunk maintains its own duration and channel data + * pointers so it can represent a subinterval of a buffer without copying. + * An AudioChunk can store its individual channels anywhere; it maintains + * separate pointers to each channel's buffer. + */ +struct AudioChunk { + typedef mozilla::AudioSampleFormat SampleFormat; + + AudioChunk() = default; + + template <typename T> + AudioChunk(already_AddRefed<ThreadSharedObject> aBuffer, + const nsTArray<const T*>& aChannelData, TrackTime aDuration, + PrincipalHandle aPrincipalHandle) + : mDuration(aDuration), + mBuffer(aBuffer), + mBufferFormat(AudioSampleTypeToFormat<T>::Format), + mPrincipalHandle(std::move(aPrincipalHandle)) { + MOZ_ASSERT(!mBuffer == aChannelData.IsEmpty(), "Appending invalid data ?"); + for (const T* data : aChannelData) { + mChannelData.AppendElement(data); + } + } + + // Generic methods + void SliceTo(TrackTime aStart, TrackTime aEnd) { + MOZ_ASSERT(aStart >= 0, "Slice out of bounds: invalid start"); + MOZ_ASSERT(aStart < aEnd, "Slice out of bounds: invalid range"); + MOZ_ASSERT(aEnd <= mDuration, "Slice out of bounds: invalid end"); + + if (mBuffer) { + MOZ_ASSERT(aStart < INT32_MAX, + "Can't slice beyond 32-bit sample lengths"); + for (uint32_t channel = 0; channel < mChannelData.Length(); ++channel) { + mChannelData[channel] = AddAudioSampleOffset( + mChannelData[channel], mBufferFormat, int32_t(aStart)); + } + } + mDuration = aEnd - aStart; + } + TrackTime GetDuration() const { return mDuration; } + bool CanCombineWithFollowing(const AudioChunk& aOther) const { + if (aOther.mBuffer != mBuffer) { + return false; + } + if (!mBuffer) { + return true; + } + if (aOther.mVolume != mVolume) { + return false; + } + if (aOther.mPrincipalHandle != mPrincipalHandle) { + return false; + } + NS_ASSERTION(aOther.mBufferFormat == mBufferFormat, + "Wrong metadata about buffer"); + NS_ASSERTION(aOther.mChannelData.Length() == mChannelData.Length(), + "Mismatched channel count"); + if (mDuration > INT32_MAX) { + return false; + } + for (uint32_t channel = 0; channel < mChannelData.Length(); ++channel) { + if (aOther.mChannelData[channel] != + AddAudioSampleOffset(mChannelData[channel], mBufferFormat, + int32_t(mDuration))) { + return false; + } + } + return true; + } + bool IsNull() const { return mBuffer == nullptr; } + void SetNull(TrackTime aDuration) { + mBuffer = nullptr; + mChannelData.Clear(); + mDuration = aDuration; + mVolume = 1.0f; + mBufferFormat = AUDIO_FORMAT_SILENCE; + mPrincipalHandle = PRINCIPAL_HANDLE_NONE; + } + + uint32_t ChannelCount() const { return mChannelData.Length(); } + + bool IsMuted() const { return mVolume == 0.0f; } + + size_t SizeOfExcludingThisIfUnshared(MallocSizeOf aMallocSizeOf) const { + return SizeOfExcludingThis(aMallocSizeOf, true); + } + + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf, bool aUnshared) const { + size_t amount = 0; + + // Possibly owned: + // - mBuffer - Can hold data that is also in the decoded audio queue. If it + // is not shared, or unshared == false it gets counted. + if (mBuffer && (!aUnshared || !mBuffer->IsShared())) { + amount += mBuffer->SizeOfIncludingThis(aMallocSizeOf); + } + + // Memory in the array is owned by mBuffer. + amount += mChannelData.ShallowSizeOfExcludingThis(aMallocSizeOf); + return amount; + } + + template <typename T> + const nsTArray<const T*>& ChannelData() const { + MOZ_ASSERT(AudioSampleTypeToFormat<T>::Format == mBufferFormat); + return *reinterpret_cast<const AutoTArray<const T*, GUESS_AUDIO_CHANNELS>*>( + &mChannelData); + } + + /** + * ChannelFloatsForWrite() should be used only when mBuffer is owned solely + * by the calling thread. + */ + template <typename T> + T* ChannelDataForWrite(size_t aChannel) { + MOZ_ASSERT(AudioSampleTypeToFormat<T>::Format == mBufferFormat); + MOZ_ASSERT(!mBuffer->IsShared()); + return static_cast<T*>(const_cast<void*>(mChannelData[aChannel])); + } + + template <typename T> + static AudioChunk FromInterleavedBuffer( + const T* aBuffer, size_t aFrames, uint32_t aChannels, + const PrincipalHandle& aPrincipalHandle) { + CheckedInt<size_t> bufferSize(sizeof(T)); + bufferSize *= aFrames; + bufferSize *= aChannels; + RefPtr<SharedBuffer> buffer = SharedBuffer::Create(bufferSize); + + AutoTArray<T*, 8> deinterleaved; + if (aChannels == 1) { + PodCopy(static_cast<T*>(buffer->Data()), aBuffer, aFrames); + deinterleaved.AppendElement(static_cast<T*>(buffer->Data())); + } else { + deinterleaved.SetLength(aChannels); + T* samples = static_cast<T*>(buffer->Data()); + + size_t offset = 0; + for (uint32_t i = 0; i < aChannels; ++i) { + deinterleaved[i] = samples + offset; + offset += aFrames; + } + + DeinterleaveAndConvertBuffer(aBuffer, static_cast<uint32_t>(aFrames), + aChannels, deinterleaved.Elements()); + } + + AutoTArray<const T*, GUESS_AUDIO_CHANNELS> channelData; + channelData.AppendElements(deinterleaved); + return AudioChunk(buffer.forget(), channelData, + static_cast<TrackTime>(aFrames), aPrincipalHandle); + } + + const PrincipalHandle& GetPrincipalHandle() const { return mPrincipalHandle; } + + TrackTime mDuration = 0; // in frames within the buffer + RefPtr<ThreadSharedObject> mBuffer; // the buffer object whose lifetime is + // managed; null means data is all zeroes + // one pointer per channel; empty if and only if mBuffer is null + CopyableAutoTArray<const void*, GUESS_AUDIO_CHANNELS> mChannelData; + float mVolume = 1.0f; // volume multiplier to apply + // format of frames in mBuffer (or silence if mBuffer is null) + SampleFormat mBufferFormat = AUDIO_FORMAT_SILENCE; + // principalHandle for the data in this chunk. + // This can be compared to an nsIPrincipal* when back on main thread. + PrincipalHandle mPrincipalHandle = PRINCIPAL_HANDLE_NONE; +}; + +/** + * A list of audio samples consisting of a sequence of slices of SharedBuffers. + * The audio rate is determined by the track, not stored in this class. + */ +class AudioSegment : public MediaSegmentBase<AudioSegment, AudioChunk> { + // The channel count that MaxChannelCount() returned last time it was called. + uint32_t mMemoizedMaxChannelCount = 0; + + public: + typedef mozilla::AudioSampleFormat SampleFormat; + + AudioSegment() : MediaSegmentBase<AudioSegment, AudioChunk>(AUDIO) {} + + AudioSegment(AudioSegment&& aSegment) = default; + + AudioSegment(const AudioSegment&) = delete; + AudioSegment& operator=(const AudioSegment&) = delete; + + ~AudioSegment() = default; + + // Resample the whole segment in place. `aResampler` is an instance of a + // resampler, initialized with `aResamplerChannelCount` channels. If this + // function finds a chunk with more channels, `aResampler` is destroyed and a + // new resampler is created, and `aResamplerChannelCount` is updated with the + // new channel count value. + template <typename T> + void Resample(nsAutoRef<SpeexResamplerState>& aResampler, + uint32_t* aResamplerChannelCount, uint32_t aInRate, + uint32_t aOutRate) { + mDuration = 0; + + for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) { + AutoTArray<nsTArray<T>, GUESS_AUDIO_CHANNELS> output; + AutoTArray<const T*, GUESS_AUDIO_CHANNELS> bufferPtrs; + AudioChunk& c = *ci; + // If this chunk is null, don't bother resampling, just alter its duration + if (c.IsNull()) { + c.mDuration = (c.mDuration * aOutRate) / aInRate; + mDuration += c.mDuration; + continue; + } + uint32_t channels = c.mChannelData.Length(); + // This might introduce a discontinuity, but a channel count change in the + // middle of a stream is not that common. This also initializes the + // resampler as late as possible. + if (channels != *aResamplerChannelCount) { + SpeexResamplerState* state = + speex_resampler_init(channels, aInRate, aOutRate, + SPEEX_RESAMPLER_QUALITY_DEFAULT, nullptr); + MOZ_ASSERT(state); + aResampler.own(state); + *aResamplerChannelCount = channels; + } + output.SetLength(channels); + bufferPtrs.SetLength(channels); + uint32_t inFrames = c.mDuration; + // Round up to allocate; the last frame may not be used. + NS_ASSERTION((UINT64_MAX - aInRate + 1) / c.mDuration >= aOutRate, + "Dropping samples"); + uint32_t outSize = + (static_cast<uint64_t>(c.mDuration) * aOutRate + aInRate - 1) / + aInRate; + for (uint32_t i = 0; i < channels; i++) { + T* out = output[i].AppendElements(outSize); + uint32_t outFrames = outSize; + + const T* in = static_cast<const T*>(c.mChannelData[i]); + dom::WebAudioUtils::SpeexResamplerProcess(aResampler.get(), i, in, + &inFrames, out, &outFrames); + MOZ_ASSERT(inFrames == c.mDuration); + + bufferPtrs[i] = out; + output[i].SetLength(outFrames); + } + MOZ_ASSERT(channels > 0); + c.mDuration = output[0].Length(); + c.mBuffer = new mozilla::SharedChannelArrayBuffer<T>(std::move(output)); + for (uint32_t i = 0; i < channels; i++) { + c.mChannelData[i] = bufferPtrs[i]; + } + mDuration += c.mDuration; + } + } + + void ResampleChunks(nsAutoRef<SpeexResamplerState>& aResampler, + uint32_t* aResamplerChannelCount, uint32_t aInRate, + uint32_t aOutRate); + + template <typename T> + void AppendFrames(already_AddRefed<ThreadSharedObject> aBuffer, + const nsTArray<const T*>& aChannelData, TrackTime aDuration, + const PrincipalHandle& aPrincipalHandle) { + AppendAndConsumeChunk(AudioChunk(std::move(aBuffer), aChannelData, + aDuration, aPrincipalHandle)); + } + void AppendSegment(const AudioSegment* aSegment) { + MOZ_ASSERT(aSegment); + + for (const AudioChunk& c : aSegment->mChunks) { + AudioChunk* chunk = AppendChunk(c.GetDuration()); + chunk->mBuffer = c.mBuffer; + chunk->mChannelData = c.mChannelData; + chunk->mBufferFormat = c.mBufferFormat; + chunk->mPrincipalHandle = c.mPrincipalHandle; + } + } + template <typename T> + void AppendFromInterleavedBuffer(const T* aBuffer, size_t aFrames, + uint32_t aChannels, + const PrincipalHandle& aPrincipalHandle) { + AppendAndConsumeChunk(AudioChunk::FromInterleavedBuffer<T>( + aBuffer, aFrames, aChannels, aPrincipalHandle)); + } + // Write the segement data into an interleaved buffer. Do mixing if the + // AudioChunk's channel count in the segment is different from aChannels. + // Returns sample count of the converted audio data. The converted data will + // be stored into aBuffer. + size_t WriteToInterleavedBuffer(nsTArray<AudioDataValue>& aBuffer, + uint32_t aChannels) const; + // Consumes aChunk, and append it to the segment if its duration is not zero. + void AppendAndConsumeChunk(AudioChunk&& aChunk) { + AudioChunk unused; + AudioChunk* chunk = &unused; + + // Always consume aChunk. The chunk's mBuffer can be non-null even if its + // duration is 0. + auto consume = MakeScopeExit([&] { + chunk->mBuffer = std::move(aChunk.mBuffer); + chunk->mChannelData = std::move(aChunk.mChannelData); + + MOZ_ASSERT(chunk->mBuffer || chunk->mChannelData.IsEmpty(), + "Appending invalid data ?"); + + chunk->mVolume = aChunk.mVolume; + chunk->mBufferFormat = aChunk.mBufferFormat; + chunk->mPrincipalHandle = std::move(aChunk.mPrincipalHandle); + }); + + if (aChunk.GetDuration() == 0) { + return; + } + + if (!mChunks.IsEmpty() && + mChunks.LastElement().CanCombineWithFollowing(aChunk)) { + mChunks.LastElement().mDuration += aChunk.GetDuration(); + mDuration += aChunk.GetDuration(); + return; + } + + chunk = AppendChunk(aChunk.mDuration); + } + void ApplyVolume(float aVolume); + // Mix the segment into a mixer, interleaved. This is useful to output a + // segment to a system audio callback. It up or down mixes to aChannelCount + // channels. + void WriteTo(AudioMixer& aMixer, uint32_t aChannelCount, + uint32_t aSampleRate); + // Mix the segment into a mixer, keeping it planar, up or down mixing to + // aChannelCount channels. + void Mix(AudioMixer& aMixer, uint32_t aChannelCount, uint32_t aSampleRate); + + // Returns the maximum channel count across all chunks in this segment. + // Should there be no chunk with a channel count we return the memoized return + // value from last time this method was called. + uint32_t MaxChannelCount() { + uint32_t channelCount = 0; + for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) { + if (ci->ChannelCount()) { + channelCount = std::max(channelCount, ci->ChannelCount()); + } + } + if (channelCount == 0) { + return mMemoizedMaxChannelCount; + } + return mMemoizedMaxChannelCount = channelCount; + } + + static Type StaticType() { return AUDIO; } + + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override { + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); + } + + PrincipalHandle GetOldestPrinciple() const { + const AudioChunk* chunk = mChunks.IsEmpty() ? nullptr : &mChunks[0]; + return chunk ? chunk->GetPrincipalHandle() : PRINCIPAL_HANDLE_NONE; + } + + // Iterate on each chunks until the input function returns true. + template <typename Function> + void IterateOnChunks(const Function&& aFunction) { + for (uint32_t idx = 0; idx < mChunks.Length(); idx++) { + if (aFunction(&mChunks[idx])) { + return; + } + } + } +}; + +template <typename SrcT> +void WriteChunk(const AudioChunk& aChunk, uint32_t aOutputChannels, + float aVolume, AudioDataValue* aOutputBuffer) { + AutoTArray<const SrcT*, GUESS_AUDIO_CHANNELS> channelData; + + channelData = aChunk.ChannelData<SrcT>().Clone(); + + if (channelData.Length() < aOutputChannels) { + // Up-mix. Note that this might actually make channelData have more + // than aOutputChannels temporarily. + AudioChannelsUpMix(&channelData, aOutputChannels, + SilentChannel::ZeroChannel<SrcT>()); + } + if (channelData.Length() > aOutputChannels) { + // Down-mix. + DownmixAndInterleave(channelData, aChunk.mDuration, aVolume, + aOutputChannels, aOutputBuffer); + } else { + InterleaveAndConvertBuffer(channelData.Elements(), aChunk.mDuration, + aVolume, aOutputChannels, aOutputBuffer); + } +} + +} // namespace mozilla + +#endif /* MOZILLA_AUDIOSEGMENT_H_ */ |