diff options
Diffstat (limited to 'dom/media/DynamicResampler.cpp')
| -rw-r--r-- | dom/media/DynamicResampler.cpp | 477 |
1 files changed, 477 insertions, 0 deletions
diff --git a/dom/media/DynamicResampler.cpp b/dom/media/DynamicResampler.cpp new file mode 100644 index 0000000000..470bbfd418 --- /dev/null +++ b/dom/media/DynamicResampler.cpp @@ -0,0 +1,477 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DynamicResampler.h" + +namespace mozilla { + +DynamicResampler::DynamicResampler(uint32_t aInRate, uint32_t aOutRate, + uint32_t aPreBufferFrames) + : mInRate(aInRate), mPreBufferFrames(aPreBufferFrames), mOutRate(aOutRate) { + MOZ_ASSERT(aInRate); + MOZ_ASSERT(aOutRate); + UpdateResampler(mOutRate, STEREO); +} + +DynamicResampler::~DynamicResampler() { + if (mResampler) { + speex_resampler_destroy(mResampler); + } +} + +void DynamicResampler::SetSampleFormat(AudioSampleFormat aFormat) { + MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_SILENCE); + MOZ_ASSERT(aFormat == AUDIO_FORMAT_S16 || aFormat == AUDIO_FORMAT_FLOAT32); + + mSampleFormat = aFormat; + for (AudioRingBuffer& b : mInternalInBuffer) { + b.SetSampleFormat(mSampleFormat); + } + if (mPreBufferFrames) { + AppendInputSilence(mPreBufferFrames); + } +} + +bool DynamicResampler::Resample(float* aOutBuffer, uint32_t* aOutFrames, + uint32_t aChannelIndex) { + MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_FLOAT32); + return ResampleInternal(aOutBuffer, aOutFrames, aChannelIndex); +} + +bool DynamicResampler::Resample(int16_t* aOutBuffer, uint32_t* aOutFrames, + uint32_t aChannelIndex) { + MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16); + return ResampleInternal(aOutBuffer, aOutFrames, aChannelIndex); +} + +void DynamicResampler::ResampleInternal(const float* aInBuffer, + uint32_t* aInFrames, float* aOutBuffer, + uint32_t* aOutFrames, + uint32_t aChannelIndex) { + MOZ_ASSERT(mResampler); + MOZ_ASSERT(mChannels); + MOZ_ASSERT(mInRate); + MOZ_ASSERT(mOutRate); + + MOZ_ASSERT(aInBuffer); + MOZ_ASSERT(aInFrames); + MOZ_ASSERT(*aInFrames > 0); + MOZ_ASSERT(aOutBuffer); + MOZ_ASSERT(aOutFrames); + MOZ_ASSERT(*aOutFrames > 0); + + MOZ_ASSERT(aChannelIndex <= mChannels); + +#ifdef DEBUG + int rv = +#endif + speex_resampler_process_float(mResampler, aChannelIndex, aInBuffer, + aInFrames, aOutBuffer, aOutFrames); + MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS); +} + +void DynamicResampler::ResampleInternal(const int16_t* aInBuffer, + uint32_t* aInFrames, + int16_t* aOutBuffer, + uint32_t* aOutFrames, + uint32_t aChannelIndex) { + MOZ_ASSERT(mResampler); + MOZ_ASSERT(mChannels); + MOZ_ASSERT(mInRate); + MOZ_ASSERT(mOutRate); + + MOZ_ASSERT(aInBuffer); + MOZ_ASSERT(aInFrames); + MOZ_ASSERT(*aInFrames > 0); + MOZ_ASSERT(aOutBuffer); + MOZ_ASSERT(aOutFrames); + MOZ_ASSERT(*aOutFrames > 0); + + MOZ_ASSERT(aChannelIndex <= mChannels); + +#ifdef DEBUG + int rv = +#endif + speex_resampler_process_int(mResampler, aChannelIndex, aInBuffer, + aInFrames, aOutBuffer, aOutFrames); + MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS); +} + +void DynamicResampler::UpdateResampler(uint32_t aOutRate, uint32_t aChannels) { + MOZ_ASSERT(aOutRate); + MOZ_ASSERT(aChannels); + + if (mChannels != aChannels) { + if (mResampler) { + speex_resampler_destroy(mResampler); + } + mResampler = speex_resampler_init(aChannels, mInRate, aOutRate, + SPEEX_RESAMPLER_QUALITY_MIN, nullptr); + MOZ_ASSERT(mResampler); + mChannels = aChannels; + mOutRate = aOutRate; + // Between mono and stereo changes, keep always allocated 2 channels to + // avoid reallocations in the most common case. + if ((mChannels == STEREO || mChannels == 1) && + mInternalInBuffer.Length() == STEREO) { + // Don't worry if format is not set it will write silence then. + if ((mSampleFormat == AUDIO_FORMAT_S16 || + mSampleFormat == AUDIO_FORMAT_FLOAT32) && + mChannels == STEREO) { + // The mono channel is always up to date. When we are going from mono + // to stereo upmix the mono to stereo channel + uint32_t bufferedDuration = mInternalInBuffer[0].AvailableRead(); + mInternalInBuffer[1].Clear(); + if (bufferedDuration) { + mInternalInBuffer[1].Write(mInternalInBuffer[0], bufferedDuration); + } + } + // Maintain stereo size + mInputTail.SetLength(STEREO); + WarmUpResampler(false); + return; + } + // upmix or downmix, for now just clear but it has to be updated + // because allocates and this is executed in audio thread. + mInternalInBuffer.Clear(); + for (uint32_t i = 0; i < mChannels; ++i) { + // Pre-allocate something big, twice the pre-buffer, or at least 100ms. + AudioRingBuffer* b = mInternalInBuffer.AppendElement( + sizeof(float) * std::max(2 * mPreBufferFrames, mInRate / 10)); + if (mSampleFormat != AUDIO_FORMAT_SILENCE) { + // In ctor this update is not needed + b->SetSampleFormat(mSampleFormat); + } + } + mInputTail.SetLength(mChannels); + return; + } + + if (mOutRate != aOutRate) { + // If the rates was the same the resampler was not being used so warm up. + if (mOutRate == mInRate) { + WarmUpResampler(true); + } + +#ifdef DEBUG + int rv = +#endif + speex_resampler_set_rate(mResampler, mInRate, aOutRate); + MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS); + mOutRate = aOutRate; + } +} + +void DynamicResampler::WarmUpResampler(bool aSkipLatency) { + MOZ_ASSERT(mInputTail.Length()); + for (uint32_t i = 0; i < mChannels; ++i) { + if (!mInputTail[i].Length()) { + continue; + } + uint32_t inFrames = mInputTail[i].Length(); + uint32_t outFrames = 5 * TailBuffer::MAXSIZE; // something big + if (mSampleFormat == AUDIO_FORMAT_S16) { + short outBuffer[5 * TailBuffer::MAXSIZE] = {}; + ResampleInternal(mInputTail[i].Buffer<short>(), &inFrames, outBuffer, + &outFrames, i); + MOZ_ASSERT(inFrames == (uint32_t)mInputTail[i].Length()); + } else { + float outBuffer[100] = {}; + ResampleInternal(mInputTail[i].Buffer<float>(), &inFrames, outBuffer, + &outFrames, i); + MOZ_ASSERT(inFrames == (uint32_t)mInputTail[i].Length()); + } + } + if (aSkipLatency) { + int inputLatency = speex_resampler_get_input_latency(mResampler); + MOZ_ASSERT(inputLatency > 0); + uint32_t ratioNum, ratioDen; + speex_resampler_get_ratio(mResampler, &ratioNum, &ratioDen); + // Ratio at this point is one so only skip the input latency. No special + // calculations are needed. + speex_resampler_set_skip_frac_num(mResampler, inputLatency * ratioDen); + } +} + +void DynamicResampler::AppendInput(const nsTArray<const float*>& aInBuffer, + uint32_t aInFrames) { + MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_FLOAT32); + AppendInputInternal(aInBuffer, aInFrames); +} +void DynamicResampler::AppendInput(const nsTArray<const int16_t*>& aInBuffer, + uint32_t aInFrames) { + MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16); + AppendInputInternal(aInBuffer, aInFrames); +} + +bool DynamicResampler::EnoughInFrames(uint32_t aOutFrames, + uint32_t aChannelIndex) const { + if (mInRate == mOutRate) { + return InFramesBuffered(aChannelIndex) >= aOutFrames; + } + if (!(mOutRate % mInRate) && !(aOutFrames % mOutRate / mInRate)) { + return InFramesBuffered(aChannelIndex) >= aOutFrames / (mOutRate / mInRate); + } + if (!(mInRate % mOutRate) && !(aOutFrames % mOutRate / mInRate)) { + return InFramesBuffered(aChannelIndex) >= aOutFrames * mInRate / mOutRate; + } + return InFramesBuffered(aChannelIndex) > aOutFrames * mInRate / mOutRate; +} + +bool DynamicResampler::CanResample(uint32_t aOutFrames) const { + for (uint32_t i = 0; i < mChannels; ++i) { + if (!EnoughInFrames(aOutFrames, i)) { + return false; + } + } + return true; +} + +void DynamicResampler::AppendInputSilence(const uint32_t aInFrames) { + MOZ_ASSERT(aInFrames); + MOZ_ASSERT(mChannels); + MOZ_ASSERT(mInternalInBuffer.Length() >= (uint32_t)mChannels); + for (uint32_t i = 0; i < mChannels; ++i) { + mInternalInBuffer[i].WriteSilence(aInFrames); + } +} + +uint32_t DynamicResampler::InFramesBuffered(uint32_t aChannelIndex) const { + MOZ_ASSERT(mChannels); + MOZ_ASSERT(aChannelIndex <= mChannels); + MOZ_ASSERT(aChannelIndex <= mInternalInBuffer.Length()); + return mInternalInBuffer[aChannelIndex].AvailableRead(); +} + +uint32_t DynamicResampler::InFramesLeftToBuffer(uint32_t aChannelIndex) const { + MOZ_ASSERT(mChannels); + MOZ_ASSERT(aChannelIndex <= mChannels); + MOZ_ASSERT(aChannelIndex <= mInternalInBuffer.Length()); + return mInternalInBuffer[aChannelIndex].AvailableWrite(); +} + +AudioChunkList::AudioChunkList(uint32_t aTotalDuration, uint32_t aChannels, + const PrincipalHandle& aPrincipalHandle) + : mPrincipalHandle(aPrincipalHandle) { + uint32_t numOfChunks = aTotalDuration / mChunkCapacity; + if (aTotalDuration % mChunkCapacity) { + ++numOfChunks; + } + CreateChunks(numOfChunks, aChannels); +} + +void AudioChunkList::CreateChunks(uint32_t aNumOfChunks, uint32_t aChannels) { + MOZ_ASSERT(!mChunks.Length()); + MOZ_ASSERT(aNumOfChunks); + MOZ_ASSERT(aChannels); + mChunks.AppendElements(aNumOfChunks); + + for (AudioChunk& chunk : mChunks) { + AutoTArray<nsTArray<float>, STEREO> buffer; + buffer.AppendElements(aChannels); + + AutoTArray<const float*, STEREO> bufferPtrs; + bufferPtrs.AppendElements(aChannels); + + for (uint32_t i = 0; i < aChannels; ++i) { + float* ptr = buffer[i].AppendElements(mChunkCapacity); + bufferPtrs[i] = ptr; + } + + chunk.mBuffer = new mozilla::SharedChannelArrayBuffer(std::move(buffer)); + chunk.mChannelData.AppendElements(aChannels); + for (uint32_t i = 0; i < aChannels; ++i) { + chunk.mChannelData[i] = bufferPtrs[i]; + } + } +} + +void AudioChunkList::UpdateToMonoOrStereo(uint32_t aChannels) { + MOZ_ASSERT(mChunks.Length()); + MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16 || + mSampleFormat == AUDIO_FORMAT_FLOAT32); + MOZ_ASSERT(aChannels == 1 || aChannels == 2); + + for (AudioChunk& chunk : mChunks) { + MOZ_ASSERT(chunk.ChannelCount() != (uint32_t)aChannels); + MOZ_ASSERT(chunk.ChannelCount() == 1 || chunk.ChannelCount() == 2); + chunk.mChannelData.SetLengthAndRetainStorage(aChannels); + if (mSampleFormat == AUDIO_FORMAT_S16) { + SharedChannelArrayBuffer<short>* channelArray = + static_cast<SharedChannelArrayBuffer<short>*>(chunk.mBuffer.get()); + channelArray->mBuffers.SetLengthAndRetainStorage(aChannels); + if (aChannels == 2) { + // This an indirect allocation, unfortunately. + channelArray->mBuffers[1].SetLength(mChunkCapacity); + chunk.mChannelData[1] = channelArray->mBuffers[1].Elements(); + } + } else { + SharedChannelArrayBuffer<float>* channelArray = + static_cast<SharedChannelArrayBuffer<float>*>(chunk.mBuffer.get()); + channelArray->mBuffers.SetLengthAndRetainStorage(aChannels); + if (aChannels == 2) { + // This an indirect allocation, unfortunately. + channelArray->mBuffers[1].SetLength(mChunkCapacity); + chunk.mChannelData[1] = channelArray->mBuffers[1].Elements(); + } + } + } +} + +void AudioChunkList::SetSampleFormat(AudioSampleFormat aFormat) { + MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_SILENCE); + MOZ_ASSERT(aFormat == AUDIO_FORMAT_S16 || aFormat == AUDIO_FORMAT_FLOAT32); + mSampleFormat = aFormat; + if (mSampleFormat == AUDIO_FORMAT_S16) { + mChunkCapacity = 2 * mChunkCapacity; + } +} + +AudioChunk& AudioChunkList::GetNext() { + AudioChunk& chunk = mChunks[mIndex]; + MOZ_ASSERT(!chunk.mChannelData.IsEmpty()); + MOZ_ASSERT(chunk.mBuffer); + MOZ_ASSERT(!chunk.mBuffer->IsShared()); + MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16 || + mSampleFormat == AUDIO_FORMAT_FLOAT32); + chunk.mDuration = 0; + chunk.mVolume = 1.0f; + chunk.mPrincipalHandle = mPrincipalHandle; + chunk.mBufferFormat = mSampleFormat; + IncrementIndex(); + return chunk; +} + +void AudioChunkList::Update(uint32_t aChannels) { + MOZ_ASSERT(mChunks.Length()); + if (mChunks[0].ChannelCount() == aChannels) { + return; + } + + // Special handling between mono and stereo to avoid reallocations. + if (aChannels <= 2 && mChunks[0].ChannelCount() <= 2) { + UpdateToMonoOrStereo(aChannels); + return; + } + + uint32_t numOfChunks = mChunks.Length(); + mChunks.ClearAndRetainStorage(); + CreateChunks(numOfChunks, aChannels); +} + +AudioResampler::AudioResampler(uint32_t aInRate, uint32_t aOutRate, + uint32_t aPreBufferFrames, + const PrincipalHandle& aPrincipalHandle) + : mResampler(aInRate, aOutRate, aPreBufferFrames), + mOutputChunks(aOutRate / 10, STEREO, aPrincipalHandle) {} + +void AudioResampler::AppendInput(const AudioSegment& aInSegment) { + MOZ_ASSERT(aInSegment.GetDuration()); + for (AudioSegment::ConstChunkIterator iter(aInSegment); !iter.IsEnded(); + iter.Next()) { + const AudioChunk& chunk = *iter; + if (!mIsSampleFormatSet) { + // We don't know the format yet and all buffers are empty. + if (chunk.mBufferFormat == AUDIO_FORMAT_SILENCE) { + // Only silence has been received and the format is unkown. Igonre it, + // if Resampler() is called it will return silence too. + continue; + } + // First no silence data, set the format once for lifetime and let it + // continue the rest of the flow. We will not get in here again. + mOutputChunks.SetSampleFormat(chunk.mBufferFormat); + mResampler.SetSampleFormat(chunk.mBufferFormat); + mIsSampleFormatSet = true; + } + MOZ_ASSERT(mIsSampleFormatSet); + if (chunk.IsNull()) { + mResampler.AppendInputSilence(chunk.GetDuration()); + continue; + } + // Make sure the channel is up to date. An AudioSegment can contain chunks + // with different channel count. + UpdateChannels(chunk.mChannelData.Length()); + if (chunk.mBufferFormat == AUDIO_FORMAT_FLOAT32) { + mResampler.AppendInput(chunk.ChannelData<float>(), chunk.GetDuration()); + } else { + mResampler.AppendInput(chunk.ChannelData<int16_t>(), chunk.GetDuration()); + } + } +} + +AudioSegment AudioResampler::Resample(uint32_t aOutFrames) { + AudioSegment segment; + + // We don't know what to do yet and we only have received silence if any just + // return what they want and leave + if (!mIsSampleFormatSet) { + segment.AppendNullData(aOutFrames); + return segment; + } + + // Not enough input frames abort. We check for the requested frames plus one. + // This is to make sure that the individual resample iteration that will + // follow up, will have enough frames even if one of them consume an extra + // frame. + if (!mResampler.CanResample(aOutFrames + 1)) { + return segment; + } + + uint32_t totalFrames = aOutFrames; + while (totalFrames) { + MOZ_ASSERT(totalFrames > 0); + AudioChunk& chunk = mOutputChunks.GetNext(); + uint32_t outFrames = std::min(totalFrames, mOutputChunks.ChunkCapacity()); + totalFrames -= outFrames; + + for (uint32_t i = 0; i < chunk.ChannelCount(); ++i) { + uint32_t outFramesUsed = outFrames; + if (chunk.mBufferFormat == AUDIO_FORMAT_FLOAT32) { +#ifdef DEBUG + bool rv = +#endif + mResampler.Resample(chunk.ChannelDataForWrite<float>(i), + &outFramesUsed, i); + MOZ_ASSERT(rv); + } else { +#ifdef DEBUG + bool rv = +#endif + mResampler.Resample(chunk.ChannelDataForWrite<int16_t>(i), + &outFramesUsed, i); + MOZ_ASSERT(rv); + } + MOZ_ASSERT(outFramesUsed == outFrames); + chunk.mDuration = outFrames; + } + + // Create a copy in order to consume that copy and not the pre-allocated + // chunk + segment.AppendAndConsumeChunk(AudioChunk(chunk)); + } + + return segment; +} + +void AudioResampler::Update(uint32_t aOutRate, uint32_t aChannels) { + mResampler.UpdateResampler(aOutRate, aChannels); + mOutputChunks.Update(aChannels); +} + +uint32_t AudioResampler::InputReadableFrames() const { + if (!mIsSampleFormatSet) { + return mResampler.mPreBufferFrames; + } + return mResampler.InFramesBuffered(0); +} + +uint32_t AudioResampler::InputWritableFrames() const { + if (!mIsSampleFormatSet) { + return mResampler.mPreBufferFrames; + } + return mResampler.InFramesLeftToBuffer(0); +} + +} // namespace mozilla |