/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "DynamicResampler.h" namespace mozilla { DynamicResampler::DynamicResampler(uint32_t aInRate, uint32_t aOutRate, uint32_t aPreBufferFrames) : mInRate(aInRate), mPreBufferFrames(aPreBufferFrames), mOutRate(aOutRate) { MOZ_ASSERT(aInRate); MOZ_ASSERT(aOutRate); UpdateResampler(mOutRate, STEREO); } DynamicResampler::~DynamicResampler() { if (mResampler) { speex_resampler_destroy(mResampler); } } void DynamicResampler::SetSampleFormat(AudioSampleFormat aFormat) { MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_SILENCE); MOZ_ASSERT(aFormat == AUDIO_FORMAT_S16 || aFormat == AUDIO_FORMAT_FLOAT32); mSampleFormat = aFormat; for (AudioRingBuffer& b : mInternalInBuffer) { b.SetSampleFormat(mSampleFormat); } if (mPreBufferFrames) { AppendInputSilence(mPreBufferFrames); } } bool DynamicResampler::Resample(float* aOutBuffer, uint32_t* aOutFrames, uint32_t aChannelIndex) { MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_FLOAT32); return ResampleInternal(aOutBuffer, aOutFrames, aChannelIndex); } bool DynamicResampler::Resample(int16_t* aOutBuffer, uint32_t* aOutFrames, uint32_t aChannelIndex) { MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16); return ResampleInternal(aOutBuffer, aOutFrames, aChannelIndex); } void DynamicResampler::ResampleInternal(const float* aInBuffer, uint32_t* aInFrames, float* aOutBuffer, uint32_t* aOutFrames, uint32_t aChannelIndex) { MOZ_ASSERT(mResampler); MOZ_ASSERT(mChannels); MOZ_ASSERT(mInRate); MOZ_ASSERT(mOutRate); MOZ_ASSERT(aInBuffer); MOZ_ASSERT(aInFrames); MOZ_ASSERT(*aInFrames > 0); MOZ_ASSERT(aOutBuffer); MOZ_ASSERT(aOutFrames); MOZ_ASSERT(*aOutFrames > 0); MOZ_ASSERT(aChannelIndex <= mChannels); #ifdef DEBUG int rv = #endif speex_resampler_process_float(mResampler, aChannelIndex, aInBuffer, aInFrames, aOutBuffer, aOutFrames); MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS); } void DynamicResampler::ResampleInternal(const int16_t* aInBuffer, uint32_t* aInFrames, int16_t* aOutBuffer, uint32_t* aOutFrames, uint32_t aChannelIndex) { MOZ_ASSERT(mResampler); MOZ_ASSERT(mChannels); MOZ_ASSERT(mInRate); MOZ_ASSERT(mOutRate); MOZ_ASSERT(aInBuffer); MOZ_ASSERT(aInFrames); MOZ_ASSERT(*aInFrames > 0); MOZ_ASSERT(aOutBuffer); MOZ_ASSERT(aOutFrames); MOZ_ASSERT(*aOutFrames > 0); MOZ_ASSERT(aChannelIndex <= mChannels); #ifdef DEBUG int rv = #endif speex_resampler_process_int(mResampler, aChannelIndex, aInBuffer, aInFrames, aOutBuffer, aOutFrames); MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS); } void DynamicResampler::UpdateResampler(uint32_t aOutRate, uint32_t aChannels) { MOZ_ASSERT(aOutRate); MOZ_ASSERT(aChannels); if (mChannels != aChannels) { if (mResampler) { speex_resampler_destroy(mResampler); } mResampler = speex_resampler_init(aChannels, mInRate, aOutRate, SPEEX_RESAMPLER_QUALITY_MIN, nullptr); MOZ_ASSERT(mResampler); mChannels = aChannels; mOutRate = aOutRate; // Between mono and stereo changes, keep always allocated 2 channels to // avoid reallocations in the most common case. if ((mChannels == STEREO || mChannels == 1) && mInternalInBuffer.Length() == STEREO) { // Don't worry if format is not set it will write silence then. if ((mSampleFormat == AUDIO_FORMAT_S16 || mSampleFormat == AUDIO_FORMAT_FLOAT32) && mChannels == STEREO) { // The mono channel is always up to date. When we are going from mono // to stereo upmix the mono to stereo channel uint32_t bufferedDuration = mInternalInBuffer[0].AvailableRead(); mInternalInBuffer[1].Clear(); if (bufferedDuration) { mInternalInBuffer[1].Write(mInternalInBuffer[0], bufferedDuration); } } // Maintain stereo size mInputTail.SetLength(STEREO); WarmUpResampler(false); return; } // upmix or downmix, for now just clear but it has to be updated // because allocates and this is executed in audio thread. mInternalInBuffer.Clear(); for (uint32_t i = 0; i < mChannels; ++i) { // Pre-allocate something big, twice the pre-buffer, or at least 100ms. AudioRingBuffer* b = mInternalInBuffer.AppendElement( sizeof(float) * std::max(2 * mPreBufferFrames, mInRate / 10)); if (mSampleFormat != AUDIO_FORMAT_SILENCE) { // In ctor this update is not needed b->SetSampleFormat(mSampleFormat); } } mInputTail.SetLength(mChannels); return; } if (mOutRate != aOutRate) { // If the rates was the same the resampler was not being used so warm up. if (mOutRate == mInRate) { WarmUpResampler(true); } #ifdef DEBUG int rv = #endif speex_resampler_set_rate(mResampler, mInRate, aOutRate); MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS); mOutRate = aOutRate; } } void DynamicResampler::WarmUpResampler(bool aSkipLatency) { MOZ_ASSERT(mInputTail.Length()); for (uint32_t i = 0; i < mChannels; ++i) { if (!mInputTail[i].Length()) { continue; } uint32_t inFrames = mInputTail[i].Length(); uint32_t outFrames = 5 * TailBuffer::MAXSIZE; // something big if (mSampleFormat == AUDIO_FORMAT_S16) { short outBuffer[5 * TailBuffer::MAXSIZE] = {}; ResampleInternal(mInputTail[i].Buffer(), &inFrames, outBuffer, &outFrames, i); MOZ_ASSERT(inFrames == (uint32_t)mInputTail[i].Length()); } else { float outBuffer[100] = {}; ResampleInternal(mInputTail[i].Buffer(), &inFrames, outBuffer, &outFrames, i); MOZ_ASSERT(inFrames == (uint32_t)mInputTail[i].Length()); } } if (aSkipLatency) { int inputLatency = speex_resampler_get_input_latency(mResampler); MOZ_ASSERT(inputLatency > 0); uint32_t ratioNum, ratioDen; speex_resampler_get_ratio(mResampler, &ratioNum, &ratioDen); // Ratio at this point is one so only skip the input latency. No special // calculations are needed. speex_resampler_set_skip_frac_num(mResampler, inputLatency * ratioDen); } } void DynamicResampler::AppendInput(const nsTArray& aInBuffer, uint32_t aInFrames) { MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_FLOAT32); AppendInputInternal(aInBuffer, aInFrames); } void DynamicResampler::AppendInput(const nsTArray& aInBuffer, uint32_t aInFrames) { MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16); AppendInputInternal(aInBuffer, aInFrames); } bool DynamicResampler::EnoughInFrames(uint32_t aOutFrames, uint32_t aChannelIndex) const { if (mInRate == mOutRate) { return InFramesBuffered(aChannelIndex) >= aOutFrames; } if (!(mOutRate % mInRate) && !(aOutFrames % mOutRate / mInRate)) { return InFramesBuffered(aChannelIndex) >= aOutFrames / (mOutRate / mInRate); } if (!(mInRate % mOutRate) && !(aOutFrames % mOutRate / mInRate)) { return InFramesBuffered(aChannelIndex) >= aOutFrames * mInRate / mOutRate; } return InFramesBuffered(aChannelIndex) > aOutFrames * mInRate / mOutRate; } bool DynamicResampler::CanResample(uint32_t aOutFrames) const { for (uint32_t i = 0; i < mChannels; ++i) { if (!EnoughInFrames(aOutFrames, i)) { return false; } } return true; } void DynamicResampler::AppendInputSilence(const uint32_t aInFrames) { MOZ_ASSERT(aInFrames); MOZ_ASSERT(mChannels); MOZ_ASSERT(mInternalInBuffer.Length() >= (uint32_t)mChannels); for (uint32_t i = 0; i < mChannels; ++i) { mInternalInBuffer[i].WriteSilence(aInFrames); } } uint32_t DynamicResampler::InFramesBuffered(uint32_t aChannelIndex) const { MOZ_ASSERT(mChannels); MOZ_ASSERT(aChannelIndex <= mChannels); MOZ_ASSERT(aChannelIndex <= mInternalInBuffer.Length()); return mInternalInBuffer[aChannelIndex].AvailableRead(); } uint32_t DynamicResampler::InFramesLeftToBuffer(uint32_t aChannelIndex) const { MOZ_ASSERT(mChannels); MOZ_ASSERT(aChannelIndex <= mChannels); MOZ_ASSERT(aChannelIndex <= mInternalInBuffer.Length()); return mInternalInBuffer[aChannelIndex].AvailableWrite(); } AudioChunkList::AudioChunkList(uint32_t aTotalDuration, uint32_t aChannels, const PrincipalHandle& aPrincipalHandle) : mPrincipalHandle(aPrincipalHandle) { uint32_t numOfChunks = aTotalDuration / mChunkCapacity; if (aTotalDuration % mChunkCapacity) { ++numOfChunks; } CreateChunks(numOfChunks, aChannels); } void AudioChunkList::CreateChunks(uint32_t aNumOfChunks, uint32_t aChannels) { MOZ_ASSERT(!mChunks.Length()); MOZ_ASSERT(aNumOfChunks); MOZ_ASSERT(aChannels); mChunks.AppendElements(aNumOfChunks); for (AudioChunk& chunk : mChunks) { AutoTArray, STEREO> buffer; buffer.AppendElements(aChannels); AutoTArray bufferPtrs; bufferPtrs.AppendElements(aChannels); for (uint32_t i = 0; i < aChannels; ++i) { float* ptr = buffer[i].AppendElements(mChunkCapacity); bufferPtrs[i] = ptr; } chunk.mBuffer = new mozilla::SharedChannelArrayBuffer(std::move(buffer)); chunk.mChannelData.AppendElements(aChannels); for (uint32_t i = 0; i < aChannels; ++i) { chunk.mChannelData[i] = bufferPtrs[i]; } } } void AudioChunkList::UpdateToMonoOrStereo(uint32_t aChannels) { MOZ_ASSERT(mChunks.Length()); MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16 || mSampleFormat == AUDIO_FORMAT_FLOAT32); MOZ_ASSERT(aChannels == 1 || aChannels == 2); for (AudioChunk& chunk : mChunks) { MOZ_ASSERT(chunk.ChannelCount() != (uint32_t)aChannels); MOZ_ASSERT(chunk.ChannelCount() == 1 || chunk.ChannelCount() == 2); chunk.mChannelData.SetLengthAndRetainStorage(aChannels); if (mSampleFormat == AUDIO_FORMAT_S16) { SharedChannelArrayBuffer* channelArray = static_cast*>(chunk.mBuffer.get()); channelArray->mBuffers.SetLengthAndRetainStorage(aChannels); if (aChannels == 2) { // This an indirect allocation, unfortunately. channelArray->mBuffers[1].SetLength(mChunkCapacity); chunk.mChannelData[1] = channelArray->mBuffers[1].Elements(); } } else { SharedChannelArrayBuffer* channelArray = static_cast*>(chunk.mBuffer.get()); channelArray->mBuffers.SetLengthAndRetainStorage(aChannels); if (aChannels == 2) { // This an indirect allocation, unfortunately. channelArray->mBuffers[1].SetLength(mChunkCapacity); chunk.mChannelData[1] = channelArray->mBuffers[1].Elements(); } } } } void AudioChunkList::SetSampleFormat(AudioSampleFormat aFormat) { MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_SILENCE); MOZ_ASSERT(aFormat == AUDIO_FORMAT_S16 || aFormat == AUDIO_FORMAT_FLOAT32); mSampleFormat = aFormat; if (mSampleFormat == AUDIO_FORMAT_S16) { mChunkCapacity = 2 * mChunkCapacity; } } AudioChunk& AudioChunkList::GetNext() { AudioChunk& chunk = mChunks[mIndex]; MOZ_ASSERT(!chunk.mChannelData.IsEmpty()); MOZ_ASSERT(chunk.mBuffer); MOZ_ASSERT(!chunk.mBuffer->IsShared()); MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16 || mSampleFormat == AUDIO_FORMAT_FLOAT32); chunk.mDuration = 0; chunk.mVolume = 1.0f; chunk.mPrincipalHandle = mPrincipalHandle; chunk.mBufferFormat = mSampleFormat; IncrementIndex(); return chunk; } void AudioChunkList::Update(uint32_t aChannels) { MOZ_ASSERT(mChunks.Length()); if (mChunks[0].ChannelCount() == aChannels) { return; } // Special handling between mono and stereo to avoid reallocations. if (aChannels <= 2 && mChunks[0].ChannelCount() <= 2) { UpdateToMonoOrStereo(aChannels); return; } uint32_t numOfChunks = mChunks.Length(); mChunks.ClearAndRetainStorage(); CreateChunks(numOfChunks, aChannels); } AudioResampler::AudioResampler(uint32_t aInRate, uint32_t aOutRate, uint32_t aPreBufferFrames, const PrincipalHandle& aPrincipalHandle) : mResampler(aInRate, aOutRate, aPreBufferFrames), mOutputChunks(aOutRate / 10, STEREO, aPrincipalHandle) {} void AudioResampler::AppendInput(const AudioSegment& aInSegment) { MOZ_ASSERT(aInSegment.GetDuration()); for (AudioSegment::ConstChunkIterator iter(aInSegment); !iter.IsEnded(); iter.Next()) { const AudioChunk& chunk = *iter; if (!mIsSampleFormatSet) { // We don't know the format yet and all buffers are empty. if (chunk.mBufferFormat == AUDIO_FORMAT_SILENCE) { // Only silence has been received and the format is unkown. Igonre it, // if Resampler() is called it will return silence too. continue; } // First no silence data, set the format once for lifetime and let it // continue the rest of the flow. We will not get in here again. mOutputChunks.SetSampleFormat(chunk.mBufferFormat); mResampler.SetSampleFormat(chunk.mBufferFormat); mIsSampleFormatSet = true; } MOZ_ASSERT(mIsSampleFormatSet); if (chunk.IsNull()) { mResampler.AppendInputSilence(chunk.GetDuration()); continue; } // Make sure the channel is up to date. An AudioSegment can contain chunks // with different channel count. UpdateChannels(chunk.mChannelData.Length()); if (chunk.mBufferFormat == AUDIO_FORMAT_FLOAT32) { mResampler.AppendInput(chunk.ChannelData(), chunk.GetDuration()); } else { mResampler.AppendInput(chunk.ChannelData(), chunk.GetDuration()); } } } AudioSegment AudioResampler::Resample(uint32_t aOutFrames) { AudioSegment segment; // We don't know what to do yet and we only have received silence if any just // return what they want and leave if (!mIsSampleFormatSet) { segment.AppendNullData(aOutFrames); return segment; } // Not enough input frames abort. We check for the requested frames plus one. // This is to make sure that the individual resample iteration that will // follow up, will have enough frames even if one of them consume an extra // frame. if (!mResampler.CanResample(aOutFrames + 1)) { return segment; } uint32_t totalFrames = aOutFrames; while (totalFrames) { MOZ_ASSERT(totalFrames > 0); AudioChunk& chunk = mOutputChunks.GetNext(); uint32_t outFrames = std::min(totalFrames, mOutputChunks.ChunkCapacity()); totalFrames -= outFrames; for (uint32_t i = 0; i < chunk.ChannelCount(); ++i) { uint32_t outFramesUsed = outFrames; if (chunk.mBufferFormat == AUDIO_FORMAT_FLOAT32) { #ifdef DEBUG bool rv = #endif mResampler.Resample(chunk.ChannelDataForWrite(i), &outFramesUsed, i); MOZ_ASSERT(rv); } else { #ifdef DEBUG bool rv = #endif mResampler.Resample(chunk.ChannelDataForWrite(i), &outFramesUsed, i); MOZ_ASSERT(rv); } MOZ_ASSERT(outFramesUsed == outFrames); chunk.mDuration = outFrames; } // Create a copy in order to consume that copy and not the pre-allocated // chunk segment.AppendAndConsumeChunk(AudioChunk(chunk)); } return segment; } void AudioResampler::Update(uint32_t aOutRate, uint32_t aChannels) { mResampler.UpdateResampler(aOutRate, aChannels); mOutputChunks.Update(aChannels); } uint32_t AudioResampler::InputReadableFrames() const { if (!mIsSampleFormatSet) { return mResampler.mPreBufferFrames; } return mResampler.InFramesBuffered(0); } uint32_t AudioResampler::InputWritableFrames() const { if (!mIsSampleFormatSet) { return mResampler.mPreBufferFrames; } return mResampler.InFramesLeftToBuffer(0); } } // namespace mozilla