summaryrefslogtreecommitdiffstats
path: root/dom/media/encoder/OpusTrackEncoder.cpp
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--dom/media/encoder/OpusTrackEncoder.cpp454
1 files changed, 454 insertions, 0 deletions
diff --git a/dom/media/encoder/OpusTrackEncoder.cpp b/dom/media/encoder/OpusTrackEncoder.cpp
new file mode 100644
index 0000000000..16b71d378e
--- /dev/null
+++ b/dom/media/encoder/OpusTrackEncoder.cpp
@@ -0,0 +1,454 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+#include "OpusTrackEncoder.h"
+#include "nsString.h"
+#include "mozilla/CheckedInt.h"
+#include "mozilla/ProfilerLabels.h"
+#include "VideoUtils.h"
+
+#include <opus/opus.h>
+
+#define LOG(args, ...)
+
+namespace mozilla {
+
+// The Opus format supports up to 8 channels, and supports multitrack audio up
+// to 255 channels, but the current implementation supports only mono and
+// stereo, and downmixes any more than that.
+constexpr int MAX_SUPPORTED_AUDIO_CHANNELS = 8;
+
+// http://www.opus-codec.org/docs/html_api-1.0.2/group__opus__encoder.html
+// In section "opus_encoder_init", channels must be 1 or 2 of input signal.
+constexpr int MAX_CHANNELS = 2;
+
+// A maximum data bytes for Opus to encode.
+constexpr int MAX_DATA_BYTES = 4096;
+
+// http://tools.ietf.org/html/draft-ietf-codec-oggopus-00#section-4
+// Second paragraph, " The granule position of an audio data page is in units
+// of PCM audio samples at a fixed rate of 48 kHz."
+constexpr int kOpusSamplingRate = 48000;
+
+// The duration of an Opus frame, and it must be 2.5, 5, 10, 20, 40 or 60 ms.
+constexpr int kFrameDurationMs = 20;
+
+// The supported sampling rate of input signal (Hz),
+// must be one of the following. Will resampled to 48kHz otherwise.
+constexpr int kOpusSupportedInputSamplingRates[] = {8000, 12000, 16000, 24000,
+ 48000};
+
+namespace {
+
+// An endian-neutral serialization of integers. Serializing T in little endian
+// format to aOutput, where T is a 16 bits or 32 bits integer.
+template <typename T>
+static void SerializeToBuffer(T aValue, nsTArray<uint8_t>* aOutput) {
+ for (uint32_t i = 0; i < sizeof(T); i++) {
+ aOutput->AppendElement((uint8_t)(0x000000ff & (aValue >> (i * 8))));
+ }
+}
+
+static inline void SerializeToBuffer(const nsCString& aComment,
+ nsTArray<uint8_t>* aOutput) {
+ // Format of serializing a string to buffer is, the length of string (32 bits,
+ // little endian), and the string.
+ SerializeToBuffer((uint32_t)(aComment.Length()), aOutput);
+ aOutput->AppendElements(aComment.get(), aComment.Length());
+}
+
+static void SerializeOpusIdHeader(uint8_t aChannelCount, uint16_t aPreskip,
+ uint32_t aInputSampleRate,
+ nsTArray<uint8_t>* aOutput) {
+ // The magic signature, null terminator has to be stripped off from strings.
+ constexpr uint8_t magic[] = "OpusHead";
+ aOutput->AppendElements(magic, sizeof(magic) - 1);
+
+ // The version must always be 1 (8 bits, unsigned).
+ aOutput->AppendElement(1);
+
+ // Number of output channels (8 bits, unsigned).
+ aOutput->AppendElement(aChannelCount);
+
+ // Number of samples (at 48 kHz) to discard from the decoder output when
+ // starting playback (16 bits, unsigned, little endian).
+ SerializeToBuffer(aPreskip, aOutput);
+
+ // The sampling rate of input source (32 bits, unsigned, little endian).
+ SerializeToBuffer(aInputSampleRate, aOutput);
+
+ // Output gain, an encoder should set this field to zero (16 bits, signed,
+ // little endian).
+ SerializeToBuffer((int16_t)0, aOutput);
+
+ // Channel mapping family. Family 0 allows only 1 or 2 channels (8 bits,
+ // unsigned).
+ aOutput->AppendElement(0);
+}
+
+static void SerializeOpusCommentHeader(const nsCString& aVendor,
+ const nsTArray<nsCString>& aComments,
+ nsTArray<uint8_t>* aOutput) {
+ // The magic signature, null terminator has to be stripped off.
+ constexpr uint8_t magic[] = "OpusTags";
+ aOutput->AppendElements(magic, sizeof(magic) - 1);
+
+ // The vendor; Should append in the following order:
+ // vendor string length (32 bits, unsigned, little endian)
+ // vendor string.
+ SerializeToBuffer(aVendor, aOutput);
+
+ // Add comments; Should append in the following order:
+ // comment list length (32 bits, unsigned, little endian)
+ // comment #0 string length (32 bits, unsigned, little endian)
+ // comment #0 string
+ // comment #1 string length (32 bits, unsigned, little endian)
+ // comment #1 string ...
+ SerializeToBuffer((uint32_t)aComments.Length(), aOutput);
+ for (uint32_t i = 0; i < aComments.Length(); ++i) {
+ SerializeToBuffer(aComments[i], aOutput);
+ }
+}
+
+bool IsSampleRateSupported(TrackRate aSampleRate) {
+ // According to www.opus-codec.org, creating an opus encoder requires the
+ // sampling rate of source signal be one of 8000, 12000, 16000, 24000, or
+ // 48000. If this constraint is not satisfied, we resample the input to 48kHz.
+ AutoTArray<int, 5> supportedSamplingRates;
+ supportedSamplingRates.AppendElements(
+ kOpusSupportedInputSamplingRates,
+ ArrayLength(kOpusSupportedInputSamplingRates));
+ return supportedSamplingRates.Contains(aSampleRate);
+}
+
+} // Anonymous namespace.
+
+OpusTrackEncoder::OpusTrackEncoder(TrackRate aTrackRate,
+ MediaQueue<EncodedFrame>& aEncodedDataQueue)
+ : AudioTrackEncoder(aTrackRate, aEncodedDataQueue),
+ mOutputSampleRate(IsSampleRateSupported(aTrackRate) ? aTrackRate
+ : kOpusSamplingRate),
+ mEncoder(nullptr),
+ mLookahead(0),
+ mLookaheadWritten(0),
+ mResampler(nullptr),
+ mNumOutputFrames(0) {}
+
+OpusTrackEncoder::~OpusTrackEncoder() {
+ if (mEncoder) {
+ opus_encoder_destroy(mEncoder);
+ }
+ if (mResampler) {
+ speex_resampler_destroy(mResampler);
+ mResampler = nullptr;
+ }
+}
+
+nsresult OpusTrackEncoder::Init(int aChannels) {
+ NS_ENSURE_TRUE((aChannels <= MAX_SUPPORTED_AUDIO_CHANNELS) && (aChannels > 0),
+ NS_ERROR_FAILURE);
+
+ // This version of encoder API only support 1 or 2 channels,
+ // So set the mChannels less or equal 2 and
+ // let InterleaveTrackData downmix pcm data.
+ mChannels = aChannels > MAX_CHANNELS ? MAX_CHANNELS : aChannels;
+
+ // Reject non-audio sample rates.
+ NS_ENSURE_TRUE(mTrackRate >= 8000, NS_ERROR_INVALID_ARG);
+ NS_ENSURE_TRUE(mTrackRate <= 192000, NS_ERROR_INVALID_ARG);
+
+ if (NeedsResampler()) {
+ int error;
+ mResampler = speex_resampler_init(mChannels, mTrackRate, kOpusSamplingRate,
+ SPEEX_RESAMPLER_QUALITY_DEFAULT, &error);
+
+ if (error != RESAMPLER_ERR_SUCCESS) {
+ return NS_ERROR_FAILURE;
+ }
+ }
+
+ int error = 0;
+ mEncoder = opus_encoder_create(mOutputSampleRate, mChannels,
+ OPUS_APPLICATION_AUDIO, &error);
+
+ if (error != OPUS_OK) {
+ return NS_ERROR_FAILURE;
+ }
+
+ if (mAudioBitrate) {
+ int bps = static_cast<int>(
+ std::min<uint32_t>(mAudioBitrate, std::numeric_limits<int>::max()));
+ error = opus_encoder_ctl(mEncoder, OPUS_SET_BITRATE(bps));
+ if (error != OPUS_OK) {
+ return NS_ERROR_FAILURE;
+ }
+ }
+
+ // In the case of Opus we need to calculate the codec delay based on the
+ // pre-skip. For more information see:
+ // https://tools.ietf.org/html/rfc7845#section-4.2
+ error = opus_encoder_ctl(mEncoder, OPUS_GET_LOOKAHEAD(&mLookahead));
+ if (error != OPUS_OK) {
+ mLookahead = 0;
+ return NS_ERROR_FAILURE;
+ }
+
+ SetInitialized();
+
+ return NS_OK;
+}
+
+int OpusTrackEncoder::GetLookahead() const {
+ return mLookahead * kOpusSamplingRate / mOutputSampleRate;
+}
+
+int OpusTrackEncoder::NumInputFramesPerPacket() const {
+ return mTrackRate * kFrameDurationMs / 1000;
+}
+
+int OpusTrackEncoder::NumOutputFramesPerPacket() const {
+ return mOutputSampleRate * kFrameDurationMs / 1000;
+}
+
+bool OpusTrackEncoder::NeedsResampler() const {
+ // A resampler is needed when mTrackRate is not supported by the opus encoder.
+ // This is equivalent to !IsSampleRateSupported(mTrackRate) but less cycles.
+ return mTrackRate != mOutputSampleRate &&
+ mOutputSampleRate == kOpusSamplingRate;
+}
+
+already_AddRefed<TrackMetadataBase> OpusTrackEncoder::GetMetadata() {
+ AUTO_PROFILER_LABEL("OpusTrackEncoder::GetMetadata", OTHER);
+
+ MOZ_ASSERT(mInitialized);
+
+ if (!mInitialized) {
+ return nullptr;
+ }
+
+ RefPtr<OpusMetadata> meta = new OpusMetadata();
+ meta->mChannels = mChannels;
+ meta->mSamplingFrequency = mTrackRate;
+
+ // Ogg and Webm timestamps are always sampled at 48k for Opus.
+ SerializeOpusIdHeader(mChannels,
+ mLookahead * (kOpusSamplingRate / mOutputSampleRate),
+ mTrackRate, &meta->mIdHeader);
+
+ nsCString vendor;
+ vendor.AppendASCII(opus_get_version_string());
+
+ nsTArray<nsCString> comments;
+ comments.AppendElement(
+ nsLiteralCString("ENCODER=Mozilla" MOZ_APP_UA_VERSION));
+
+ SerializeOpusCommentHeader(vendor, comments, &meta->mCommentHeader);
+
+ return meta.forget();
+}
+
+nsresult OpusTrackEncoder::Encode(AudioSegment* aSegment) {
+ AUTO_PROFILER_LABEL("OpusTrackEncoder::Encode", OTHER);
+
+ MOZ_ASSERT(aSegment);
+ MOZ_ASSERT(mInitialized || mCanceled);
+
+ if (mCanceled || IsEncodingComplete()) {
+ return NS_ERROR_FAILURE;
+ }
+
+ if (!mInitialized) {
+ // calculation below depends on the truth that mInitialized is true.
+ return NS_ERROR_FAILURE;
+ }
+
+ int result = 0;
+ // Loop until we run out of packets of input data
+ while (result >= 0 && !IsEncodingComplete()) {
+ // re-sampled frames left last time which didn't fit into an Opus packet
+ // duration.
+ const int framesLeft = mResampledLeftover.Length() / mChannels;
+ MOZ_ASSERT(NumOutputFramesPerPacket() >= framesLeft);
+ // Fetch input frames such that there will be n frames where (n +
+ // framesLeft) >= NumOutputFramesPerPacket() after re-sampling.
+ const int framesToFetch = NumInputFramesPerPacket() -
+ (framesLeft * mTrackRate / kOpusSamplingRate) +
+ (NeedsResampler() ? 1 : 0);
+
+ if (!mEndOfStream && aSegment->GetDuration() < framesToFetch) {
+ // Not enough raw data
+ return NS_OK;
+ }
+
+ // Start encoding data.
+ AutoTArray<AudioDataValue, 9600> pcm;
+ pcm.SetLength(NumOutputFramesPerPacket() * mChannels);
+
+ int frameCopied = 0;
+
+ for (AudioSegment::ChunkIterator iter(*aSegment);
+ !iter.IsEnded() && frameCopied < framesToFetch; iter.Next()) {
+ AudioChunk chunk = *iter;
+
+ // Chunk to the required frame size.
+ TrackTime frameToCopy =
+ std::min(chunk.GetDuration(),
+ static_cast<TrackTime>(framesToFetch - frameCopied));
+
+ // Possible greatest value of framesToFetch = 3844: see
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=1349421#c8. frameToCopy
+ // should not be able to exceed this value.
+ MOZ_ASSERT(frameToCopy <= 3844, "frameToCopy exceeded expected range");
+
+ if (!chunk.IsNull()) {
+ // Append the interleaved data to the end of pcm buffer.
+ AudioTrackEncoder::InterleaveTrackData(
+ chunk, frameToCopy, mChannels,
+ pcm.Elements() + frameCopied * mChannels);
+ } else {
+ CheckedInt<int> memsetLength =
+ CheckedInt<int>(frameToCopy) * mChannels * sizeof(AudioDataValue);
+ if (!memsetLength.isValid()) {
+ // This should never happen, but we use a defensive check because
+ // we really don't want a bad memset
+ MOZ_ASSERT_UNREACHABLE("memsetLength invalid!");
+ return NS_ERROR_FAILURE;
+ }
+ memset(pcm.Elements() + frameCopied * mChannels, 0,
+ memsetLength.value());
+ }
+
+ frameCopied += frameToCopy;
+ }
+
+ // Possible greatest value of framesToFetch = 3844: see
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=1349421#c8. frameCopied
+ // should not be able to exceed this value.
+ MOZ_ASSERT(frameCopied <= 3844, "frameCopied exceeded expected range");
+
+ int framesInPCM = frameCopied;
+ if (mResampler) {
+ AutoTArray<AudioDataValue, 9600> resamplingDest;
+ uint32_t inframes = frameCopied;
+ uint32_t outframes = inframes * kOpusSamplingRate / mTrackRate + 1;
+
+ // We want to consume all the input data, so we slightly oversize the
+ // resampled data buffer so we can fit the output data in. We cannot
+ // really predict the output frame count at each call.
+ resamplingDest.SetLength(outframes * mChannels);
+
+#if MOZ_SAMPLE_TYPE_S16
+ short* in = reinterpret_cast<short*>(pcm.Elements());
+ short* out = reinterpret_cast<short*>(resamplingDest.Elements());
+ speex_resampler_process_interleaved_int(mResampler, in, &inframes, out,
+ &outframes);
+#else
+ float* in = reinterpret_cast<float*>(pcm.Elements());
+ float* out = reinterpret_cast<float*>(resamplingDest.Elements());
+ speex_resampler_process_interleaved_float(mResampler, in, &inframes, out,
+ &outframes);
+#endif
+
+ MOZ_ASSERT(pcm.Length() >= mResampledLeftover.Length());
+ PodCopy(pcm.Elements(), mResampledLeftover.Elements(),
+ mResampledLeftover.Length());
+
+ uint32_t outframesToCopy = std::min(
+ outframes,
+ static_cast<uint32_t>(NumOutputFramesPerPacket() - framesLeft));
+
+ MOZ_ASSERT(pcm.Length() - mResampledLeftover.Length() >=
+ outframesToCopy * mChannels);
+ PodCopy(pcm.Elements() + mResampledLeftover.Length(),
+ resamplingDest.Elements(), outframesToCopy * mChannels);
+ int frameLeftover = outframes - outframesToCopy;
+ mResampledLeftover.SetLength(frameLeftover * mChannels);
+ PodCopy(mResampledLeftover.Elements(),
+ resamplingDest.Elements() + outframesToCopy * mChannels,
+ mResampledLeftover.Length());
+ // This is always at 48000Hz.
+ framesInPCM = framesLeft + outframesToCopy;
+ }
+
+ // Remove the raw data which has been pulled to pcm buffer.
+ // The value of frameCopied should be equal to (or smaller than, if eos)
+ // NumOutputFramesPerPacket().
+ aSegment->RemoveLeading(frameCopied);
+
+ // Has reached the end of input stream and all queued data has pulled for
+ // encoding.
+ bool isFinalPacket = false;
+ if (aSegment->GetDuration() == 0 && mEndOfStream &&
+ framesInPCM < NumOutputFramesPerPacket()) {
+ // Pad |mLookahead| samples to the end of the track to prevent loss of
+ // original data.
+ const int toWrite = std::min(mLookahead - mLookaheadWritten,
+ NumOutputFramesPerPacket() - framesInPCM);
+ PodZero(pcm.Elements() + framesInPCM * mChannels, toWrite * mChannels);
+ mLookaheadWritten += toWrite;
+ framesInPCM += toWrite;
+ if (mLookaheadWritten == mLookahead) {
+ isFinalPacket = true;
+ }
+ }
+
+ MOZ_ASSERT_IF(!isFinalPacket, framesInPCM == NumOutputFramesPerPacket());
+
+ // Append null data to pcm buffer if the leftover data is not enough for
+ // opus encoder.
+ if (framesInPCM < NumOutputFramesPerPacket() && isFinalPacket) {
+ PodZero(pcm.Elements() + framesInPCM * mChannels,
+ (NumOutputFramesPerPacket() - framesInPCM) * mChannels);
+ }
+ auto frameData = MakeRefPtr<EncodedFrame::FrameData>();
+ // Encode the data with Opus Encoder.
+ frameData->SetLength(MAX_DATA_BYTES);
+ // result is returned as opus error code if it is negative.
+ result = 0;
+#ifdef MOZ_SAMPLE_TYPE_S16
+ const opus_int16* pcmBuf = static_cast<opus_int16*>(pcm.Elements());
+ result = opus_encode(mEncoder, pcmBuf, NumOutputFramesPerPacket(),
+ frameData->Elements(), MAX_DATA_BYTES);
+#else
+ const float* pcmBuf = static_cast<float*>(pcm.Elements());
+ result = opus_encode_float(mEncoder, pcmBuf, NumOutputFramesPerPacket(),
+ frameData->Elements(), MAX_DATA_BYTES);
+#endif
+ frameData->SetLength(result >= 0 ? result : 0);
+
+ if (result < 0) {
+ LOG("[Opus] Fail to encode data! Result: %s.", opus_strerror(result));
+ }
+ if (isFinalPacket) {
+ if (mResampler) {
+ speex_resampler_destroy(mResampler);
+ mResampler = nullptr;
+ }
+ mResampledLeftover.SetLength(0);
+ }
+
+ // timestamp should be the time of the first sample
+ mEncodedDataQueue.Push(MakeAndAddRef<EncodedFrame>(
+ media::TimeUnit(mNumOutputFrames + mLookahead, mOutputSampleRate),
+ static_cast<uint64_t>(framesInPCM) * kOpusSamplingRate /
+ mOutputSampleRate,
+ kOpusSamplingRate, EncodedFrame::OPUS_AUDIO_FRAME,
+ std::move(frameData)));
+
+ mNumOutputFrames += NumOutputFramesPerPacket();
+ LOG("[Opus] mOutputTimeStamp %.3f.",
+ media::TimeUnit(mNumOutputFrames, mOutputSampleRate).ToSeconds());
+
+ if (isFinalPacket) {
+ LOG("[Opus] Done encoding.");
+ mEncodedDataQueue.Finish();
+ }
+ }
+
+ return result >= 0 ? NS_OK : NS_ERROR_FAILURE;
+}
+
+} // namespace mozilla
+
+#undef LOG