summaryrefslogtreecommitdiffstats
path: root/dom/media/encoder/TrackEncoder.cpp
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /dom/media/encoder/TrackEncoder.cpp
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/encoder/TrackEncoder.cpp')
-rw-r--r--dom/media/encoder/TrackEncoder.cpp822
1 files changed, 822 insertions, 0 deletions
diff --git a/dom/media/encoder/TrackEncoder.cpp b/dom/media/encoder/TrackEncoder.cpp
new file mode 100644
index 0000000000..8e03fd6fe3
--- /dev/null
+++ b/dom/media/encoder/TrackEncoder.cpp
@@ -0,0 +1,822 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "TrackEncoder.h"
+
+#include "AudioChannelFormat.h"
+#include "DriftCompensation.h"
+#include "MediaTrackGraph.h"
+#include "MediaTrackListener.h"
+#include "mozilla/AbstractThread.h"
+#include "mozilla/Logging.h"
+#include "mozilla/ProfilerLabels.h"
+#include "mozilla/RollingMean.h"
+#include "VideoUtils.h"
+#include "mozilla/Telemetry.h"
+
+namespace mozilla {
+
+LazyLogModule gTrackEncoderLog("TrackEncoder");
+#define TRACK_LOG(type, msg) MOZ_LOG(gTrackEncoderLog, type, msg)
+
+constexpr int DEFAULT_CHANNELS = 1;
+constexpr int DEFAULT_FRAME_WIDTH = 640;
+constexpr int DEFAULT_FRAME_HEIGHT = 480;
+constexpr int DEFAULT_FRAME_RATE = 30;
+// 10 second threshold if the audio encoder cannot be initialized.
+constexpr int AUDIO_INIT_FAILED_DURATION = 10;
+// 30 second threshold if the video encoder cannot be initialized.
+constexpr int VIDEO_INIT_FAILED_DURATION = 30;
+constexpr int FRAMERATE_DETECTION_ROLLING_WINDOW = 3;
+constexpr size_t FRAMERATE_DETECTION_MIN_CHUNKS = 5;
+constexpr int FRAMERATE_DETECTION_MAX_DURATION_S = 6;
+
+TrackEncoder::TrackEncoder(TrackRate aTrackRate,
+ MediaQueue<EncodedFrame>& aEncodedDataQueue)
+ : mInitialized(false),
+ mStarted(false),
+ mEndOfStream(false),
+ mCanceled(false),
+ mInitCounter(0),
+ mSuspended(false),
+ mTrackRate(aTrackRate),
+ mEncodedDataQueue(aEncodedDataQueue) {}
+
+bool TrackEncoder::IsInitialized() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mInitialized;
+}
+
+bool TrackEncoder::IsStarted() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mStarted;
+}
+
+bool TrackEncoder::IsEncodingComplete() const {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mEncodedDataQueue.IsFinished();
+}
+
+void TrackEncoder::SetInitialized() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ if (mInitialized) {
+ return;
+ }
+
+ mInitialized = true;
+
+ for (auto& l : mListeners.Clone()) {
+ l->Initialized(this);
+ }
+}
+
+void TrackEncoder::SetStarted() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ if (mStarted) {
+ return;
+ }
+
+ mStarted = true;
+
+ for (auto& l : mListeners.Clone()) {
+ l->Started(this);
+ }
+}
+
+void TrackEncoder::OnError() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ Cancel();
+
+ for (auto& l : mListeners.Clone()) {
+ l->Error(this);
+ }
+}
+
+void TrackEncoder::RegisterListener(TrackEncoderListener* aListener) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ MOZ_ASSERT(!mListeners.Contains(aListener));
+ mListeners.AppendElement(aListener);
+}
+
+bool TrackEncoder::UnregisterListener(TrackEncoderListener* aListener) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mListeners.RemoveElement(aListener);
+}
+
+void TrackEncoder::SetWorkerThread(AbstractThread* aWorkerThread) {
+ mWorkerThread = aWorkerThread;
+}
+
+void AudioTrackEncoder::Suspend() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder %p]: Suspend(), was %s", this,
+ mSuspended ? "suspended" : "live"));
+
+ if (mSuspended) {
+ return;
+ }
+
+ mSuspended = true;
+}
+
+void AudioTrackEncoder::Resume() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder %p]: Resume(), was %s", this,
+ mSuspended ? "suspended" : "live"));
+
+ if (!mSuspended) {
+ return;
+ }
+
+ mSuspended = false;
+}
+
+void AudioTrackEncoder::AppendAudioSegment(AudioSegment&& aSegment) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ AUTO_PROFILER_LABEL("AudioTrackEncoder::AppendAudioSegment", OTHER);
+ TRACK_LOG(LogLevel::Verbose,
+ ("[AudioTrackEncoder %p]: AppendAudioSegment() duration=%" PRIu64,
+ this, aSegment.GetDuration()));
+
+ if (mCanceled) {
+ return;
+ }
+
+ if (mEndOfStream) {
+ return;
+ }
+
+ TryInit(mOutgoingBuffer, aSegment.GetDuration());
+
+ if (mSuspended) {
+ return;
+ }
+
+ SetStarted();
+ mOutgoingBuffer.AppendFrom(&aSegment);
+
+ if (!mInitialized) {
+ return;
+ }
+
+ if (NS_FAILED(Encode(&mOutgoingBuffer))) {
+ OnError();
+ return;
+ }
+
+ MOZ_ASSERT_IF(IsEncodingComplete(), mOutgoingBuffer.IsEmpty());
+}
+
+void AudioTrackEncoder::TryInit(const AudioSegment& aSegment,
+ TrackTime aDuration) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ if (mInitialized) {
+ return;
+ }
+
+ mInitCounter++;
+ TRACK_LOG(LogLevel::Debug,
+ ("[AudioTrackEncoder %p]: Inited the audio encoder %d times", this,
+ mInitCounter));
+
+ for (AudioSegment::ConstChunkIterator iter(aSegment); !iter.IsEnded();
+ iter.Next()) {
+ // The number of channels is determined by the first non-null chunk, and
+ // thus the audio encoder is initialized at this time.
+ if (iter->IsNull()) {
+ continue;
+ }
+
+ nsresult rv = Init(iter->mChannelData.Length());
+
+ if (NS_SUCCEEDED(rv)) {
+ TRACK_LOG(LogLevel::Info,
+ ("[AudioTrackEncoder %p]: Successfully initialized!", this));
+ return;
+ } else {
+ TRACK_LOG(
+ LogLevel::Error,
+ ("[AudioTrackEncoder %p]: Failed to initialize the encoder!", this));
+ OnError();
+ return;
+ }
+ break;
+ }
+
+ mNotInitDuration += aDuration;
+ if (!mInitialized &&
+ ((mNotInitDuration - 1) / mTrackRate >= AUDIO_INIT_FAILED_DURATION) &&
+ mInitCounter > 1) {
+ // Perform a best effort initialization since we haven't gotten any
+ // data yet. Motivated by issues like Bug 1336367
+ TRACK_LOG(LogLevel::Warning,
+ ("[AudioTrackEncoder]: Initialize failed for %ds. Attempting to "
+ "init with %d (default) channels!",
+ AUDIO_INIT_FAILED_DURATION, DEFAULT_CHANNELS));
+ nsresult rv = Init(DEFAULT_CHANNELS);
+ if (NS_FAILED(rv)) {
+ TRACK_LOG(LogLevel::Error,
+ ("[AudioTrackEncoder %p]: Default-channel-init failed.", this));
+ OnError();
+ return;
+ }
+ }
+}
+
+void AudioTrackEncoder::Cancel() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info, ("[AudioTrackEncoder %p]: Cancel()", this));
+ mCanceled = true;
+ mEndOfStream = true;
+ mOutgoingBuffer.Clear();
+ mEncodedDataQueue.Finish();
+}
+
+void AudioTrackEncoder::NotifyEndOfStream() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info,
+ ("[AudioTrackEncoder %p]: NotifyEndOfStream()", this));
+
+ if (!mCanceled && !mInitialized) {
+ // If source audio track is completely silent till the end of encoding,
+ // initialize the encoder with a default channel count.
+ Init(DEFAULT_CHANNELS);
+ }
+
+ if (mEndOfStream) {
+ return;
+ }
+
+ mEndOfStream = true;
+
+ if (NS_FAILED(Encode(&mOutgoingBuffer))) {
+ mOutgoingBuffer.Clear();
+ OnError();
+ }
+
+ MOZ_ASSERT(mOutgoingBuffer.GetDuration() == 0);
+}
+
+/*static*/
+void AudioTrackEncoder::InterleaveTrackData(AudioChunk& aChunk,
+ int32_t aDuration,
+ uint32_t aOutputChannels,
+ AudioDataValue* aOutput) {
+ uint32_t numChannelsToCopy = std::min(
+ aOutputChannels, static_cast<uint32_t>(aChunk.mChannelData.Length()));
+ switch (aChunk.mBufferFormat) {
+ case AUDIO_FORMAT_S16: {
+ AutoTArray<const int16_t*, 2> array;
+ array.SetLength(numChannelsToCopy);
+ for (uint32_t i = 0; i < array.Length(); i++) {
+ array[i] = static_cast<const int16_t*>(aChunk.mChannelData[i]);
+ }
+ InterleaveTrackData(array, aDuration, aOutputChannels, aOutput,
+ aChunk.mVolume);
+ break;
+ }
+ case AUDIO_FORMAT_FLOAT32: {
+ AutoTArray<const float*, 2> array;
+ array.SetLength(numChannelsToCopy);
+ for (uint32_t i = 0; i < array.Length(); i++) {
+ array[i] = static_cast<const float*>(aChunk.mChannelData[i]);
+ }
+ InterleaveTrackData(array, aDuration, aOutputChannels, aOutput,
+ aChunk.mVolume);
+ break;
+ }
+ case AUDIO_FORMAT_SILENCE: {
+ MOZ_ASSERT(false, "To implement.");
+ }
+ };
+}
+
+/*static*/
+void AudioTrackEncoder::DeInterleaveTrackData(AudioDataValue* aInput,
+ int32_t aDuration,
+ int32_t aChannels,
+ AudioDataValue* aOutput) {
+ for (int32_t i = 0; i < aChannels; ++i) {
+ for (int32_t j = 0; j < aDuration; ++j) {
+ aOutput[i * aDuration + j] = aInput[i + j * aChannels];
+ }
+ }
+}
+
+size_t AudioTrackEncoder::SizeOfExcludingThis(
+ mozilla::MallocSizeOf aMallocSizeOf) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mOutgoingBuffer.SizeOfExcludingThis(aMallocSizeOf);
+}
+
+VideoTrackEncoder::VideoTrackEncoder(
+ RefPtr<DriftCompensator> aDriftCompensator, TrackRate aTrackRate,
+ MediaQueue<EncodedFrame>& aEncodedDataQueue,
+ FrameDroppingMode aFrameDroppingMode)
+ : TrackEncoder(aTrackRate, aEncodedDataQueue),
+ mDriftCompensator(std::move(aDriftCompensator)),
+ mEncodedTicks(0),
+ mVideoBitrate(0),
+ mFrameDroppingMode(aFrameDroppingMode),
+ mEnabled(true) {
+ mLastChunk.mDuration = 0;
+}
+
+void VideoTrackEncoder::Suspend(const TimeStamp& aTime) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info,
+ ("[VideoTrackEncoder %p]: Suspend() at %.3fs, was %s", this,
+ mStartTime.IsNull() ? 0.0 : (aTime - mStartTime).ToSeconds(),
+ mSuspended ? "suspended" : "live"));
+
+ if (mSuspended) {
+ return;
+ }
+
+ mSuspended = true;
+ mSuspendTime = aTime;
+}
+
+void VideoTrackEncoder::Resume(const TimeStamp& aTime) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ if (!mSuspended) {
+ return;
+ }
+
+ TRACK_LOG(
+ LogLevel::Info,
+ ("[VideoTrackEncoder %p]: Resume() after %.3fs, was %s", this,
+ (aTime - mSuspendTime).ToSeconds(), mSuspended ? "suspended" : "live"));
+
+ mSuspended = false;
+
+ TimeDuration suspendDuration = aTime - mSuspendTime;
+ if (!mLastChunk.mTimeStamp.IsNull()) {
+ VideoChunk* nextChunk = mIncomingBuffer.FindChunkContaining(aTime);
+ MOZ_ASSERT_IF(nextChunk, nextChunk->mTimeStamp <= aTime);
+ if (nextChunk) {
+ nextChunk->mTimeStamp = aTime;
+ }
+ mLastChunk.mTimeStamp += suspendDuration;
+ }
+ if (!mStartTime.IsNull()) {
+ mStartTime += suspendDuration;
+ }
+
+ mSuspendTime = TimeStamp();
+}
+
+void VideoTrackEncoder::Disable(const TimeStamp& aTime) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Debug, ("[VideoTrackEncoder %p]: Disable()", this));
+
+ if (mStartTime.IsNull()) {
+ // We haven't started yet. No need to touch future frames.
+ mEnabled = false;
+ return;
+ }
+
+ // Advancing currentTime to process any frames in mIncomingBuffer between
+ // mCurrentTime and aTime.
+ AdvanceCurrentTime(aTime);
+ if (!mLastChunk.mTimeStamp.IsNull()) {
+ // Insert a black frame at t=aTime into mIncomingBuffer, to trigger the
+ // shift to black at the right moment.
+ VideoSegment tempSegment;
+ tempSegment.AppendFrom(&mIncomingBuffer);
+ mIncomingBuffer.AppendFrame(do_AddRef(mLastChunk.mFrame.GetImage()),
+ mLastChunk.mFrame.GetIntrinsicSize(),
+ mLastChunk.mFrame.GetPrincipalHandle(), true,
+ aTime);
+ mIncomingBuffer.AppendFrom(&tempSegment);
+ }
+ mEnabled = false;
+}
+
+void VideoTrackEncoder::Enable(const TimeStamp& aTime) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Debug, ("[VideoTrackEncoder %p]: Enable()", this));
+
+ if (mStartTime.IsNull()) {
+ // We haven't started yet. No need to touch future frames.
+ mEnabled = true;
+ return;
+ }
+
+ // Advancing currentTime to process any frames in mIncomingBuffer between
+ // mCurrentTime and aTime.
+ AdvanceCurrentTime(aTime);
+ if (!mLastChunk.mTimeStamp.IsNull()) {
+ // Insert a real frame at t=aTime into mIncomingBuffer, to trigger the
+ // shift from black at the right moment.
+ VideoSegment tempSegment;
+ tempSegment.AppendFrom(&mIncomingBuffer);
+ mIncomingBuffer.AppendFrame(do_AddRef(mLastChunk.mFrame.GetImage()),
+ mLastChunk.mFrame.GetIntrinsicSize(),
+ mLastChunk.mFrame.GetPrincipalHandle(),
+ mLastChunk.mFrame.GetForceBlack(), aTime);
+ mIncomingBuffer.AppendFrom(&tempSegment);
+ }
+ mEnabled = true;
+}
+
+void VideoTrackEncoder::AppendVideoSegment(VideoSegment&& aSegment) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: AppendVideoSegment()", this));
+
+ if (mCanceled) {
+ return;
+ }
+
+ if (mEndOfStream) {
+ return;
+ }
+
+ for (VideoSegment::ConstChunkIterator iter(aSegment); !iter.IsEnded();
+ iter.Next()) {
+ if (iter->IsNull()) {
+ // A null image was sent. This is a signal from the source that we should
+ // clear any images buffered in the future.
+ mIncomingBuffer.Clear();
+ continue; // Don't append iter, as it is null.
+ }
+ if (VideoChunk* c = mIncomingBuffer.GetLastChunk()) {
+ if (iter->mTimeStamp < c->mTimeStamp) {
+ // Time went backwards. This can happen when a MediaDecoder seeks.
+ // We need to handle this by removing any frames buffered in the future
+ // and start over at iter->mTimeStamp.
+ mIncomingBuffer.Clear();
+ }
+ }
+ SetStarted();
+ mIncomingBuffer.AppendFrame(do_AddRef(iter->mFrame.GetImage()),
+ iter->mFrame.GetIntrinsicSize(),
+ iter->mFrame.GetPrincipalHandle(),
+ iter->mFrame.GetForceBlack(), iter->mTimeStamp);
+ }
+ aSegment.Clear();
+}
+
+void VideoTrackEncoder::Init(const VideoSegment& aSegment,
+ const TimeStamp& aTime,
+ size_t aFrameRateDetectionMinChunks) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ MOZ_ASSERT(!aTime.IsNull());
+
+ if (mInitialized) {
+ return;
+ }
+
+ mInitCounter++;
+ TRACK_LOG(LogLevel::Debug,
+ ("[VideoTrackEncoder %p]: Init the video encoder %d times", this,
+ mInitCounter));
+
+ Maybe<float> framerate;
+ if (!aSegment.IsEmpty()) {
+ // The number of whole frames, i.e., with known duration.
+ size_t frameCount = 0;
+ RollingMean<TimeDuration, TimeDuration> meanDuration(
+ FRAMERATE_DETECTION_ROLLING_WINDOW);
+ VideoSegment::ConstChunkIterator iter(aSegment);
+ TimeStamp previousChunkTime = iter->mTimeStamp;
+ iter.Next();
+ for (; !iter.IsEnded(); iter.Next(), ++frameCount) {
+ meanDuration.insert(iter->mTimeStamp - previousChunkTime);
+ previousChunkTime = iter->mTimeStamp;
+ }
+ TRACK_LOG(LogLevel::Debug, ("[VideoTrackEncoder %p]: Init() frameCount=%zu",
+ this, frameCount));
+ if (frameCount >= aFrameRateDetectionMinChunks) {
+ if (meanDuration.empty()) {
+ // No whole frames available, use aTime as end time.
+ framerate = Some(1.0f / (aTime - mStartTime).ToSeconds());
+ } else {
+ // We want some frames for estimating the framerate.
+ framerate = Some(1.0f / meanDuration.mean().ToSeconds());
+ }
+ } else if ((aTime - mStartTime).ToSeconds() >
+ FRAMERATE_DETECTION_MAX_DURATION_S) {
+ // Instead of failing init after the fail-timeout, we fallback to a very
+ // low rate.
+ framerate = Some(static_cast<float>(frameCount) /
+ (aTime - mStartTime).ToSeconds());
+ }
+ }
+
+ if (framerate) {
+ for (VideoSegment::ConstChunkIterator iter(aSegment); !iter.IsEnded();
+ iter.Next()) {
+ if (iter->IsNull()) {
+ continue;
+ }
+
+ gfx::IntSize imgsize = iter->mFrame.GetImage()->GetSize();
+ gfx::IntSize intrinsicSize = iter->mFrame.GetIntrinsicSize();
+ nsresult rv = Init(imgsize.width, imgsize.height, intrinsicSize.width,
+ intrinsicSize.height, *framerate);
+
+ if (NS_SUCCEEDED(rv)) {
+ TRACK_LOG(LogLevel::Info,
+ ("[VideoTrackEncoder %p]: Successfully initialized!", this));
+ return;
+ }
+
+ TRACK_LOG(
+ LogLevel::Error,
+ ("[VideoTrackEncoder %p]: Failed to initialize the encoder!", this));
+ OnError();
+ break;
+ }
+ }
+
+ if (((aTime - mStartTime).ToSeconds() > VIDEO_INIT_FAILED_DURATION) &&
+ mInitCounter > 1) {
+ TRACK_LOG(LogLevel::Warning,
+ ("[VideoTrackEncoder %p]: No successful init for %ds.", this,
+ VIDEO_INIT_FAILED_DURATION));
+ OnError();
+ return;
+ }
+}
+
+void VideoTrackEncoder::Cancel() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ TRACK_LOG(LogLevel::Info, ("[VideoTrackEncoder %p]: Cancel()", this));
+ mCanceled = true;
+ mEndOfStream = true;
+ mIncomingBuffer.Clear();
+ mOutgoingBuffer.Clear();
+ mLastChunk.SetNull(0);
+ mEncodedDataQueue.Finish();
+}
+
+void VideoTrackEncoder::NotifyEndOfStream() {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+
+ if (mCanceled) {
+ return;
+ }
+
+ if (mEndOfStream) {
+ // We have already been notified.
+ return;
+ }
+
+ mEndOfStream = true;
+ TRACK_LOG(LogLevel::Info,
+ ("[VideoTrackEncoder %p]: NotifyEndOfStream()", this));
+
+ if (!mLastChunk.IsNull()) {
+ RefPtr<layers::Image> lastImage = mLastChunk.mFrame.GetImage();
+ const TimeStamp now = TimeStamp::Now();
+ TimeStamp currentTime = mSuspended ? mSuspendTime : mCurrentTime;
+ currentTime = mDriftCompensator->GetVideoTime(now, currentTime);
+ TimeDuration absoluteEndTime = currentTime - mStartTime;
+ CheckedInt64 duration =
+ UsecsToFrames(absoluteEndTime.ToMicroseconds(), mTrackRate) -
+ mEncodedTicks;
+ if (duration.isValid() && duration.value() > 0) {
+ mEncodedTicks += duration.value();
+ TRACK_LOG(LogLevel::Debug,
+ ("[VideoTrackEncoder %p]: Appending last video frame %p at pos "
+ "%.3fs, "
+ "track-end=%.3fs",
+ this, lastImage.get(),
+ (mLastChunk.mTimeStamp - mStartTime).ToSeconds(),
+ absoluteEndTime.ToSeconds()));
+ mOutgoingBuffer.AppendFrame(
+ lastImage.forget(), mLastChunk.mFrame.GetIntrinsicSize(),
+ PRINCIPAL_HANDLE_NONE, mLastChunk.mFrame.GetForceBlack() || !mEnabled,
+ mLastChunk.mTimeStamp);
+ mOutgoingBuffer.ExtendLastFrameBy(duration.value());
+ }
+
+ if (!mInitialized) {
+ // Try to init without waiting for an accurate framerate.
+ Init(mOutgoingBuffer, currentTime, 0);
+ }
+ }
+
+ if (mCanceled) {
+ // Previous Init failed and we got canceled. Nothing to do here.
+ return;
+ }
+
+ mIncomingBuffer.Clear();
+ mLastChunk.SetNull(0);
+
+ if (NS_WARN_IF(!mInitialized)) {
+ // Still not initialized. There was probably no real frame at all, perhaps
+ // by muting. Initialize the encoder with default frame width, frame
+ // height, and frame rate.
+ Init(DEFAULT_FRAME_WIDTH, DEFAULT_FRAME_HEIGHT, DEFAULT_FRAME_WIDTH,
+ DEFAULT_FRAME_HEIGHT, DEFAULT_FRAME_RATE);
+ }
+
+ if (NS_FAILED(Encode(&mOutgoingBuffer))) {
+ OnError();
+ }
+
+ MOZ_ASSERT(mOutgoingBuffer.IsEmpty());
+}
+
+void VideoTrackEncoder::SetStartOffset(const TimeStamp& aStartOffset) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ MOZ_ASSERT(mCurrentTime.IsNull());
+ TRACK_LOG(LogLevel::Info, ("[VideoTrackEncoder %p]: SetStartOffset()", this));
+ mStartTime = aStartOffset;
+ mCurrentTime = aStartOffset;
+}
+
+void VideoTrackEncoder::AdvanceCurrentTime(const TimeStamp& aTime) {
+ AUTO_PROFILER_LABEL("VideoTrackEncoder::AdvanceCurrentTime", OTHER);
+
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ MOZ_ASSERT(!mStartTime.IsNull());
+ MOZ_ASSERT(!mCurrentTime.IsNull());
+
+ if (mCanceled) {
+ return;
+ }
+
+ if (mEndOfStream) {
+ return;
+ }
+
+ if (mSuspended) {
+ TRACK_LOG(
+ LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: AdvanceCurrentTime() suspended at %.3fs",
+ this, (mCurrentTime - mStartTime).ToSeconds()));
+ mCurrentTime = aTime;
+ mIncomingBuffer.ForgetUpToTime(mCurrentTime);
+ return;
+ }
+
+ TRACK_LOG(LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: AdvanceCurrentTime() to %.3fs", this,
+ (aTime - mStartTime).ToSeconds()));
+
+ // Grab frames within the currentTime range from the incoming buffer.
+ VideoSegment tempSegment;
+ {
+ VideoChunk* previousChunk = &mLastChunk;
+ auto appendDupes = [&](const TimeStamp& aUpTo) {
+ while ((aUpTo - previousChunk->mTimeStamp).ToSeconds() > 1.0) {
+ // We encode at least one frame per second, even if there are none
+ // flowing.
+ previousChunk->mTimeStamp += TimeDuration::FromSeconds(1.0);
+ tempSegment.AppendFrame(
+ do_AddRef(previousChunk->mFrame.GetImage()),
+ previousChunk->mFrame.GetIntrinsicSize(),
+ previousChunk->mFrame.GetPrincipalHandle(),
+ previousChunk->mFrame.GetForceBlack() || !mEnabled,
+ previousChunk->mTimeStamp);
+ TRACK_LOG(
+ LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: Duplicating video frame (%p) at pos %.3f",
+ this, previousChunk->mFrame.GetImage(),
+ (previousChunk->mTimeStamp - mStartTime).ToSeconds()));
+ }
+ };
+ for (VideoSegment::ChunkIterator iter(mIncomingBuffer); !iter.IsEnded();
+ iter.Next()) {
+ MOZ_ASSERT(!iter->IsNull());
+ if (!previousChunk->IsNull() &&
+ iter->mTimeStamp <= previousChunk->mTimeStamp) {
+ // This frame starts earlier than previousChunk. Skip.
+ continue;
+ }
+ if (iter->mTimeStamp >= aTime) {
+ // This frame starts in the future. Stop.
+ break;
+ }
+ if (!previousChunk->IsNull()) {
+ appendDupes(iter->mTimeStamp);
+ }
+ tempSegment.AppendFrame(
+ do_AddRef(iter->mFrame.GetImage()), iter->mFrame.GetIntrinsicSize(),
+ iter->mFrame.GetPrincipalHandle(),
+ iter->mFrame.GetForceBlack() || !mEnabled, iter->mTimeStamp);
+ TRACK_LOG(LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: Taking video frame (%p) at pos %.3f",
+ this, iter->mFrame.GetImage(),
+ (iter->mTimeStamp - mStartTime).ToSeconds()));
+ previousChunk = &*iter;
+ }
+ if (!previousChunk->IsNull()) {
+ appendDupes(aTime);
+ }
+ }
+ mCurrentTime = aTime;
+ mIncomingBuffer.ForgetUpToTime(mCurrentTime);
+
+ // Convert tempSegment timestamps to durations and add chunks with known
+ // duration to mOutgoingBuffer.
+ const TimeStamp now = TimeStamp::Now();
+ for (VideoSegment::ConstChunkIterator iter(tempSegment); !iter.IsEnded();
+ iter.Next()) {
+ VideoChunk chunk = *iter;
+
+ if (mLastChunk.mTimeStamp.IsNull()) {
+ // This is the first real chunk in the track. Make it start at the
+ // beginning of the track.
+ MOZ_ASSERT(!iter->mTimeStamp.IsNull());
+
+ TRACK_LOG(
+ LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: Got the first video frame (%p) at pos %.3f "
+ "(moving it to beginning)",
+ this, iter->mFrame.GetImage(),
+ (iter->mTimeStamp - mStartTime).ToSeconds()));
+
+ mLastChunk = *iter;
+ mLastChunk.mTimeStamp = mStartTime;
+ continue;
+ }
+
+ MOZ_ASSERT(!mLastChunk.IsNull());
+ MOZ_ASSERT(!chunk.IsNull());
+
+ TimeDuration absoluteEndTime =
+ mDriftCompensator->GetVideoTime(now, chunk.mTimeStamp) - mStartTime;
+ TRACK_LOG(LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: Appending video frame %p, at pos %.3fs "
+ "until %.3fs",
+ this, mLastChunk.mFrame.GetImage(),
+ (mDriftCompensator->GetVideoTime(now, mLastChunk.mTimeStamp) -
+ mStartTime)
+ .ToSeconds(),
+ absoluteEndTime.ToSeconds()));
+ CheckedInt64 duration =
+ UsecsToFrames(absoluteEndTime.ToMicroseconds(), mTrackRate) -
+ mEncodedTicks;
+ if (!duration.isValid()) {
+ NS_ERROR("Duration overflow");
+ return;
+ }
+
+ if (duration.value() <= 0) {
+ // A frame either started before the last frame (can happen when
+ // multiple frames are added before SetStartOffset), or
+ // two frames were so close together that they ended up at the same
+ // position. We handle both cases by ignoring the previous frame.
+
+ TRACK_LOG(LogLevel::Verbose,
+ ("[VideoTrackEncoder %p]: Duration from frame %p to frame %p "
+ "is %" PRId64 ". Ignoring %p",
+ this, mLastChunk.mFrame.GetImage(), iter->mFrame.GetImage(),
+ duration.value(), mLastChunk.mFrame.GetImage()));
+
+ TimeStamp t = mLastChunk.mTimeStamp;
+ mLastChunk = *iter;
+ mLastChunk.mTimeStamp = t;
+ continue;
+ }
+
+ mEncodedTicks += duration.value();
+ mOutgoingBuffer.AppendFrame(
+ do_AddRef(mLastChunk.mFrame.GetImage()),
+ mLastChunk.mFrame.GetIntrinsicSize(), PRINCIPAL_HANDLE_NONE,
+ mLastChunk.mFrame.GetForceBlack() || !mEnabled, mLastChunk.mTimeStamp);
+ mOutgoingBuffer.ExtendLastFrameBy(duration.value());
+ mLastChunk = chunk;
+ }
+
+ if (mOutgoingBuffer.IsEmpty()) {
+ return;
+ }
+
+ Init(mOutgoingBuffer, mCurrentTime, FRAMERATE_DETECTION_MIN_CHUNKS);
+
+ if (!mInitialized) {
+ return;
+ }
+
+ if (NS_FAILED(Encode(&mOutgoingBuffer))) {
+ OnError();
+ return;
+ }
+
+ MOZ_ASSERT(mOutgoingBuffer.IsEmpty());
+}
+
+size_t VideoTrackEncoder::SizeOfExcludingThis(
+ mozilla::MallocSizeOf aMallocSizeOf) {
+ MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
+ return mIncomingBuffer.SizeOfExcludingThis(aMallocSizeOf) +
+ mOutgoingBuffer.SizeOfExcludingThis(aMallocSizeOf);
+}
+
+} // namespace mozilla
+
+#undef TRACK_LOG