summaryrefslogtreecommitdiffstats
path: root/dom/media/encoder/TrackEncoder.h
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--dom/media/encoder/TrackEncoder.h501
1 files changed, 501 insertions, 0 deletions
diff --git a/dom/media/encoder/TrackEncoder.h b/dom/media/encoder/TrackEncoder.h
new file mode 100644
index 0000000000..879949874f
--- /dev/null
+++ b/dom/media/encoder/TrackEncoder.h
@@ -0,0 +1,501 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef TrackEncoder_h_
+#define TrackEncoder_h_
+
+#include "AudioSegment.h"
+#include "EncodedFrame.h"
+#include "MediaQueue.h"
+#include "MediaTrackGraph.h"
+#include "TrackMetadataBase.h"
+#include "VideoSegment.h"
+
+namespace mozilla {
+
+class AbstractThread;
+class DriftCompensator;
+class TrackEncoder;
+
+class TrackEncoderListener {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoderListener)
+
+ /**
+ * Called when the TrackEncoder has received its first real data.
+ */
+ virtual void Started(TrackEncoder* aEncoder) = 0;
+
+ /**
+ * Called when the TrackEncoder's underlying encoder has been successfully
+ * initialized and there's non-null data ready to be encoded.
+ */
+ virtual void Initialized(TrackEncoder* aEncoder) = 0;
+
+ /**
+ * Called after the TrackEncoder hit an unexpected error, causing it to
+ * abort operation.
+ */
+ virtual void Error(TrackEncoder* aEncoder) = 0;
+
+ protected:
+ virtual ~TrackEncoderListener() = default;
+};
+
+/**
+ * Base class of AudioTrackEncoder and VideoTrackEncoder. Lifetime managed by
+ * MediaEncoder. All methods are to be called only on the worker thread.
+ *
+ * The control APIs are all called by MediaEncoder on its dedicated thread. Data
+ * is encoded as soon as it has been appended (and time has advanced past its
+ * end in case of video) and pushed to mEncodedDataQueue.
+ */
+class TrackEncoder {
+ public:
+ TrackEncoder(TrackRate aTrackRate,
+ MediaQueue<EncodedFrame>& aEncodedDataQueue);
+
+ /**
+ * Called by MediaEncoder to cancel the encoding.
+ */
+ virtual void Cancel() = 0;
+
+ /**
+ * Notifies us that we have reached the end of the stream and no more data
+ * will be appended.
+ */
+ virtual void NotifyEndOfStream() = 0;
+
+ /**
+ * Creates and sets up meta data for a specific codec, called on the worker
+ * thread.
+ */
+ virtual already_AddRefed<TrackMetadataBase> GetMetadata() = 0;
+
+ /**
+ * MediaQueue containing encoded data, that is pushed as soon as it's ready.
+ */
+ MediaQueue<EncodedFrame>& EncodedDataQueue() { return mEncodedDataQueue; }
+
+ /**
+ * Returns true once this TrackEncoder is initialized.
+ */
+ bool IsInitialized();
+
+ /**
+ * Returns true once this TrackEncoder has received some data.
+ */
+ bool IsStarted();
+
+ /**
+ * True if the track encoder has encoded all source segments coming from
+ * MediaTrackGraph. Call on the worker thread.
+ */
+ bool IsEncodingComplete() const;
+
+ /**
+ * Registers a listener to events from this TrackEncoder.
+ * We hold a strong reference to the listener.
+ */
+ void RegisterListener(TrackEncoderListener* aListener);
+
+ /**
+ * Unregisters a listener from events from this TrackEncoder.
+ * The listener will stop receiving events synchronously.
+ */
+ bool UnregisterListener(TrackEncoderListener* aListener);
+
+ virtual void SetBitrate(const uint32_t aBitrate) = 0;
+
+ /**
+ * It's optional to set the worker thread, but if you do we'll assert that
+ * we are in the worker thread in every method that gets called.
+ */
+ void SetWorkerThread(AbstractThread* aWorkerThread);
+
+ /**
+ * Measure size of internal buffers.
+ */
+ virtual size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) = 0;
+
+ protected:
+ virtual ~TrackEncoder() { MOZ_ASSERT(mListeners.IsEmpty()); }
+
+ /**
+ * If this TrackEncoder was not already initialized, it is set to initialized
+ * and listeners are notified.
+ */
+ void SetInitialized();
+
+ /**
+ * If this TrackEncoder was not already marked started, its started state is
+ * set and listeners are notified.
+ */
+ void SetStarted();
+
+ /**
+ * Called after an error. Cancels the encoding and notifies listeners.
+ */
+ void OnError();
+
+ /**
+ * True if the track encoder has been initialized successfully.
+ */
+ bool mInitialized;
+
+ /**
+ * True if the track encoder has received data.
+ */
+ bool mStarted;
+
+ /**
+ * True once all data until the end of the input track has been received.
+ */
+ bool mEndOfStream;
+
+ /**
+ * True once this encoding has been cancelled.
+ */
+ bool mCanceled;
+
+ // How many times we have tried to initialize the encoder.
+ uint32_t mInitCounter;
+
+ /**
+ * True if this TrackEncoder is currently suspended.
+ */
+ bool mSuspended;
+
+ /**
+ * The track rate of source media.
+ */
+ const TrackRate mTrackRate;
+
+ /**
+ * If set we assert that all methods are called on this thread.
+ */
+ RefPtr<AbstractThread> mWorkerThread;
+
+ /**
+ * MediaQueue where encoded data ends up. Note that metadata goes out of band.
+ */
+ MediaQueue<EncodedFrame>& mEncodedDataQueue;
+
+ nsTArray<RefPtr<TrackEncoderListener>> mListeners;
+};
+
+class AudioTrackEncoder : public TrackEncoder {
+ public:
+ AudioTrackEncoder(TrackRate aTrackRate,
+ MediaQueue<EncodedFrame>& aEncodedDataQueue)
+ : TrackEncoder(aTrackRate, aEncodedDataQueue),
+ mChannels(0),
+ mNotInitDuration(0),
+ mAudioBitrate(0) {}
+
+ /**
+ * Suspends encoding from now, i.e., all future audio data received through
+ * AppendAudioSegment() until the next Resume() will be dropped.
+ */
+ void Suspend();
+
+ /**
+ * Resumes encoding starting now, i.e., data from the next
+ * AppendAudioSegment() will get encoded.
+ */
+ void Resume();
+
+ /**
+ * Appends and consumes track data from aSegment.
+ */
+ void AppendAudioSegment(AudioSegment&& aSegment);
+
+ template <typename T>
+ static void InterleaveTrackData(nsTArray<const T*>& aInput, int32_t aDuration,
+ uint32_t aOutputChannels,
+ AudioDataValue* aOutput, float aVolume) {
+ if (aInput.Length() < aOutputChannels) {
+ // Up-mix. This might make the mChannelData have more than aChannels.
+ AudioChannelsUpMix(&aInput, aOutputChannels,
+ SilentChannel::ZeroChannel<T>());
+ }
+
+ if (aInput.Length() > aOutputChannels) {
+ DownmixAndInterleave(aInput, aDuration, aVolume, aOutputChannels,
+ aOutput);
+ } else {
+ InterleaveAndConvertBuffer(aInput.Elements(), aDuration, aVolume,
+ aOutputChannels, aOutput);
+ }
+ }
+
+ /**
+ * Interleaves the track data and stores the result into aOutput. Might need
+ * to up-mix or down-mix the channel data if the channels number of this chunk
+ * is different from aOutputChannels. The channel data from aChunk might be
+ * modified by up-mixing.
+ */
+ static void InterleaveTrackData(AudioChunk& aChunk, int32_t aDuration,
+ uint32_t aOutputChannels,
+ AudioDataValue* aOutput);
+
+ /**
+ * De-interleaves the aInput data and stores the result into aOutput.
+ * No up-mix or down-mix operations inside.
+ */
+ static void DeInterleaveTrackData(AudioDataValue* aInput, int32_t aDuration,
+ int32_t aChannels, AudioDataValue* aOutput);
+
+ /**
+ * Measure size of internal buffers.
+ */
+ size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
+
+ void SetBitrate(const uint32_t aBitrate) override {
+ mAudioBitrate = aBitrate;
+ }
+
+ /**
+ * Tries to initiate the AudioEncoder based on data in aSegment.
+ * This can be re-called often, as it will exit early should we already be
+ * initiated. mInitiated will only be set if there was enough data in
+ * aSegment to infer metadata. If mInitiated gets set, listeners are notified.
+ *
+ * Not having enough data in aSegment to initiate the encoder for an
+ * accumulated aDuration of one second will make us initiate with a default
+ * number of channels.
+ *
+ * If we attempt to initiate the underlying encoder but fail, we Cancel() and
+ * notify listeners.
+ */
+ void TryInit(const AudioSegment& aSegment, TrackTime aDuration);
+
+ void Cancel() override;
+
+ /**
+ * Dispatched from MediaTrackGraph when we have finished feeding data to
+ * mOutgoingBuffer.
+ */
+ void NotifyEndOfStream() override;
+
+ protected:
+ /**
+ * Number of samples per channel in a pcm buffer. This is also the value of
+ * frame size required by audio encoder, and listeners will be notified when
+ * at least this much data has been added to mOutgoingBuffer.
+ */
+ virtual int NumInputFramesPerPacket() const { return 0; }
+
+ /**
+ * Initializes the audio encoder. The call of this method is delayed until we
+ * have received the first valid track from MediaTrackGraph.
+ */
+ virtual nsresult Init(int aChannels) = 0;
+
+ /**
+ * Encodes buffered data and pushes it to mEncodedDataQueue.
+ */
+ virtual nsresult Encode(AudioSegment* aSegment) = 0;
+
+ /**
+ * The number of channels are used for processing PCM data in the audio
+ * encoder. This value comes from the first valid audio chunk. If encoder
+ * can't support the channels in the chunk, downmix PCM stream can be
+ * performed. This value also be used to initialize the audio encoder.
+ */
+ int mChannels;
+
+ /**
+ * A segment queue of outgoing audio track data to the encoder.
+ * The contents of mOutgoingBuffer will always be what has been appended on
+ * the encoder thread but not yet consumed by the encoder sub class.
+ */
+ AudioSegment mOutgoingBuffer;
+
+ TrackTime mNotInitDuration;
+
+ uint32_t mAudioBitrate;
+};
+
+enum class FrameDroppingMode {
+ ALLOW, // Allowed to drop frames to keep up under load
+ DISALLOW, // Must not drop any frames, even if it means we will OOM
+};
+
+class VideoTrackEncoder : public TrackEncoder {
+ public:
+ VideoTrackEncoder(RefPtr<DriftCompensator> aDriftCompensator,
+ TrackRate aTrackRate,
+ MediaQueue<EncodedFrame>& aEncodedDataQueue,
+ FrameDroppingMode aFrameDroppingMode);
+
+ /**
+ * Suspends encoding from aTime, i.e., all video frame with a timestamp
+ * between aTime and the timestamp of the next Resume() will be dropped.
+ */
+ void Suspend(const TimeStamp& aTime);
+
+ /**
+ * Resumes encoding starting at aTime.
+ */
+ void Resume(const TimeStamp& aTime);
+
+ /**
+ * Makes the video black from aTime.
+ */
+ void Disable(const TimeStamp& aTime);
+
+ /**
+ * Makes the video non-black from aTime.
+ *
+ * NB that it could still be forced black for other reasons, like principals.
+ */
+ void Enable(const TimeStamp& aTime);
+
+ /**
+ * Appends source video frames to mIncomingBuffer. We only append the source
+ * chunk if the image is different from mLastChunk's image. Called on the
+ * MediaTrackGraph thread.
+ */
+ void AppendVideoSegment(VideoSegment&& aSegment);
+
+ /**
+ * Measure size of internal buffers.
+ */
+ size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
+
+ void SetBitrate(const uint32_t aBitrate) override {
+ mVideoBitrate = aBitrate;
+ }
+
+ /**
+ * Tries to initiate the VideoEncoder based on data in aSegment.
+ * This can be re-called often, as it will exit early should we already be
+ * initiated. mInitiated will only be set if there was enough data in
+ * aSegment to infer metadata. If mInitiated gets set, listeners are notified.
+ * The amount of chunks needed can be controlled by
+ * aFrameRateDetectionMinChunks which denotes the minimum number of chunks
+ * needed to infer the framerate.
+ *
+ * Failing to initiate the encoder for an accumulated aDuration of 30 seconds
+ * is seen as an error and will cancel the current encoding.
+ */
+ void Init(const VideoSegment& aSegment, const TimeStamp& aTime,
+ size_t aFrameRateDetectionMinChunks);
+
+ TrackTime SecondsToMediaTime(double aS) const {
+ NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX / TRACK_RATE_MAX,
+ "Bad seconds");
+ return mTrackRate * aS;
+ }
+
+ /**
+ * MediaTrackGraph notifies us about the time of the track's start.
+ * This gets called on the MediaEncoder thread after a dispatch.
+ */
+ void SetStartOffset(const TimeStamp& aStartOffset);
+
+ void Cancel() override;
+
+ /**
+ * Notifies us that we have reached the end of the stream and no more data
+ * will be appended to mIncomingBuffer.
+ */
+ void NotifyEndOfStream() override;
+
+ /**
+ * Dispatched from MediaTrackGraph when it has run an iteration so we can
+ * hand more data to the encoder.
+ */
+ void AdvanceCurrentTime(const TimeStamp& aTime);
+
+ protected:
+ /**
+ * Initialize the video encoder. In order to collect the value of width and
+ * height of source frames, this initialization is delayed until we have
+ * received the first valid video frame from MediaTrackGraph.
+ * Listeners will be notified after it has been successfully initialized.
+ */
+ virtual nsresult Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
+ int32_t aDisplayHeight, float aEstimatedFrameRate) = 0;
+
+ /**
+ * Encodes data in the outgoing buffer and pushes it to mEncodedDataQueue.
+ */
+ virtual nsresult Encode(VideoSegment* aSegment) = 0;
+
+ /**
+ * Drift compensator for re-clocking incoming video frame wall-clock
+ * timestamps to audio time.
+ */
+ const RefPtr<DriftCompensator> mDriftCompensator;
+
+ /**
+ * The last unique frame and duration so far handled by
+ * NotifyAdvanceCurrentTime. When a new frame is detected, mLastChunk is added
+ * to mOutgoingBuffer.
+ */
+ VideoChunk mLastChunk;
+
+ /**
+ * A segment queue of incoming video track data, from listeners.
+ * The duration of mIncomingBuffer is irrelevant as we only look at TimeStamps
+ * of frames. Consumed data is replaced by null data.
+ */
+ VideoSegment mIncomingBuffer;
+
+ /**
+ * A segment queue of outgoing video track data to the encoder.
+ * The contents of mOutgoingBuffer will always be what has been consumed from
+ * mIncomingBuffer (up to mCurrentTime) but not yet consumed by the encoder
+ * sub class. There won't be any null data at the beginning of mOutgoingBuffer
+ * unless explicitly pushed by the producer.
+ */
+ VideoSegment mOutgoingBuffer;
+
+ /**
+ * The number of mTrackRate ticks we have passed to mOutgoingBuffer.
+ */
+ TrackTime mEncodedTicks;
+
+ /**
+ * The time up to which we have forwarded data from mIncomingBuffer to
+ * mOutgoingBuffer.
+ */
+ TimeStamp mCurrentTime;
+
+ /**
+ * The time the video track started, so the start of the video track can be
+ * synced to the start of the audio track.
+ *
+ * Note that this time will progress during suspension, to make sure the
+ * incoming frames stay in sync with the output.
+ */
+ TimeStamp mStartTime;
+
+ /**
+ * The time Suspend was called on the MediaRecorder, so we can calculate the
+ * duration on the next Resume().
+ */
+ TimeStamp mSuspendTime;
+
+ uint32_t mVideoBitrate;
+
+ /**
+ * ALLOW to drop frames under load.
+ * DISALLOW to encode all frames, mainly for testing.
+ */
+ FrameDroppingMode mFrameDroppingMode;
+
+ /**
+ * True if the video MediaTrackTrack this VideoTrackEncoder is attached to is
+ * currently enabled. While false, we encode all frames as black.
+ */
+ bool mEnabled;
+};
+
+} // namespace mozilla
+
+#endif