Adding upstream version 115.8.0esr.upstream/115.8.0esr

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 501 insertions, 0 deletions

diff --git a/dom/media/encoder/TrackEncoder.h b/dom/media/encoder/TrackEncoder.h
new file mode 100644
index 0000000000..879949874f
--- /dev/null
+++ b/dom/media/encoder/TrackEncoder.h
@@ -0,0 +1,501 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef TrackEncoder_h_
+#define TrackEncoder_h_
+
+#include "AudioSegment.h"
+#include "EncodedFrame.h"
+#include "MediaQueue.h"
+#include "MediaTrackGraph.h"
+#include "TrackMetadataBase.h"
+#include "VideoSegment.h"
+
+namespace mozilla {
+
+class AbstractThread;
+class DriftCompensator;
+class TrackEncoder;
+
+class TrackEncoderListener {
+ public:
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoderListener)
+
+  /**
+   * Called when the TrackEncoder has received its first real data.
+   */
+  virtual void Started(TrackEncoder* aEncoder) = 0;
+
+  /**
+   * Called when the TrackEncoder's underlying encoder has been successfully
+   * initialized and there's non-null data ready to be encoded.
+   */
+  virtual void Initialized(TrackEncoder* aEncoder) = 0;
+
+  /**
+   * Called after the TrackEncoder hit an unexpected error, causing it to
+   * abort operation.
+   */
+  virtual void Error(TrackEncoder* aEncoder) = 0;
+
+ protected:
+  virtual ~TrackEncoderListener() = default;
+};
+
+/**
+ * Base class of AudioTrackEncoder and VideoTrackEncoder. Lifetime managed by
+ * MediaEncoder. All methods are to be called only on the worker thread.
+ *
+ * The control APIs are all called by MediaEncoder on its dedicated thread. Data
+ * is encoded as soon as it has been appended (and time has advanced past its
+ * end in case of video) and pushed to mEncodedDataQueue.
+ */
+class TrackEncoder {
+ public:
+  TrackEncoder(TrackRate aTrackRate,
+               MediaQueue<EncodedFrame>& aEncodedDataQueue);
+
+  /**
+   * Called by MediaEncoder to cancel the encoding.
+   */
+  virtual void Cancel() = 0;
+
+  /**
+   * Notifies us that we have reached the end of the stream and no more data
+   * will be appended.
+   */
+  virtual void NotifyEndOfStream() = 0;
+
+  /**
+   * Creates and sets up meta data for a specific codec, called on the worker
+   * thread.
+   */
+  virtual already_AddRefed<TrackMetadataBase> GetMetadata() = 0;
+
+  /**
+   * MediaQueue containing encoded data, that is pushed as soon as it's ready.
+   */
+  MediaQueue<EncodedFrame>& EncodedDataQueue() { return mEncodedDataQueue; }
+
+  /**
+   * Returns true once this TrackEncoder is initialized.
+   */
+  bool IsInitialized();
+
+  /**
+   * Returns true once this TrackEncoder has received some data.
+   */
+  bool IsStarted();
+
+  /**
+   * True if the track encoder has encoded all source segments coming from
+   * MediaTrackGraph. Call on the worker thread.
+   */
+  bool IsEncodingComplete() const;
+
+  /**
+   * Registers a listener to events from this TrackEncoder.
+   * We hold a strong reference to the listener.
+   */
+  void RegisterListener(TrackEncoderListener* aListener);
+
+  /**
+   * Unregisters a listener from events from this TrackEncoder.
+   * The listener will stop receiving events synchronously.
+   */
+  bool UnregisterListener(TrackEncoderListener* aListener);
+
+  virtual void SetBitrate(const uint32_t aBitrate) = 0;
+
+  /**
+   * It's optional to set the worker thread, but if you do we'll assert that
+   * we are in the worker thread in every method that gets called.
+   */
+  void SetWorkerThread(AbstractThread* aWorkerThread);
+
+  /**
+   * Measure size of internal buffers.
+   */
+  virtual size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) = 0;
+
+ protected:
+  virtual ~TrackEncoder() { MOZ_ASSERT(mListeners.IsEmpty()); }
+
+  /**
+   * If this TrackEncoder was not already initialized, it is set to initialized
+   * and listeners are notified.
+   */
+  void SetInitialized();
+
+  /**
+   * If this TrackEncoder was not already marked started, its started state is
+   * set and listeners are notified.
+   */
+  void SetStarted();
+
+  /**
+   * Called after an error. Cancels the encoding and notifies listeners.
+   */
+  void OnError();
+
+  /**
+   * True if the track encoder has been initialized successfully.
+   */
+  bool mInitialized;
+
+  /**
+   * True if the track encoder has received data.
+   */
+  bool mStarted;
+
+  /**
+   * True once all data until the end of the input track has been received.
+   */
+  bool mEndOfStream;
+
+  /**
+   * True once this encoding has been cancelled.
+   */
+  bool mCanceled;
+
+  // How many times we have tried to initialize the encoder.
+  uint32_t mInitCounter;
+
+  /**
+   * True if this TrackEncoder is currently suspended.
+   */
+  bool mSuspended;
+
+  /**
+   * The track rate of source media.
+   */
+  const TrackRate mTrackRate;
+
+  /**
+   * If set we assert that all methods are called on this thread.
+   */
+  RefPtr<AbstractThread> mWorkerThread;
+
+  /**
+   * MediaQueue where encoded data ends up. Note that metadata goes out of band.
+   */
+  MediaQueue<EncodedFrame>& mEncodedDataQueue;
+
+  nsTArray<RefPtr<TrackEncoderListener>> mListeners;
+};
+
+class AudioTrackEncoder : public TrackEncoder {
+ public:
+  AudioTrackEncoder(TrackRate aTrackRate,
+                    MediaQueue<EncodedFrame>& aEncodedDataQueue)
+      : TrackEncoder(aTrackRate, aEncodedDataQueue),
+        mChannels(0),
+        mNotInitDuration(0),
+        mAudioBitrate(0) {}
+
+  /**
+   * Suspends encoding from now, i.e., all future audio data received through
+   * AppendAudioSegment() until the next Resume() will be dropped.
+   */
+  void Suspend();
+
+  /**
+   * Resumes encoding starting now, i.e., data from the next
+   * AppendAudioSegment() will get encoded.
+   */
+  void Resume();
+
+  /**
+   * Appends and consumes track data from aSegment.
+   */
+  void AppendAudioSegment(AudioSegment&& aSegment);
+
+  template <typename T>
+  static void InterleaveTrackData(nsTArray<const T*>& aInput, int32_t aDuration,
+                                  uint32_t aOutputChannels,
+                                  AudioDataValue* aOutput, float aVolume) {
+    if (aInput.Length() < aOutputChannels) {
+      // Up-mix. This might make the mChannelData have more than aChannels.
+      AudioChannelsUpMix(&aInput, aOutputChannels,
+                         SilentChannel::ZeroChannel<T>());
+    }
+
+    if (aInput.Length() > aOutputChannels) {
+      DownmixAndInterleave(aInput, aDuration, aVolume, aOutputChannels,
+                           aOutput);
+    } else {
+      InterleaveAndConvertBuffer(aInput.Elements(), aDuration, aVolume,
+                                 aOutputChannels, aOutput);
+    }
+  }
+
+  /**
+   * Interleaves the track data and stores the result into aOutput. Might need
+   * to up-mix or down-mix the channel data if the channels number of this chunk
+   * is different from aOutputChannels. The channel data from aChunk might be
+   * modified by up-mixing.
+   */
+  static void InterleaveTrackData(AudioChunk& aChunk, int32_t aDuration,
+                                  uint32_t aOutputChannels,
+                                  AudioDataValue* aOutput);
+
+  /**
+   * De-interleaves the aInput data and stores the result into aOutput.
+   * No up-mix or down-mix operations inside.
+   */
+  static void DeInterleaveTrackData(AudioDataValue* aInput, int32_t aDuration,
+                                    int32_t aChannels, AudioDataValue* aOutput);
+
+  /**
+   * Measure size of internal buffers.
+   */
+  size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
+
+  void SetBitrate(const uint32_t aBitrate) override {
+    mAudioBitrate = aBitrate;
+  }
+
+  /**
+   * Tries to initiate the AudioEncoder based on data in aSegment.
+   * This can be re-called often, as it will exit early should we already be
+   * initiated. mInitiated will only be set if there was enough data in
+   * aSegment to infer metadata. If mInitiated gets set, listeners are notified.
+   *
+   * Not having enough data in aSegment to initiate the encoder for an
+   * accumulated aDuration of one second will make us initiate with a default
+   * number of channels.
+   *
+   * If we attempt to initiate the underlying encoder but fail, we Cancel() and
+   * notify listeners.
+   */
+  void TryInit(const AudioSegment& aSegment, TrackTime aDuration);
+
+  void Cancel() override;
+
+  /**
+   * Dispatched from MediaTrackGraph when we have finished feeding data to
+   * mOutgoingBuffer.
+   */
+  void NotifyEndOfStream() override;
+
+ protected:
+  /**
+   * Number of samples per channel in a pcm buffer. This is also the value of
+   * frame size required by audio encoder, and listeners will be notified when
+   * at least this much data has been added to mOutgoingBuffer.
+   */
+  virtual int NumInputFramesPerPacket() const { return 0; }
+
+  /**
+   * Initializes the audio encoder. The call of this method is delayed until we
+   * have received the first valid track from MediaTrackGraph.
+   */
+  virtual nsresult Init(int aChannels) = 0;
+
+  /**
+   * Encodes buffered data and pushes it to mEncodedDataQueue.
+   */
+  virtual nsresult Encode(AudioSegment* aSegment) = 0;
+
+  /**
+   * The number of channels are used for processing PCM data in the audio
+   * encoder. This value comes from the first valid audio chunk. If encoder
+   * can't support the channels in the chunk, downmix PCM stream can be
+   * performed. This value also be used to initialize the audio encoder.
+   */
+  int mChannels;
+
+  /**
+   * A segment queue of outgoing audio track data to the encoder.
+   * The contents of mOutgoingBuffer will always be what has been appended on
+   * the encoder thread but not yet consumed by the encoder sub class.
+   */
+  AudioSegment mOutgoingBuffer;
+
+  TrackTime mNotInitDuration;
+
+  uint32_t mAudioBitrate;
+};
+
+enum class FrameDroppingMode {
+  ALLOW,     // Allowed to drop frames to keep up under load
+  DISALLOW,  // Must not drop any frames, even if it means we will OOM
+};
+
+class VideoTrackEncoder : public TrackEncoder {
+ public:
+  VideoTrackEncoder(RefPtr<DriftCompensator> aDriftCompensator,
+                    TrackRate aTrackRate,
+                    MediaQueue<EncodedFrame>& aEncodedDataQueue,
+                    FrameDroppingMode aFrameDroppingMode);
+
+  /**
+   * Suspends encoding from aTime, i.e., all video frame with a timestamp
+   * between aTime and the timestamp of the next Resume() will be dropped.
+   */
+  void Suspend(const TimeStamp& aTime);
+
+  /**
+   * Resumes encoding starting at aTime.
+   */
+  void Resume(const TimeStamp& aTime);
+
+  /**
+   * Makes the video black from aTime.
+   */
+  void Disable(const TimeStamp& aTime);
+
+  /**
+   * Makes the video non-black from aTime.
+   *
+   * NB that it could still be forced black for other reasons, like principals.
+   */
+  void Enable(const TimeStamp& aTime);
+
+  /**
+   * Appends source video frames to mIncomingBuffer. We only append the source
+   * chunk if the image is different from mLastChunk's image. Called on the
+   * MediaTrackGraph thread.
+   */
+  void AppendVideoSegment(VideoSegment&& aSegment);
+
+  /**
+   * Measure size of internal buffers.
+   */
+  size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
+
+  void SetBitrate(const uint32_t aBitrate) override {
+    mVideoBitrate = aBitrate;
+  }
+
+  /**
+   * Tries to initiate the VideoEncoder based on data in aSegment.
+   * This can be re-called often, as it will exit early should we already be
+   * initiated. mInitiated will only be set if there was enough data in
+   * aSegment to infer metadata. If mInitiated gets set, listeners are notified.
+   * The amount of chunks needed can be controlled by
+   * aFrameRateDetectionMinChunks which denotes the minimum number of chunks
+   * needed to infer the framerate.
+   *
+   * Failing to initiate the encoder for an accumulated aDuration of 30 seconds
+   * is seen as an error and will cancel the current encoding.
+   */
+  void Init(const VideoSegment& aSegment, const TimeStamp& aTime,
+            size_t aFrameRateDetectionMinChunks);
+
+  TrackTime SecondsToMediaTime(double aS) const {
+    NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX / TRACK_RATE_MAX,
+                 "Bad seconds");
+    return mTrackRate * aS;
+  }
+
+  /**
+   * MediaTrackGraph notifies us about the time of the track's start.
+   * This gets called on the MediaEncoder thread after a dispatch.
+   */
+  void SetStartOffset(const TimeStamp& aStartOffset);
+
+  void Cancel() override;
+
+  /**
+   * Notifies us that we have reached the end of the stream and no more data
+   * will be appended to mIncomingBuffer.
+   */
+  void NotifyEndOfStream() override;
+
+  /**
+   * Dispatched from MediaTrackGraph when it has run an iteration so we can
+   * hand more data to the encoder.
+   */
+  void AdvanceCurrentTime(const TimeStamp& aTime);
+
+ protected:
+  /**
+   * Initialize the video encoder. In order to collect the value of width and
+   * height of source frames, this initialization is delayed until we have
+   * received the first valid video frame from MediaTrackGraph.
+   * Listeners will be notified after it has been successfully initialized.
+   */
+  virtual nsresult Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
+                        int32_t aDisplayHeight, float aEstimatedFrameRate) = 0;
+
+  /**
+   * Encodes data in the outgoing buffer and pushes it to mEncodedDataQueue.
+   */
+  virtual nsresult Encode(VideoSegment* aSegment) = 0;
+
+  /**
+   * Drift compensator for re-clocking incoming video frame wall-clock
+   * timestamps to audio time.
+   */
+  const RefPtr<DriftCompensator> mDriftCompensator;
+
+  /**
+   * The last unique frame and duration so far handled by
+   * NotifyAdvanceCurrentTime. When a new frame is detected, mLastChunk is added
+   * to mOutgoingBuffer.
+   */
+  VideoChunk mLastChunk;
+
+  /**
+   * A segment queue of incoming video track data, from listeners.
+   * The duration of mIncomingBuffer is irrelevant as we only look at TimeStamps
+   * of frames. Consumed data is replaced by null data.
+   */
+  VideoSegment mIncomingBuffer;
+
+  /**
+   * A segment queue of outgoing video track data to the encoder.
+   * The contents of mOutgoingBuffer will always be what has been consumed from
+   * mIncomingBuffer (up to mCurrentTime) but not yet consumed by the encoder
+   * sub class. There won't be any null data at the beginning of mOutgoingBuffer
+   * unless explicitly pushed by the producer.
+   */
+  VideoSegment mOutgoingBuffer;
+
+  /**
+   * The number of mTrackRate ticks we have passed to mOutgoingBuffer.
+   */
+  TrackTime mEncodedTicks;
+
+  /**
+   * The time up to which we have forwarded data from mIncomingBuffer to
+   * mOutgoingBuffer.
+   */
+  TimeStamp mCurrentTime;
+
+  /**
+   * The time the video track started, so the start of the video track can be
+   * synced to the start of the audio track.
+   *
+   * Note that this time will progress during suspension, to make sure the
+   * incoming frames stay in sync with the output.
+   */
+  TimeStamp mStartTime;
+
+  /**
+   * The time Suspend was called on the MediaRecorder, so we can calculate the
+   * duration on the next Resume().
+   */
+  TimeStamp mSuspendTime;
+
+  uint32_t mVideoBitrate;
+
+  /**
+   * ALLOW to drop frames under load.
+   * DISALLOW to encode all frames, mainly for testing.
+   */
+  FrameDroppingMode mFrameDroppingMode;
+
+  /**
+   * True if the video MediaTrackTrack this VideoTrackEncoder is attached to is
+   * currently enabled. While false, we encode all frames as black.
+   */
+  bool mEnabled;
+};
+
+}  // namespace mozilla
+
+#endif