summaryrefslogtreecommitdiffstats
path: root/dom/media/encoder/MediaEncoder.h
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
commit6bf0a5cb5034a7e684dcc3500e841785237ce2dd (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /dom/media/encoder/MediaEncoder.h
parentInitial commit. (diff)
downloadthunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.tar.xz
thunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/encoder/MediaEncoder.h')
-rw-r--r--dom/media/encoder/MediaEncoder.h400
1 files changed, 400 insertions, 0 deletions
diff --git a/dom/media/encoder/MediaEncoder.h b/dom/media/encoder/MediaEncoder.h
new file mode 100644
index 0000000000..dae887edc6
--- /dev/null
+++ b/dom/media/encoder/MediaEncoder.h
@@ -0,0 +1,400 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MediaEncoder_h_
+#define MediaEncoder_h_
+
+#include "ContainerWriter.h"
+#include "CubebUtils.h"
+#include "MediaQueue.h"
+#include "MediaTrackGraph.h"
+#include "MediaTrackListener.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MemoryReporting.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/UniquePtr.h"
+#include "nsIMemoryReporter.h"
+#include "TrackEncoder.h"
+
+namespace mozilla {
+
+class DriftCompensator;
+class Muxer;
+class Runnable;
+class TaskQueue;
+
+namespace dom {
+class AudioNode;
+class AudioStreamTrack;
+class BlobImpl;
+class MediaStreamTrack;
+class MutableBlobStorage;
+class VideoStreamTrack;
+} // namespace dom
+
+class DriftCompensator;
+
+/**
+ * MediaEncoder is the framework of encoding module, it controls and manages
+ * procedures between Muxer, ContainerWriter and TrackEncoder. ContainerWriter
+ * writes the encoded track data into a specific container (e.g. ogg, webm).
+ * AudioTrackEncoder and VideoTrackEncoder are subclasses of TrackEncoder, and
+ * are responsible for encoding raw data coming from MediaStreamTracks.
+ *
+ * MediaEncoder solves threading issues by doing message passing to a TaskQueue
+ * (the "encoder thread") as passed in to the constructor. Each
+ * MediaStreamTrack to be recorded is set up with a MediaTrackListener.
+ * Typically there are a non-direct track listeners for audio, direct listeners
+ * for video, and there is always a non-direct listener on each track for
+ * time-keeping. The listeners forward data to their corresponding TrackEncoders
+ * on the encoder thread.
+ *
+ * The MediaEncoder listens to events from all TrackEncoders, and in turn
+ * signals events to interested parties. Typically a MediaRecorder::Session.
+ * The MediaEncoder automatically encodes incoming data, muxes it, writes it
+ * into a container and stores the container data into a MutableBlobStorage.
+ * It is timeslice-aware so that it can notify listeners when it's time to
+ * expose a blob due to filling the timeslice.
+ *
+ * MediaEncoder is designed to be a passive component, neither does it own or is
+ * in charge of managing threads. Instead this is done by its owner.
+ *
+ * For example, usage from MediaRecorder of this component would be:
+ * 1) Create an encoder with a valid MIME type. Note that there are more
+ * configuration options, see the docs on MediaEncoder::CreateEncoder.
+ * => encoder = MediaEncoder::CreateEncoder(aMIMEType);
+ * It then creates track encoders and the appropriate ContainerWriter
+ * according to the MIME type
+ *
+ * 2) Connect handlers through MediaEventListeners to the MediaEncoder's
+ * MediaEventSources, StartedEvent(), DataAvailableEvent(), ErrorEvent() and
+ * ShutdownEvent().
+ * => listener = encoder->DataAvailableEvent().Connect(mainThread, &OnBlob);
+ *
+ * 3) Connect the sources to be recorded. Either through:
+ * => encoder->ConnectAudioNode(node);
+ * or
+ * => encoder->ConnectMediaStreamTrack(track);
+ * These should not be mixed. When connecting MediaStreamTracks there is
+ * support for at most one of each kind.
+ *
+ * 4) MediaEncoder automatically encodes data from the connected tracks, muxes
+ * them and writes it all into a blob, including metadata. When the blob
+ * contains at least `timeslice` worth of data it notifies the
+ * DataAvailableEvent that was connected in step 2.
+ * => void OnBlob(RefPtr<BlobImpl> aBlob) {
+ * => DispatchBlobEvent(Blob::Create(GetOwnerGlobal(), aBlob));
+ * => };
+ *
+ * 5) To stop encoding, there are multiple options:
+ *
+ * 5.1) Stop() for a graceful stop.
+ * => encoder->Stop();
+ *
+ * 5.2) Cancel() for an immediate stop, if you don't need the data currently
+ * buffered.
+ * => encoder->Cancel();
+ *
+ * 5.3) When all input tracks end, the MediaEncoder will automatically stop
+ * and shut down.
+ */
+class MediaEncoder {
+ private:
+ class AudioTrackListener;
+ class VideoTrackListener;
+ class EncoderListener;
+
+ public:
+ using BlobPromise =
+ MozPromise<RefPtr<dom::BlobImpl>, nsresult, false /* IsExclusive */>;
+ using SizeOfPromise = MozPromise<size_t, size_t, true /* IsExclusive */>;
+
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaEncoder)
+
+ MediaEncoder(RefPtr<TaskQueue> aEncoderThread,
+ RefPtr<DriftCompensator> aDriftCompensator,
+ UniquePtr<ContainerWriter> aWriter,
+ UniquePtr<AudioTrackEncoder> aAudioEncoder,
+ UniquePtr<VideoTrackEncoder> aVideoEncoder,
+ UniquePtr<MediaQueue<EncodedFrame>> aEncodedAudioQueue,
+ UniquePtr<MediaQueue<EncodedFrame>> aEncodedVideoQueue,
+ TrackRate aTrackRate, const nsAString& aMIMEType,
+ uint64_t aMaxMemory, TimeDuration aTimeslice);
+
+ /**
+ * Called on main thread from MediaRecorder::Pause.
+ */
+ void Suspend();
+
+ /**
+ * Called on main thread from MediaRecorder::Resume.
+ */
+ void Resume();
+
+ /**
+ * Disconnects the input tracks, causing the encoding to stop.
+ */
+ void DisconnectTracks();
+
+ /**
+ * Connects an AudioNode with the appropriate encoder.
+ */
+ void ConnectAudioNode(dom::AudioNode* aNode, uint32_t aOutput);
+
+ /**
+ * Connects a MediaStreamTrack with the appropriate encoder.
+ */
+ void ConnectMediaStreamTrack(dom::MediaStreamTrack* aTrack);
+
+ /**
+ * Removes a connected MediaStreamTrack.
+ */
+ void RemoveMediaStreamTrack(dom::MediaStreamTrack* aTrack);
+
+ /**
+ * Creates an encoder with the given MIME type. This must be a valid MIME type
+ * or we will crash hard.
+ * Bitrates are given either explicit, or with 0 for defaults.
+ * aTrackRate is the rate in which data will be fed to the TrackEncoders.
+ * aMaxMemory is the maximum number of bytes of muxed data allowed in memory.
+ * Beyond that the blob is moved to a temporary file.
+ * aTimeslice is the minimum duration of muxed data we gather before
+ * automatically issuing a dataavailable event.
+ */
+ static already_AddRefed<MediaEncoder> CreateEncoder(
+ RefPtr<TaskQueue> aEncoderThread, const nsAString& aMimeType,
+ uint32_t aAudioBitrate, uint32_t aVideoBitrate, uint8_t aTrackTypes,
+ TrackRate aTrackRate, uint64_t aMaxMemory, TimeDuration aTimeslice);
+
+ /**
+ * Encodes raw data for all tracks to aOutputBufs. The buffer of container
+ * data is allocated in ContainerWriter::GetContainerData().
+ *
+ * On its first call, metadata is also encoded. TrackEncoders must have been
+ * initialized before this is called.
+ */
+ nsresult GetEncodedData(nsTArray<nsTArray<uint8_t>>* aOutputBufs);
+
+ /**
+ * Asserts that Shutdown() has been called. Reasons are encoding
+ * complete, encounter an error, or being canceled by its caller.
+ */
+ void AssertShutdownCalled() { MOZ_ASSERT(mShutdownPromise); }
+
+ /**
+ * Stops (encoding any data currently buffered) the encoding and shuts down
+ * the encoder using Shutdown().
+ */
+ RefPtr<GenericNonExclusivePromise> Stop();
+
+ /**
+ * Cancels (discarding any data currently buffered) the encoding and shuts
+ * down the encoder using Shutdown().
+ */
+ RefPtr<GenericNonExclusivePromise> Cancel();
+
+ bool HasError();
+
+ static bool IsWebMEncoderEnabled();
+
+ /**
+ * Updates internal state when track encoders are all initialized.
+ */
+ void UpdateInitialized();
+
+ /**
+ * Updates internal state when track encoders are all initialized, and
+ * notifies listeners that this MediaEncoder has been started.
+ */
+ void UpdateStarted();
+
+ MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf)
+ /*
+ * Measure the size of the buffer, and heap memory in bytes occupied by
+ * mAudioEncoder and mVideoEncoder.
+ */
+ RefPtr<SizeOfPromise> SizeOfExcludingThis(
+ mozilla::MallocSizeOf aMallocSizeOf);
+
+ /**
+ * Encode, mux and store into blob storage what has been buffered until now,
+ * then return the blob backed by that storage.
+ */
+ RefPtr<BlobPromise> RequestData();
+
+ // Event that gets notified when all track encoders have received data.
+ MediaEventSource<void>& StartedEvent() { return mStartedEvent; }
+ // Event that gets notified when there was an error preventing continued
+ // recording somewhere in the MediaEncoder stack.
+ MediaEventSource<void>& ErrorEvent() { return mErrorEvent; }
+ // Event that gets notified when the MediaEncoder stack has been shut down.
+ MediaEventSource<void>& ShutdownEvent() { return mShutdownEvent; }
+ // Event that gets notified after we have muxed at least mTimeslice worth of
+ // data into the current blob storage.
+ MediaEventSource<RefPtr<dom::BlobImpl>>& DataAvailableEvent() {
+ return mDataAvailableEvent;
+ }
+
+ protected:
+ ~MediaEncoder();
+
+ private:
+ /**
+ * Sets mGraphTrack if not already set, using a new stream from aTrack's
+ * graph.
+ */
+ void EnsureGraphTrackFrom(MediaTrack* aTrack);
+
+ /**
+ * Takes a regular runnable and dispatches it to the graph wrapped in a
+ * ControlMessage.
+ */
+ void RunOnGraph(already_AddRefed<Runnable> aRunnable);
+
+ /**
+ * Shuts down gracefully if there is no remaining live track encoder.
+ */
+ void MaybeShutdown();
+
+ /**
+ * Waits for TrackEncoders to shut down, then shuts down the MediaEncoder and
+ * cleans up track encoders.
+ */
+ RefPtr<GenericNonExclusivePromise> Shutdown();
+
+ /**
+ * Sets mError to true, notifies listeners of the error if mError changed,
+ * and stops encoding.
+ */
+ void SetError();
+
+ /**
+ * Creates a new MutableBlobStorage if one doesn't exist.
+ */
+ void MaybeCreateMutableBlobStorage();
+
+ /**
+ * Called when an encoded audio frame has been pushed by the audio encoder.
+ */
+ void OnEncodedAudioPushed(const RefPtr<EncodedFrame>& aFrame);
+
+ /**
+ * Called when an encoded video frame has been pushed by the video encoder.
+ */
+ void OnEncodedVideoPushed(const RefPtr<EncodedFrame>& aFrame);
+
+ /**
+ * If enough data has been pushed to the muxer, extract it into the current
+ * blob storage. If more than mTimeslice data has been pushed to the muxer
+ * since the last DataAvailableEvent was notified, also gather the blob and
+ * notify MediaRecorder.
+ */
+ void MaybeExtractOrGatherBlob();
+
+ // Extracts encoded and muxed data into the current blob storage, creating one
+ // if it doesn't exist. The returned promise resolves when data has been
+ // stored into the blob.
+ RefPtr<GenericPromise> Extract();
+
+ // Stops gathering data into the current blob and resolves when the current
+ // blob is available. Future data will be stored in a new blob.
+ // Should a previous async GatherBlob() operation still be in progress, we'll
+ // wait for it to finish before starting this one.
+ RefPtr<BlobPromise> GatherBlob();
+
+ RefPtr<BlobPromise> GatherBlobImpl();
+
+ const RefPtr<nsISerialEventTarget> mMainThread;
+ const RefPtr<TaskQueue> mEncoderThread;
+ const RefPtr<DriftCompensator> mDriftCompensator;
+
+ const UniquePtr<MediaQueue<EncodedFrame>> mEncodedAudioQueue;
+ const UniquePtr<MediaQueue<EncodedFrame>> mEncodedVideoQueue;
+
+ const UniquePtr<Muxer> mMuxer;
+ const UniquePtr<AudioTrackEncoder> mAudioEncoder;
+ const RefPtr<AudioTrackListener> mAudioListener;
+ const UniquePtr<VideoTrackEncoder> mVideoEncoder;
+ const RefPtr<VideoTrackListener> mVideoListener;
+ const RefPtr<EncoderListener> mEncoderListener;
+
+ public:
+ const nsString mMimeType;
+
+ // Max memory to use for the MutableBlobStorage.
+ const uint64_t mMaxMemory;
+
+ // The interval of passing encoded data from MutableBlobStorage to
+ // onDataAvailable handler.
+ const TimeDuration mTimeslice;
+
+ private:
+ MediaEventListener mAudioPushListener;
+ MediaEventListener mAudioFinishListener;
+ MediaEventListener mVideoPushListener;
+ MediaEventListener mVideoFinishListener;
+
+ MediaEventProducer<void> mStartedEvent;
+ MediaEventProducer<void> mErrorEvent;
+ MediaEventProducer<void> mShutdownEvent;
+ MediaEventProducer<RefPtr<dom::BlobImpl>> mDataAvailableEvent;
+
+ // The AudioNode we are encoding.
+ // Will be null when input is media stream or destination node.
+ RefPtr<dom::AudioNode> mAudioNode;
+ // Pipe-track for allowing a track listener on a non-destination AudioNode.
+ // Will be null when input is media stream or destination node.
+ RefPtr<AudioNodeTrack> mPipeTrack;
+ // Input port that connect mAudioNode to mPipeTrack.
+ // Will be null when input is media stream or destination node.
+ RefPtr<MediaInputPort> mInputPort;
+ // An audio track that we are encoding. Will be null if the input stream
+ // doesn't contain audio on start() or if the input is an AudioNode.
+ RefPtr<dom::AudioStreamTrack> mAudioTrack;
+ // A video track that we are encoding. Will be null if the input stream
+ // doesn't contain video on start() or if the input is an AudioNode.
+ RefPtr<dom::VideoStreamTrack> mVideoTrack;
+
+ // A stream to keep the MediaTrackGraph alive while we're recording.
+ RefPtr<SharedDummyTrack> mGraphTrack;
+
+ // A buffer to cache muxed encoded data.
+ RefPtr<dom::MutableBlobStorage> mMutableBlobStorage;
+ // If set, is a promise for the latest GatherBlob() operation. Allows
+ // GatherBlob() operations to be serialized in order to avoid races.
+ RefPtr<BlobPromise> mBlobPromise;
+ // The end time of the muxed data in the last gathered blob. If more than one
+ // track is present, this is the end time of the track that ends the earliest
+ // in the last blob. Encoder thread only.
+ media::TimeUnit mLastBlobTime;
+ // The end time of the muxed data in the current blob storage. If more than
+ // one track is present, this is the end time of the track that ends the
+ // earliest in the current blob storage. Encoder thread only.
+ media::TimeUnit mLastExtractTime;
+ // The end time of encoded audio data sent to the muxer. Positive infinity if
+ // there is no audio encoder. Encoder thread only.
+ media::TimeUnit mMuxedAudioEndTime;
+ // The end time of encoded video data sent to the muxer. Positive infinity if
+ // there is no video encoder. Encoder thread only.
+ media::TimeUnit mMuxedVideoEndTime;
+
+ TimeStamp mStartTime;
+ bool mInitialized;
+ bool mStarted;
+ bool mCompleted;
+ bool mError;
+ // Set when shutdown starts.
+ RefPtr<GenericNonExclusivePromise> mShutdownPromise;
+ // Get duration from create encoder, for logging purpose
+ double GetEncodeTimeStamp() {
+ TimeDuration decodeTime;
+ decodeTime = TimeStamp::Now() - mStartTime;
+ return decodeTime.ToMilliseconds();
+ }
+};
+
+} // namespace mozilla
+
+#endif