firefox/dom/media/encoder/MediaEncoder.h

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef MediaEncoder_h_
#define MediaEncoder_h_

#include "ContainerWriter.h"
#include "CubebUtils.h"
#include "MediaQueue.h"
#include "MediaTrackGraph.h"
#include "MediaTrackListener.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/MemoryReporting.h"
#include "mozilla/MozPromise.h"
#include "mozilla/UniquePtr.h"
#include "nsIMemoryReporter.h"
#include "TrackEncoder.h"

namespace mozilla {

class DriftCompensator;
class Muxer;
class Runnable;
class TaskQueue;

namespace dom {
class AudioNode;
class AudioStreamTrack;
class BlobImpl;
class MediaStreamTrack;
class MutableBlobStorage;
class VideoStreamTrack;
}  // namespace dom

class DriftCompensator;

/**
 * MediaEncoder is the framework of encoding module, it controls and manages
 * procedures between Muxer, ContainerWriter and TrackEncoder. ContainerWriter
 * writes the encoded track data into a specific container (e.g. ogg, webm).
 * AudioTrackEncoder and VideoTrackEncoder are subclasses of TrackEncoder, and
 * are responsible for encoding raw data coming from MediaStreamTracks.
 *
 * MediaEncoder solves threading issues by doing message passing to a TaskQueue
 * (the "encoder thread") as passed in to the constructor. Each
 * MediaStreamTrack to be recorded is set up with a MediaTrackListener.
 * Typically there are a non-direct track listeners for audio, direct listeners
 * for video, and there is always a non-direct listener on each track for
 * time-keeping. The listeners forward data to their corresponding TrackEncoders
 * on the encoder thread.
 *
 * The MediaEncoder listens to events from all TrackEncoders, and in turn
 * signals events to interested parties. Typically a MediaRecorder::Session.
 * The MediaEncoder automatically encodes incoming data, muxes it, writes it
 * into a container and stores the container data into a MutableBlobStorage.
 * It is timeslice-aware so that it can notify listeners when it's time to
 * expose a blob due to filling the timeslice.
 *
 * MediaEncoder is designed to be a passive component, neither does it own or is
 * in charge of managing threads. Instead this is done by its owner.
 *
 * For example, usage from MediaRecorder of this component would be:
 * 1) Create an encoder with a valid MIME type. Note that there are more
 *    configuration options, see the docs on MediaEncoder::CreateEncoder.
 *    => encoder = MediaEncoder::CreateEncoder(aMIMEType);
 *    It then creates track encoders and the appropriate ContainerWriter
 *    according to the MIME type
 *
 * 2) Connect handlers through MediaEventListeners to the MediaEncoder's
 *    MediaEventSources, StartedEvent(), DataAvailableEvent(), ErrorEvent() and
 *    ShutdownEvent().
 *    => listener = encoder->DataAvailableEvent().Connect(mainThread, &OnBlob);
 *
 * 3) Connect the sources to be recorded. Either through:
 *    => encoder->ConnectAudioNode(node);
 *    or
 *    => encoder->ConnectMediaStreamTrack(track);
 *    These should not be mixed. When connecting MediaStreamTracks there is
 *    support for at most one of each kind.
 *
 * 4) MediaEncoder automatically encodes data from the connected tracks, muxes
 *    them and writes it all into a blob, including metadata. When the blob
 *    contains at least `timeslice` worth of data it notifies the
 *    DataAvailableEvent that was connected in step 2.
 *    => void OnBlob(RefPtr<BlobImpl> aBlob) {
 *    =>   DispatchBlobEvent(Blob::Create(GetOwnerGlobal(), aBlob));
 *    => };
 *
 * 5) To stop encoding, there are multiple options:
 *
 *    5.1) Stop() for a graceful stop.
 *         => encoder->Stop();
 *
 *    5.2) Cancel() for an immediate stop, if you don't need the data currently
 *         buffered.
 *         => encoder->Cancel();
 *
 *    5.3) When all input tracks end, the MediaEncoder will automatically stop
 *         and shut down.
 */
class MediaEncoder {
 private:
  class AudioTrackListener;
  class VideoTrackListener;
  class EncoderListener;

 public:
  using BlobPromise =
      MozPromise<RefPtr<dom::BlobImpl>, nsresult, false /* IsExclusive */>;
  using SizeOfPromise = MozPromise<size_t, size_t, true /* IsExclusive */>;

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaEncoder)

 private:
  MediaEncoder(RefPtr<TaskQueue> aEncoderThread,
               RefPtr<DriftCompensator> aDriftCompensator,
               UniquePtr<ContainerWriter> aWriter,
               UniquePtr<AudioTrackEncoder> aAudioEncoder,
               UniquePtr<VideoTrackEncoder> aVideoEncoder,
               UniquePtr<MediaQueue<EncodedFrame>> aEncodedAudioQueue,
               UniquePtr<MediaQueue<EncodedFrame>> aEncodedVideoQueue,
               TrackRate aTrackRate, const nsAString& aMIMEType,
               uint64_t aMaxMemory, TimeDuration aTimeslice);

 public:
  /**
   * Called on main thread from MediaRecorder::Pause.
   */
  void Suspend();

  /**
   * Called on main thread from MediaRecorder::Resume.
   */
  void Resume();

  /**
   * Disconnects the input tracks, causing the encoding to stop.
   */
  void DisconnectTracks();

  /**
   * Connects an AudioNode with the appropriate encoder.
   */
  void ConnectAudioNode(dom::AudioNode* aNode, uint32_t aOutput);

  /**
   * Connects a MediaStreamTrack with the appropriate encoder.
   */
  void ConnectMediaStreamTrack(dom::MediaStreamTrack* aTrack);

  /**
   * Removes a connected MediaStreamTrack.
   */
  void RemoveMediaStreamTrack(dom::MediaStreamTrack* aTrack);

  /**
   * Creates an encoder with the given MIME type. This must be a valid MIME type
   * or we will crash hard.
   * Bitrates are given either explicit, or with 0 for defaults.
   * aTrackRate is the rate in which data will be fed to the TrackEncoders.
   * aMaxMemory is the maximum number of bytes of muxed data allowed in memory.
   * Beyond that the blob is moved to a temporary file.
   * aTimeslice is the minimum duration of muxed data we gather before
   * automatically issuing a dataavailable event.
   */
  static already_AddRefed<MediaEncoder> CreateEncoder(
      RefPtr<TaskQueue> aEncoderThread, const nsAString& aMimeType,
      uint32_t aAudioBitrate, uint32_t aVideoBitrate, uint8_t aTrackTypes,
      TrackRate aTrackRate, uint64_t aMaxMemory, TimeDuration aTimeslice);

  /**
   * Encodes raw data for all tracks to aOutputBufs. The buffer of container
   * data is allocated in ContainerWriter::GetContainerData().
   *
   * On its first call, metadata is also encoded. TrackEncoders must have been
   * initialized before this is called.
   */
  nsresult GetEncodedData(nsTArray<nsTArray<uint8_t>>* aOutputBufs);

  /**
   * Asserts that Shutdown() has been called. Reasons are encoding
   * complete, encounter an error, or being canceled by its caller.
   */
  void AssertShutdownCalled() { MOZ_ASSERT(mShutdownPromise); }

  /**
   * Stops (encoding any data currently buffered) the encoding and shuts down
   * the encoder using Shutdown().
   */
  RefPtr<GenericNonExclusivePromise> Stop();

  /**
   * Cancels (discarding any data currently buffered) the encoding and shuts
   * down the encoder using Shutdown().
   */
  RefPtr<GenericNonExclusivePromise> Cancel();

  bool HasError();

  static bool IsWebMEncoderEnabled();

  /**
   * Updates internal state when track encoders are all initialized.
   */
  void UpdateInitialized();

  /**
   * Updates internal state when track encoders are all initialized, and
   * notifies listeners that this MediaEncoder has been started.
   */
  void UpdateStarted();

  MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf)
  /*
   * Measure the size of the buffer, and heap memory in bytes occupied by
   * mAudioEncoder and mVideoEncoder.
   */
  RefPtr<SizeOfPromise> SizeOfExcludingThis(
      mozilla::MallocSizeOf aMallocSizeOf);

  /**
   * Encode, mux and store into blob storage what has been buffered until now,
   * then return the blob backed by that storage.
   */
  RefPtr<BlobPromise> RequestData();

  // Event that gets notified when all track encoders have received data.
  MediaEventSource<void>& StartedEvent() { return mStartedEvent; }
  // Event that gets notified when there was an error preventing continued
  // recording somewhere in the MediaEncoder stack.
  MediaEventSource<void>& ErrorEvent() { return mErrorEvent; }
  // Event that gets notified when the MediaEncoder stack has been shut down.
  MediaEventSource<void>& ShutdownEvent() { return mShutdownEvent; }
  // Event that gets notified after we have muxed at least mTimeslice worth of
  // data into the current blob storage.
  MediaEventSource<RefPtr<dom::BlobImpl>>& DataAvailableEvent() {
    return mDataAvailableEvent;
  }

 protected:
  ~MediaEncoder();

 private:
  /**
   * Registers listeners.
   */
  void RegisterListeners();

  /**
   * Sets mGraphTrack if not already set, using a new stream from aTrack's
   * graph.
   */
  void EnsureGraphTrackFrom(MediaTrack* aTrack);

  /**
   * Shuts down gracefully if there is no remaining live track encoder.
   */
  void MaybeShutdown();

  /**
   * Waits for TrackEncoders to shut down, then shuts down the MediaEncoder and
   * cleans up track encoders.
   */
  RefPtr<GenericNonExclusivePromise> Shutdown();

  /**
   * Sets mError to true, notifies listeners of the error if mError changed,
   * and stops encoding.
   */
  void SetError();

  /**
   * Creates a new MutableBlobStorage if one doesn't exist.
   */
  void MaybeCreateMutableBlobStorage();

  /**
   * Called when an encoded audio frame has been pushed by the audio encoder.
   */
  void OnEncodedAudioPushed(const RefPtr<EncodedFrame>& aFrame);

  /**
   * Called when an encoded video frame has been pushed by the video encoder.
   */
  void OnEncodedVideoPushed(const RefPtr<EncodedFrame>& aFrame);

  /**
   * If enough data has been pushed to the muxer, extract it into the current
   * blob storage. If more than mTimeslice data has been pushed to the muxer
   * since the last DataAvailableEvent was notified, also gather the blob and
   * notify MediaRecorder.
   */
  void MaybeExtractOrGatherBlob();

  // Extracts encoded and muxed data into the current blob storage, creating one
  // if it doesn't exist. The returned promise resolves when data has been
  // stored into the blob.
  RefPtr<GenericPromise> Extract();

  // Stops gathering data into the current blob and resolves when the current
  // blob is available. Future data will be stored in a new blob.
  // Should a previous async GatherBlob() operation still be in progress, we'll
  // wait for it to finish before starting this one.
  RefPtr<BlobPromise> GatherBlob();

  RefPtr<BlobPromise> GatherBlobImpl();

  const RefPtr<nsISerialEventTarget> mMainThread;
  const RefPtr<TaskQueue> mEncoderThread;
  const RefPtr<DriftCompensator> mDriftCompensator;

  const UniquePtr<MediaQueue<EncodedFrame>> mEncodedAudioQueue;
  const UniquePtr<MediaQueue<EncodedFrame>> mEncodedVideoQueue;

  const UniquePtr<Muxer> mMuxer;
  const UniquePtr<AudioTrackEncoder> mAudioEncoder;
  const RefPtr<AudioTrackListener> mAudioListener;
  const UniquePtr<VideoTrackEncoder> mVideoEncoder;
  const RefPtr<VideoTrackListener> mVideoListener;
  const RefPtr<EncoderListener> mEncoderListener;

 public:
  const nsString mMimeType;

  // Max memory to use for the MutableBlobStorage.
  const uint64_t mMaxMemory;

  // The interval of passing encoded data from MutableBlobStorage to
  // onDataAvailable handler.
  const TimeDuration mTimeslice;

 private:
  MediaEventListener mAudioPushListener;
  MediaEventListener mAudioFinishListener;
  MediaEventListener mVideoPushListener;
  MediaEventListener mVideoFinishListener;

  MediaEventProducer<void> mStartedEvent;
  MediaEventProducer<void> mErrorEvent;
  MediaEventProducer<void> mShutdownEvent;
  MediaEventProducer<RefPtr<dom::BlobImpl>> mDataAvailableEvent;

  // The AudioNode we are encoding.
  // Will be null when input is media stream or destination node.
  RefPtr<dom::AudioNode> mAudioNode;
  // Pipe-track for allowing a track listener on a non-destination AudioNode.
  // Will be null when input is media stream or destination node.
  RefPtr<AudioNodeTrack> mPipeTrack;
  // Input port that connect mAudioNode to mPipeTrack.
  // Will be null when input is media stream or destination node.
  RefPtr<MediaInputPort> mInputPort;
  // An audio track that we are encoding. Will be null if the input stream
  // doesn't contain audio on start() or if the input is an AudioNode.
  RefPtr<dom::AudioStreamTrack> mAudioTrack;
  // A video track that we are encoding. Will be null if the input stream
  // doesn't contain video on start() or if the input is an AudioNode.
  RefPtr<dom::VideoStreamTrack> mVideoTrack;

  // A stream to keep the MediaTrackGraph alive while we're recording.
  RefPtr<SharedDummyTrack> mGraphTrack;

  // A buffer to cache muxed encoded data.
  RefPtr<dom::MutableBlobStorage> mMutableBlobStorage;
  // If set, is a promise for the latest GatherBlob() operation. Allows
  // GatherBlob() operations to be serialized in order to avoid races.
  RefPtr<BlobPromise> mBlobPromise;
  // The end time of the muxed data in the last gathered blob. If more than one
  // track is present, this is the end time of the track that ends the earliest
  // in the last blob. Encoder thread only.
  media::TimeUnit mLastBlobTime;
  // The end time of the muxed data in the current blob storage. If more than
  // one track is present, this is the end time of the track that ends the
  // earliest in the current blob storage. Encoder thread only.
  media::TimeUnit mLastExtractTime;
  // The end time of encoded audio data sent to the muxer. Positive infinity if
  // there is no audio encoder. Encoder thread only.
  media::TimeUnit mMuxedAudioEndTime;
  // The end time of encoded video data sent to the muxer. Positive infinity if
  // there is no video encoder. Encoder thread only.
  media::TimeUnit mMuxedVideoEndTime;

  TimeStamp mStartTime;
  bool mInitialized;
  bool mStarted;
  bool mCompleted;
  bool mError;
  // Set when shutdown starts.
  RefPtr<GenericNonExclusivePromise> mShutdownPromise;
  // Get duration from create encoder, for logging purpose
  double GetEncodeTimeStamp() {
    TimeDuration decodeTime;
    decodeTime = TimeStamp::Now() - mStartTime;
    return decodeTime.ToMilliseconds();
  }
};

}  // namespace mozilla

#endif