Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 638 insertions, 0 deletions

diff --git a/dom/media/MediaStreamTrack.h b/dom/media/MediaStreamTrack.h
new file mode 100644
index 0000000000..30ef7a3d93
--- /dev/null
+++ b/dom/media/MediaStreamTrack.h
@@ -0,0 +1,638 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MEDIASTREAMTRACK_H_
+#define MEDIASTREAMTRACK_H_
+
+#include "MediaTrackConstraints.h"
+#include "PrincipalChangeObserver.h"
+#include "PrincipalHandle.h"
+#include "mozilla/DOMEventTargetHelper.h"
+#include "mozilla/dom/MediaStreamTrackBinding.h"
+#include "mozilla/dom/MediaTrackSettingsBinding.h"
+#include "mozilla/media/MediaUtils.h"
+#include "mozilla/WeakPtr.h"
+#include "nsError.h"
+#include "nsID.h"
+#include "nsIPrincipal.h"
+#include "PerformanceRecorder.h"
+
+namespace mozilla {
+
+class DOMMediaStream;
+class MediaEnginePhotoCallback;
+class MediaInputPort;
+class MediaTrack;
+class MediaTrackGraph;
+class MediaTrackGraphImpl;
+class MediaTrackListener;
+class DirectMediaTrackListener;
+class PeerConnectionImpl;
+class PeerIdentity;
+class ProcessedMediaTrack;
+class RemoteSourceStreamInfo;
+class SourceStreamInfo;
+class MediaMgrError;
+
+namespace dom {
+
+class AudioStreamTrack;
+class VideoStreamTrack;
+enum class CallerType : uint32_t;
+
+/**
+ * Common interface through which a MediaStreamTrack can communicate with its
+ * producer on the main thread.
+ *
+ * Kept alive by a strong ref in all MediaStreamTracks (original and clones)
+ * sharing this source.
+ */
+class MediaStreamTrackSource : public nsISupports {
+  NS_DECL_CYCLE_COLLECTING_ISUPPORTS
+  NS_DECL_CYCLE_COLLECTION_CLASS(MediaStreamTrackSource)
+
+ public:
+  class Sink : public SupportsWeakPtr {
+   public:
+    /**
+     * Must be constant throughout the Sink's lifetime.
+     *
+     * Return true to keep the MediaStreamTrackSource where this sink is
+     * registered alive.
+     * Return false to allow the source to stop.
+     *
+     * Typically MediaStreamTrack::Sink returns true and other Sinks
+     * (like HTMLMediaElement::StreamCaptureTrackSource) return false.
+     */
+    virtual bool KeepsSourceAlive() const = 0;
+
+    /**
+     * Return true to ensure that the MediaStreamTrackSource where this Sink is
+     * registered is kept turned on and active.
+     * Return false to allow the source to pause, and any underlying devices to
+     * temporarily stop.
+     *
+     * When the underlying enabled state of the sink changes,
+     * call MediaStreamTrackSource::SinkEnabledStateChanged().
+     *
+     * Typically MediaStreamTrack returns the track's enabled state and other
+     * Sinks (like HTMLMediaElement::StreamCaptureTrackSource) return false so
+     * control over device state remains with tracks and their enabled state.
+     */
+    virtual bool Enabled() const = 0;
+
+    /**
+     * Called when the principal of the MediaStreamTrackSource where this sink
+     * is registered has changed.
+     */
+    virtual void PrincipalChanged() = 0;
+
+    /**
+     * Called when the muted state of the MediaStreamTrackSource where this sink
+     * is registered has changed.
+     */
+    virtual void MutedChanged(bool aNewState) = 0;
+
+    /**
+     * Called when the MediaStreamTrackSource where this sink is registered has
+     * stopped producing data for good, i.e., it has ended.
+     */
+    virtual void OverrideEnded() = 0;
+
+   protected:
+    virtual ~Sink() = default;
+  };
+
+  MediaStreamTrackSource(nsIPrincipal* aPrincipal, const nsString& aLabel,
+                         TrackingId aTrackingId)
+      : mPrincipal(aPrincipal),
+        mLabel(aLabel),
+        mTrackingId(std::move(aTrackingId)),
+        mStopped(false) {}
+
+  /**
+   * Use to clean up any resources that have to be cleaned before the
+   * destructor is called. It is often too late in the destructor because
+   * of garbage collection having removed the members already.
+   */
+  virtual void Destroy() {}
+
+  /**
+   * Gets the source's MediaSourceEnum for usage by PeerConnections.
+   */
+  virtual MediaSourceEnum GetMediaSource() const = 0;
+
+  /**
+   * Get this TrackSource's principal.
+   */
+  nsIPrincipal* GetPrincipal() const { return mPrincipal; }
+
+  /**
+   * This is used in WebRTC. A peerIdentity constrained MediaStreamTrack cannot
+   * be sent across the network to anything other than a peer with the provided
+   * identity. If this is set, then GetPrincipal() should return an instance of
+   * NullPrincipal.
+   *
+   * A track's PeerIdentity is immutable and will not change during the track's
+   * lifetime.
+   */
+  virtual const PeerIdentity* GetPeerIdentity() const { return nullptr; }
+
+  /**
+   * MediaStreamTrack::GetLabel (see spec) calls through to here.
+   */
+  void GetLabel(nsAString& aLabel) { aLabel.Assign(mLabel); }
+
+  /**
+   * Whether this TrackSource provides video frames with an alpha channel. Only
+   * applies to video sources. Used by HTMLVideoElement.
+   */
+  virtual bool HasAlpha() const { return false; }
+
+  /**
+   * Forwards a photo request to backends that support it. Other backends return
+   * NS_ERROR_NOT_IMPLEMENTED to indicate that a MediaTrackGraph-based fallback
+   * should be used.
+   */
+  virtual nsresult TakePhoto(MediaEnginePhotoCallback*) const {
+    return NS_ERROR_NOT_IMPLEMENTED;
+  }
+
+  typedef MozPromise<bool /* aIgnored */, RefPtr<MediaMgrError>, true>
+      ApplyConstraintsPromise;
+
+  /**
+   * We provide a fallback solution to ApplyConstraints() here.
+   * Sources that support ApplyConstraints() will have to override it.
+   */
+  virtual RefPtr<ApplyConstraintsPromise> ApplyConstraints(
+      const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType);
+
+  /**
+   * Same for GetSettings (no-op).
+   */
+  virtual void GetSettings(dom::MediaTrackSettings& aResult){};
+
+  /**
+   * Called by the source interface when all registered sinks with
+   * KeepsSourceAlive() == true have unregistered.
+   */
+  virtual void Stop() = 0;
+
+  /**
+   * Called by the source interface when all registered sinks with
+   * KeepsSourceAlive() == true become disabled.
+   */
+  virtual void Disable() = 0;
+
+  /**
+   * Called by the source interface when at least one registered sink with
+   * KeepsSourceAlive() == true become enabled.
+   */
+  virtual void Enable() = 0;
+
+  /**
+   * Called when a Sink's Enabled() state changed. Will iterate through all
+   * sinks and notify the source of the aggregated enabled state.
+   *
+   * Note that a Sink with KeepsSourceAlive() == false counts as disabled.
+   */
+  void SinkEnabledStateChanged() {
+    if (IsEnabled()) {
+      Enable();
+    } else {
+      Disable();
+    }
+  }
+
+  /**
+   * Called by each MediaStreamTrack clone on initialization.
+   */
+  void RegisterSink(Sink* aSink) {
+    MOZ_ASSERT(NS_IsMainThread());
+    if (mStopped) {
+      return;
+    }
+    mSinks.AppendElement(aSink);
+    while (mSinks.RemoveElement(nullptr)) {
+      MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
+    }
+  }
+
+  /**
+   * Called by each MediaStreamTrack clone on Stop() if supported by the
+   * source (us) or destruction.
+   */
+  void UnregisterSink(Sink* aSink) {
+    MOZ_ASSERT(NS_IsMainThread());
+    while (mSinks.RemoveElement(nullptr)) {
+      MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed");
+    }
+    if (mSinks.RemoveElement(aSink) && !IsActive()) {
+      MOZ_ASSERT(!aSink->KeepsSourceAlive() || !mStopped,
+                 "When the last sink keeping the source alive is removed, "
+                 "we should still be live");
+      Stop();
+      mStopped = true;
+    }
+    if (!mStopped) {
+      SinkEnabledStateChanged();
+    }
+  }
+
+ protected:
+  virtual ~MediaStreamTrackSource() = default;
+
+  bool IsActive() {
+    for (const WeakPtr<Sink>& sink : mSinks) {
+      if (sink && sink->KeepsSourceAlive()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  bool IsEnabled() {
+    for (const WeakPtr<Sink>& sink : mSinks) {
+      if (sink && sink->KeepsSourceAlive() && sink->Enabled()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Called by a sub class when the principal has changed.
+   * Notifies all sinks.
+   */
+  void PrincipalChanged() {
+    MOZ_ASSERT(NS_IsMainThread());
+    for (auto& sink : mSinks.Clone()) {
+      if (!sink) {
+        DebugOnly<bool> removed = mSinks.RemoveElement(sink);
+        MOZ_ASSERT(!removed, "Sink was not explicitly removed");
+        continue;
+      }
+      sink->PrincipalChanged();
+    }
+  }
+
+  /**
+   * Called by a sub class when the source's muted state has changed. Note that
+   * the source is responsible for making the content black/silent during mute.
+   * Notifies all sinks.
+   */
+  void MutedChanged(bool aNewState) {
+    MOZ_ASSERT(NS_IsMainThread());
+    for (auto& sink : mSinks.Clone()) {
+      if (!sink) {
+        DebugOnly<bool> removed = mSinks.RemoveElement(sink);
+        MOZ_ASSERT(!removed, "Sink was not explicitly removed");
+        continue;
+      }
+      sink->MutedChanged(aNewState);
+    }
+  }
+
+  /**
+   * Called by a sub class when the source has stopped producing data for good,
+   * i.e., it has ended. Notifies all sinks.
+   */
+  void OverrideEnded() {
+    MOZ_ASSERT(NS_IsMainThread());
+    for (auto& sink : mSinks.Clone()) {
+      if (!sink) {
+        DebugOnly<bool> removed = mSinks.RemoveElement(sink);
+        MOZ_ASSERT(!removed, "Sink was not explicitly removed");
+        continue;
+      }
+      sink->OverrideEnded();
+    }
+  }
+
+  // Principal identifying who may access the contents of this source.
+  RefPtr<nsIPrincipal> mPrincipal;
+
+  // Currently registered sinks.
+  nsTArray<WeakPtr<Sink>> mSinks;
+
+ public:
+  // The label of the track we are the source of per the MediaStreamTrack spec.
+  const nsString mLabel;
+
+  // Set for all video sources; an id for tracking the source of the video
+  // frames for this track.
+  const TrackingId mTrackingId;
+
+ protected:
+  // True if all MediaStreamTrack users have unregistered from this source and
+  // Stop() has been called.
+  bool mStopped;
+};
+
+/**
+ * Base class that consumers of a MediaStreamTrack can use to get notifications
+ * about state changes in the track.
+ */
+class MediaStreamTrackConsumer : public SupportsWeakPtr {
+ public:
+  /**
+   * Called when the track's readyState transitions to "ended".
+   * Unlike the "ended" event exposed to script this is called for any reason,
+   * including MediaStreamTrack::Stop().
+   */
+  virtual void NotifyEnded(MediaStreamTrack* aTrack){};
+
+  /**
+   * Called when the track's enabled state changes.
+   */
+  virtual void NotifyEnabledChanged(MediaStreamTrack* aTrack, bool aEnabled){};
+};
+
+// clang-format off
+/**
+ * DOM wrapper for MediaTrackGraph-MediaTracks.
+ *
+ * To account for cloning, a MediaStreamTrack wraps two internal (and chained)
+ * MediaTracks:
+ *   1. mInputTrack
+ *      - Controlled by the producer of the data in the track. The producer
+ *        decides on lifetime of the MediaTrack and the track inside it.
+ *      - It can be any type of MediaTrack.
+ *      - Contains one track only.
+ *   2. mTrack
+ *      - A ForwardedInputTrack representing this MediaStreamTrack.
+ *      - Its data is piped from mInputTrack through mPort.
+ *      - Contains one track only.
+ *      - When this MediaStreamTrack is enabled/disabled this is reflected in
+ *        the chunks in the track in mTrack.
+ *      - When this MediaStreamTrack has ended, mTrack gets destroyed.
+ *        Note that mInputTrack is unaffected, such that any clones of mTrack
+ *        can live on. When all clones are ended, this is signaled to the
+ *        producer via our MediaStreamTrackSource. It is then likely to destroy
+ *        mInputTrack.
+ *
+ * A graphical representation of how tracks are connected when cloned follows:
+ *
+ * MediaStreamTrack A
+ *       mInputTrack     mTrack
+ *            t1 ---------> t1
+ *               \
+ *                -----
+ * MediaStreamTrack B  \  (clone of A)
+ *       mInputTrack   \ mTrack
+ *            *          -> t1
+ *
+ *   (*) is a copy of A's mInputTrack
+ */
+// clang-format on
+class MediaStreamTrack : public DOMEventTargetHelper, public SupportsWeakPtr {
+  // PeerConnection and friends need to know our owning DOMStream and track id.
+  friend class mozilla::PeerConnectionImpl;
+  friend class mozilla::SourceStreamInfo;
+  friend class mozilla::RemoteSourceStreamInfo;
+
+  class MTGListener;
+  class TrackSink;
+
+ public:
+  MediaStreamTrack(
+      nsPIDOMWindowInner* aWindow, mozilla::MediaTrack* aInputTrack,
+      MediaStreamTrackSource* aSource,
+      MediaStreamTrackState aReadyState = MediaStreamTrackState::Live,
+      bool aMuted = false,
+      const MediaTrackConstraints& aConstraints = MediaTrackConstraints());
+
+  NS_DECL_ISUPPORTS_INHERITED
+  NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrack,
+                                           DOMEventTargetHelper)
+
+  nsPIDOMWindowInner* GetParentObject() const { return mWindow; }
+  JSObject* WrapObject(JSContext* aCx,
+                       JS::Handle<JSObject*> aGivenProto) override;
+
+  virtual AudioStreamTrack* AsAudioStreamTrack() { return nullptr; }
+  virtual VideoStreamTrack* AsVideoStreamTrack() { return nullptr; }
+
+  virtual const AudioStreamTrack* AsAudioStreamTrack() const { return nullptr; }
+  virtual const VideoStreamTrack* AsVideoStreamTrack() const { return nullptr; }
+
+  // WebIDL
+  virtual void GetKind(nsAString& aKind) = 0;
+  void GetId(nsAString& aID) const;
+  virtual void GetLabel(nsAString& aLabel, CallerType /* aCallerType */) {
+    GetSource().GetLabel(aLabel);
+  }
+  bool Enabled() const { return mEnabled; }
+  void SetEnabled(bool aEnabled);
+  bool Muted() { return mMuted; }
+  void Stop();
+  void GetConstraints(dom::MediaTrackConstraints& aResult);
+  void GetSettings(dom::MediaTrackSettings& aResult, CallerType aCallerType);
+
+  already_AddRefed<Promise> ApplyConstraints(
+      const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType,
+      ErrorResult& aRv);
+  already_AddRefed<MediaStreamTrack> Clone();
+  MediaStreamTrackState ReadyState() { return mReadyState; }
+
+  IMPL_EVENT_HANDLER(mute)
+  IMPL_EVENT_HANDLER(unmute)
+  IMPL_EVENT_HANDLER(ended)
+
+  /**
+   * Convenience (and legacy) method for when ready state is "ended".
+   */
+  bool Ended() const { return mReadyState == MediaStreamTrackState::Ended; }
+
+  /**
+   * Get this track's principal.
+   */
+  nsIPrincipal* GetPrincipal() const { return mPrincipal; }
+
+  /**
+   * Get this track's PeerIdentity.
+   */
+  const PeerIdentity* GetPeerIdentity() const {
+    return GetSource().GetPeerIdentity();
+  }
+
+  ProcessedMediaTrack* GetTrack() const;
+  MediaTrackGraph* Graph() const;
+  MediaTrackGraphImpl* GraphImpl() const;
+
+  MediaStreamTrackSource& GetSource() const {
+    MOZ_RELEASE_ASSERT(mSource,
+                       "The track source is only removed on destruction");
+    return *mSource;
+  }
+
+  // Webrtc allows the remote side to name tracks whatever it wants, and we
+  // need to surface this to content.
+  void AssignId(const nsAString& aID) { mID = aID; }
+
+  /**
+   * Add a PrincipalChangeObserver to this track.
+   *
+   * Returns true if it was successfully added.
+   *
+   * Ownership of the PrincipalChangeObserver remains with the caller, and it's
+   * the caller's responsibility to remove the observer before it dies.
+   */
+  bool AddPrincipalChangeObserver(
+      PrincipalChangeObserver<MediaStreamTrack>* aObserver);
+
+  /**
+   * Remove an added PrincipalChangeObserver from this track.
+   *
+   * Returns true if it was successfully removed.
+   */
+  bool RemovePrincipalChangeObserver(
+      PrincipalChangeObserver<MediaStreamTrack>* aObserver);
+
+  /**
+   * Add a MediaStreamTrackConsumer to this track.
+   *
+   * Adding the same consumer multiple times is prohibited.
+   */
+  void AddConsumer(MediaStreamTrackConsumer* aConsumer);
+
+  /**
+   * Remove an added MediaStreamTrackConsumer from this track.
+   */
+  void RemoveConsumer(MediaStreamTrackConsumer* aConsumer);
+
+  /**
+   * Adds a MediaTrackListener to the MediaTrackGraph representation of
+   * this track.
+   */
+  virtual void AddListener(MediaTrackListener* aListener);
+
+  /**
+   * Removes a MediaTrackListener from the MediaTrackGraph representation
+   * of this track.
+   */
+  void RemoveListener(MediaTrackListener* aListener);
+
+  /**
+   * Attempts to add a direct track listener to this track.
+   * Callers must listen to the NotifyInstalled event to know if installing
+   * the listener succeeded (tracks originating from SourceMediaTracks) or
+   * failed (e.g., WebAudio originated tracks).
+   */
+  virtual void AddDirectListener(DirectMediaTrackListener* aListener);
+  void RemoveDirectListener(DirectMediaTrackListener* aListener);
+
+  /**
+   * Sets up a MediaInputPort from the underlying track that this
+   * MediaStreamTrack represents, to aTrack, and returns it.
+   */
+  already_AddRefed<MediaInputPort> ForwardTrackContentsTo(
+      ProcessedMediaTrack* aTrack);
+
+ protected:
+  virtual ~MediaStreamTrack();
+
+  /**
+   * Forces the ready state to a particular value, for instance when we're
+   * cloning an already ended track.
+   */
+  virtual void SetReadyState(MediaStreamTrackState aState);
+
+  /**
+   * Notified by the MediaTrackGraph, through our owning MediaStream on the
+   * main thread.
+   *
+   * Note that this sets the track to ended and raises the "ended" event
+   * synchronously.
+   */
+  void OverrideEnded();
+
+  /**
+   * Called by the MTGListener when this track's PrincipalHandle changes on
+   * the MediaTrackGraph thread. When the PrincipalHandle matches the pending
+   * principal we know that the principal change has propagated to consumers.
+   */
+  void NotifyPrincipalHandleChanged(const PrincipalHandle& aNewPrincipalHandle);
+
+  /**
+   * Called when this track's readyState transitions to "ended".
+   * Notifies all MediaStreamTrackConsumers that this track ended.
+   */
+  void NotifyEnded();
+
+  /**
+   * Called when this track's enabled state has changed.
+   * Notifies all MediaStreamTrackConsumers.
+   */
+  void NotifyEnabledChanged();
+
+  /**
+   * Called when mSource's principal has changed.
+   */
+  void PrincipalChanged();
+
+  /**
+   * Called when mSource's muted state has changed.
+   */
+  void MutedChanged(bool aNewState);
+
+  /**
+   * Sets this track's muted state without raising any events.
+   * Only really set by cloning. See MutedChanged for runtime changes.
+   */
+  void SetMuted(bool aMuted) { mMuted = aMuted; }
+
+  virtual void Destroy();
+
+  /**
+   * Sets the principal and notifies PrincipalChangeObservers if it changes.
+   */
+  void SetPrincipal(nsIPrincipal* aPrincipal);
+
+  /**
+   * Creates a new MediaStreamTrack with the same kind, input track, input
+   * track ID and source as this MediaStreamTrack.
+   */
+  virtual already_AddRefed<MediaStreamTrack> CloneInternal() = 0;
+
+  nsTArray<PrincipalChangeObserver<MediaStreamTrack>*>
+      mPrincipalChangeObservers;
+
+  nsTArray<WeakPtr<MediaStreamTrackConsumer>> mConsumers;
+
+  // We need this to track our parent object.
+  nsCOMPtr<nsPIDOMWindowInner> mWindow;
+
+  // The input MediaTrack assigned us by the data producer.
+  // Owned by the producer.
+  const RefPtr<mozilla::MediaTrack> mInputTrack;
+  // The MediaTrack representing this MediaStreamTrack in the MediaTrackGraph.
+  // Set on construction if we're live. Valid until we end. Owned by us.
+  RefPtr<ProcessedMediaTrack> mTrack;
+  // The MediaInputPort connecting mInputTrack to mTrack. Set on construction
+  // if mInputTrack is non-destroyed and we're live. Valid until we end. Owned
+  // by us.
+  RefPtr<MediaInputPort> mPort;
+  RefPtr<MediaStreamTrackSource> mSource;
+  const UniquePtr<TrackSink> mSink;
+  nsCOMPtr<nsIPrincipal> mPrincipal;
+  nsCOMPtr<nsIPrincipal> mPendingPrincipal;
+  RefPtr<MTGListener> mMTGListener;
+  // Keep tracking MediaTrackListener and DirectMediaTrackListener,
+  // so we can remove them in |Destory|.
+  nsTArray<RefPtr<MediaTrackListener>> mTrackListeners;
+  nsTArray<RefPtr<DirectMediaTrackListener>> mDirectTrackListeners;
+  nsString mID;
+  MediaStreamTrackState mReadyState;
+  bool mEnabled;
+  bool mMuted;
+  dom::MediaTrackConstraints mConstraints;
+};
+
+}  // namespace dom
+}  // namespace mozilla
+
+#endif /* MEDIASTREAMTRACK_H_ */