diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
commit | 36d22d82aa202bb199967e9512281e9a53db42c9 (patch) | |
tree | 105e8c98ddea1c1e4784a60a5a6410fa416be2de /dom/media/MediaStreamTrack.h | |
parent | Initial commit. (diff) | |
download | firefox-esr-upstream.tar.xz firefox-esr-upstream.zip |
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/MediaStreamTrack.h')
-rw-r--r-- | dom/media/MediaStreamTrack.h | 638 |
1 files changed, 638 insertions, 0 deletions
diff --git a/dom/media/MediaStreamTrack.h b/dom/media/MediaStreamTrack.h new file mode 100644 index 0000000000..30ef7a3d93 --- /dev/null +++ b/dom/media/MediaStreamTrack.h @@ -0,0 +1,638 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MEDIASTREAMTRACK_H_ +#define MEDIASTREAMTRACK_H_ + +#include "MediaTrackConstraints.h" +#include "PrincipalChangeObserver.h" +#include "PrincipalHandle.h" +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/dom/MediaStreamTrackBinding.h" +#include "mozilla/dom/MediaTrackSettingsBinding.h" +#include "mozilla/media/MediaUtils.h" +#include "mozilla/WeakPtr.h" +#include "nsError.h" +#include "nsID.h" +#include "nsIPrincipal.h" +#include "PerformanceRecorder.h" + +namespace mozilla { + +class DOMMediaStream; +class MediaEnginePhotoCallback; +class MediaInputPort; +class MediaTrack; +class MediaTrackGraph; +class MediaTrackGraphImpl; +class MediaTrackListener; +class DirectMediaTrackListener; +class PeerConnectionImpl; +class PeerIdentity; +class ProcessedMediaTrack; +class RemoteSourceStreamInfo; +class SourceStreamInfo; +class MediaMgrError; + +namespace dom { + +class AudioStreamTrack; +class VideoStreamTrack; +enum class CallerType : uint32_t; + +/** + * Common interface through which a MediaStreamTrack can communicate with its + * producer on the main thread. + * + * Kept alive by a strong ref in all MediaStreamTracks (original and clones) + * sharing this source. + */ +class MediaStreamTrackSource : public nsISupports { + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_CLASS(MediaStreamTrackSource) + + public: + class Sink : public SupportsWeakPtr { + public: + /** + * Must be constant throughout the Sink's lifetime. + * + * Return true to keep the MediaStreamTrackSource where this sink is + * registered alive. + * Return false to allow the source to stop. + * + * Typically MediaStreamTrack::Sink returns true and other Sinks + * (like HTMLMediaElement::StreamCaptureTrackSource) return false. + */ + virtual bool KeepsSourceAlive() const = 0; + + /** + * Return true to ensure that the MediaStreamTrackSource where this Sink is + * registered is kept turned on and active. + * Return false to allow the source to pause, and any underlying devices to + * temporarily stop. + * + * When the underlying enabled state of the sink changes, + * call MediaStreamTrackSource::SinkEnabledStateChanged(). + * + * Typically MediaStreamTrack returns the track's enabled state and other + * Sinks (like HTMLMediaElement::StreamCaptureTrackSource) return false so + * control over device state remains with tracks and their enabled state. + */ + virtual bool Enabled() const = 0; + + /** + * Called when the principal of the MediaStreamTrackSource where this sink + * is registered has changed. + */ + virtual void PrincipalChanged() = 0; + + /** + * Called when the muted state of the MediaStreamTrackSource where this sink + * is registered has changed. + */ + virtual void MutedChanged(bool aNewState) = 0; + + /** + * Called when the MediaStreamTrackSource where this sink is registered has + * stopped producing data for good, i.e., it has ended. + */ + virtual void OverrideEnded() = 0; + + protected: + virtual ~Sink() = default; + }; + + MediaStreamTrackSource(nsIPrincipal* aPrincipal, const nsString& aLabel, + TrackingId aTrackingId) + : mPrincipal(aPrincipal), + mLabel(aLabel), + mTrackingId(std::move(aTrackingId)), + mStopped(false) {} + + /** + * Use to clean up any resources that have to be cleaned before the + * destructor is called. It is often too late in the destructor because + * of garbage collection having removed the members already. + */ + virtual void Destroy() {} + + /** + * Gets the source's MediaSourceEnum for usage by PeerConnections. + */ + virtual MediaSourceEnum GetMediaSource() const = 0; + + /** + * Get this TrackSource's principal. + */ + nsIPrincipal* GetPrincipal() const { return mPrincipal; } + + /** + * This is used in WebRTC. A peerIdentity constrained MediaStreamTrack cannot + * be sent across the network to anything other than a peer with the provided + * identity. If this is set, then GetPrincipal() should return an instance of + * NullPrincipal. + * + * A track's PeerIdentity is immutable and will not change during the track's + * lifetime. + */ + virtual const PeerIdentity* GetPeerIdentity() const { return nullptr; } + + /** + * MediaStreamTrack::GetLabel (see spec) calls through to here. + */ + void GetLabel(nsAString& aLabel) { aLabel.Assign(mLabel); } + + /** + * Whether this TrackSource provides video frames with an alpha channel. Only + * applies to video sources. Used by HTMLVideoElement. + */ + virtual bool HasAlpha() const { return false; } + + /** + * Forwards a photo request to backends that support it. Other backends return + * NS_ERROR_NOT_IMPLEMENTED to indicate that a MediaTrackGraph-based fallback + * should be used. + */ + virtual nsresult TakePhoto(MediaEnginePhotoCallback*) const { + return NS_ERROR_NOT_IMPLEMENTED; + } + + typedef MozPromise<bool /* aIgnored */, RefPtr<MediaMgrError>, true> + ApplyConstraintsPromise; + + /** + * We provide a fallback solution to ApplyConstraints() here. + * Sources that support ApplyConstraints() will have to override it. + */ + virtual RefPtr<ApplyConstraintsPromise> ApplyConstraints( + const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType); + + /** + * Same for GetSettings (no-op). + */ + virtual void GetSettings(dom::MediaTrackSettings& aResult){}; + + /** + * Called by the source interface when all registered sinks with + * KeepsSourceAlive() == true have unregistered. + */ + virtual void Stop() = 0; + + /** + * Called by the source interface when all registered sinks with + * KeepsSourceAlive() == true become disabled. + */ + virtual void Disable() = 0; + + /** + * Called by the source interface when at least one registered sink with + * KeepsSourceAlive() == true become enabled. + */ + virtual void Enable() = 0; + + /** + * Called when a Sink's Enabled() state changed. Will iterate through all + * sinks and notify the source of the aggregated enabled state. + * + * Note that a Sink with KeepsSourceAlive() == false counts as disabled. + */ + void SinkEnabledStateChanged() { + if (IsEnabled()) { + Enable(); + } else { + Disable(); + } + } + + /** + * Called by each MediaStreamTrack clone on initialization. + */ + void RegisterSink(Sink* aSink) { + MOZ_ASSERT(NS_IsMainThread()); + if (mStopped) { + return; + } + mSinks.AppendElement(aSink); + while (mSinks.RemoveElement(nullptr)) { + MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed"); + } + } + + /** + * Called by each MediaStreamTrack clone on Stop() if supported by the + * source (us) or destruction. + */ + void UnregisterSink(Sink* aSink) { + MOZ_ASSERT(NS_IsMainThread()); + while (mSinks.RemoveElement(nullptr)) { + MOZ_ASSERT_UNREACHABLE("Sink was not explicitly removed"); + } + if (mSinks.RemoveElement(aSink) && !IsActive()) { + MOZ_ASSERT(!aSink->KeepsSourceAlive() || !mStopped, + "When the last sink keeping the source alive is removed, " + "we should still be live"); + Stop(); + mStopped = true; + } + if (!mStopped) { + SinkEnabledStateChanged(); + } + } + + protected: + virtual ~MediaStreamTrackSource() = default; + + bool IsActive() { + for (const WeakPtr<Sink>& sink : mSinks) { + if (sink && sink->KeepsSourceAlive()) { + return true; + } + } + return false; + } + + bool IsEnabled() { + for (const WeakPtr<Sink>& sink : mSinks) { + if (sink && sink->KeepsSourceAlive() && sink->Enabled()) { + return true; + } + } + return false; + } + + /** + * Called by a sub class when the principal has changed. + * Notifies all sinks. + */ + void PrincipalChanged() { + MOZ_ASSERT(NS_IsMainThread()); + for (auto& sink : mSinks.Clone()) { + if (!sink) { + DebugOnly<bool> removed = mSinks.RemoveElement(sink); + MOZ_ASSERT(!removed, "Sink was not explicitly removed"); + continue; + } + sink->PrincipalChanged(); + } + } + + /** + * Called by a sub class when the source's muted state has changed. Note that + * the source is responsible for making the content black/silent during mute. + * Notifies all sinks. + */ + void MutedChanged(bool aNewState) { + MOZ_ASSERT(NS_IsMainThread()); + for (auto& sink : mSinks.Clone()) { + if (!sink) { + DebugOnly<bool> removed = mSinks.RemoveElement(sink); + MOZ_ASSERT(!removed, "Sink was not explicitly removed"); + continue; + } + sink->MutedChanged(aNewState); + } + } + + /** + * Called by a sub class when the source has stopped producing data for good, + * i.e., it has ended. Notifies all sinks. + */ + void OverrideEnded() { + MOZ_ASSERT(NS_IsMainThread()); + for (auto& sink : mSinks.Clone()) { + if (!sink) { + DebugOnly<bool> removed = mSinks.RemoveElement(sink); + MOZ_ASSERT(!removed, "Sink was not explicitly removed"); + continue; + } + sink->OverrideEnded(); + } + } + + // Principal identifying who may access the contents of this source. + RefPtr<nsIPrincipal> mPrincipal; + + // Currently registered sinks. + nsTArray<WeakPtr<Sink>> mSinks; + + public: + // The label of the track we are the source of per the MediaStreamTrack spec. + const nsString mLabel; + + // Set for all video sources; an id for tracking the source of the video + // frames for this track. + const TrackingId mTrackingId; + + protected: + // True if all MediaStreamTrack users have unregistered from this source and + // Stop() has been called. + bool mStopped; +}; + +/** + * Base class that consumers of a MediaStreamTrack can use to get notifications + * about state changes in the track. + */ +class MediaStreamTrackConsumer : public SupportsWeakPtr { + public: + /** + * Called when the track's readyState transitions to "ended". + * Unlike the "ended" event exposed to script this is called for any reason, + * including MediaStreamTrack::Stop(). + */ + virtual void NotifyEnded(MediaStreamTrack* aTrack){}; + + /** + * Called when the track's enabled state changes. + */ + virtual void NotifyEnabledChanged(MediaStreamTrack* aTrack, bool aEnabled){}; +}; + +// clang-format off +/** + * DOM wrapper for MediaTrackGraph-MediaTracks. + * + * To account for cloning, a MediaStreamTrack wraps two internal (and chained) + * MediaTracks: + * 1. mInputTrack + * - Controlled by the producer of the data in the track. The producer + * decides on lifetime of the MediaTrack and the track inside it. + * - It can be any type of MediaTrack. + * - Contains one track only. + * 2. mTrack + * - A ForwardedInputTrack representing this MediaStreamTrack. + * - Its data is piped from mInputTrack through mPort. + * - Contains one track only. + * - When this MediaStreamTrack is enabled/disabled this is reflected in + * the chunks in the track in mTrack. + * - When this MediaStreamTrack has ended, mTrack gets destroyed. + * Note that mInputTrack is unaffected, such that any clones of mTrack + * can live on. When all clones are ended, this is signaled to the + * producer via our MediaStreamTrackSource. It is then likely to destroy + * mInputTrack. + * + * A graphical representation of how tracks are connected when cloned follows: + * + * MediaStreamTrack A + * mInputTrack mTrack + * t1 ---------> t1 + * \ + * ----- + * MediaStreamTrack B \ (clone of A) + * mInputTrack \ mTrack + * * -> t1 + * + * (*) is a copy of A's mInputTrack + */ +// clang-format on +class MediaStreamTrack : public DOMEventTargetHelper, public SupportsWeakPtr { + // PeerConnection and friends need to know our owning DOMStream and track id. + friend class mozilla::PeerConnectionImpl; + friend class mozilla::SourceStreamInfo; + friend class mozilla::RemoteSourceStreamInfo; + + class MTGListener; + class TrackSink; + + public: + MediaStreamTrack( + nsPIDOMWindowInner* aWindow, mozilla::MediaTrack* aInputTrack, + MediaStreamTrackSource* aSource, + MediaStreamTrackState aReadyState = MediaStreamTrackState::Live, + bool aMuted = false, + const MediaTrackConstraints& aConstraints = MediaTrackConstraints()); + + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(MediaStreamTrack, + DOMEventTargetHelper) + + nsPIDOMWindowInner* GetParentObject() const { return mWindow; } + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + virtual AudioStreamTrack* AsAudioStreamTrack() { return nullptr; } + virtual VideoStreamTrack* AsVideoStreamTrack() { return nullptr; } + + virtual const AudioStreamTrack* AsAudioStreamTrack() const { return nullptr; } + virtual const VideoStreamTrack* AsVideoStreamTrack() const { return nullptr; } + + // WebIDL + virtual void GetKind(nsAString& aKind) = 0; + void GetId(nsAString& aID) const; + virtual void GetLabel(nsAString& aLabel, CallerType /* aCallerType */) { + GetSource().GetLabel(aLabel); + } + bool Enabled() const { return mEnabled; } + void SetEnabled(bool aEnabled); + bool Muted() { return mMuted; } + void Stop(); + void GetConstraints(dom::MediaTrackConstraints& aResult); + void GetSettings(dom::MediaTrackSettings& aResult, CallerType aCallerType); + + already_AddRefed<Promise> ApplyConstraints( + const dom::MediaTrackConstraints& aConstraints, CallerType aCallerType, + ErrorResult& aRv); + already_AddRefed<MediaStreamTrack> Clone(); + MediaStreamTrackState ReadyState() { return mReadyState; } + + IMPL_EVENT_HANDLER(mute) + IMPL_EVENT_HANDLER(unmute) + IMPL_EVENT_HANDLER(ended) + + /** + * Convenience (and legacy) method for when ready state is "ended". + */ + bool Ended() const { return mReadyState == MediaStreamTrackState::Ended; } + + /** + * Get this track's principal. + */ + nsIPrincipal* GetPrincipal() const { return mPrincipal; } + + /** + * Get this track's PeerIdentity. + */ + const PeerIdentity* GetPeerIdentity() const { + return GetSource().GetPeerIdentity(); + } + + ProcessedMediaTrack* GetTrack() const; + MediaTrackGraph* Graph() const; + MediaTrackGraphImpl* GraphImpl() const; + + MediaStreamTrackSource& GetSource() const { + MOZ_RELEASE_ASSERT(mSource, + "The track source is only removed on destruction"); + return *mSource; + } + + // Webrtc allows the remote side to name tracks whatever it wants, and we + // need to surface this to content. + void AssignId(const nsAString& aID) { mID = aID; } + + /** + * Add a PrincipalChangeObserver to this track. + * + * Returns true if it was successfully added. + * + * Ownership of the PrincipalChangeObserver remains with the caller, and it's + * the caller's responsibility to remove the observer before it dies. + */ + bool AddPrincipalChangeObserver( + PrincipalChangeObserver<MediaStreamTrack>* aObserver); + + /** + * Remove an added PrincipalChangeObserver from this track. + * + * Returns true if it was successfully removed. + */ + bool RemovePrincipalChangeObserver( + PrincipalChangeObserver<MediaStreamTrack>* aObserver); + + /** + * Add a MediaStreamTrackConsumer to this track. + * + * Adding the same consumer multiple times is prohibited. + */ + void AddConsumer(MediaStreamTrackConsumer* aConsumer); + + /** + * Remove an added MediaStreamTrackConsumer from this track. + */ + void RemoveConsumer(MediaStreamTrackConsumer* aConsumer); + + /** + * Adds a MediaTrackListener to the MediaTrackGraph representation of + * this track. + */ + virtual void AddListener(MediaTrackListener* aListener); + + /** + * Removes a MediaTrackListener from the MediaTrackGraph representation + * of this track. + */ + void RemoveListener(MediaTrackListener* aListener); + + /** + * Attempts to add a direct track listener to this track. + * Callers must listen to the NotifyInstalled event to know if installing + * the listener succeeded (tracks originating from SourceMediaTracks) or + * failed (e.g., WebAudio originated tracks). + */ + virtual void AddDirectListener(DirectMediaTrackListener* aListener); + void RemoveDirectListener(DirectMediaTrackListener* aListener); + + /** + * Sets up a MediaInputPort from the underlying track that this + * MediaStreamTrack represents, to aTrack, and returns it. + */ + already_AddRefed<MediaInputPort> ForwardTrackContentsTo( + ProcessedMediaTrack* aTrack); + + protected: + virtual ~MediaStreamTrack(); + + /** + * Forces the ready state to a particular value, for instance when we're + * cloning an already ended track. + */ + virtual void SetReadyState(MediaStreamTrackState aState); + + /** + * Notified by the MediaTrackGraph, through our owning MediaStream on the + * main thread. + * + * Note that this sets the track to ended and raises the "ended" event + * synchronously. + */ + void OverrideEnded(); + + /** + * Called by the MTGListener when this track's PrincipalHandle changes on + * the MediaTrackGraph thread. When the PrincipalHandle matches the pending + * principal we know that the principal change has propagated to consumers. + */ + void NotifyPrincipalHandleChanged(const PrincipalHandle& aNewPrincipalHandle); + + /** + * Called when this track's readyState transitions to "ended". + * Notifies all MediaStreamTrackConsumers that this track ended. + */ + void NotifyEnded(); + + /** + * Called when this track's enabled state has changed. + * Notifies all MediaStreamTrackConsumers. + */ + void NotifyEnabledChanged(); + + /** + * Called when mSource's principal has changed. + */ + void PrincipalChanged(); + + /** + * Called when mSource's muted state has changed. + */ + void MutedChanged(bool aNewState); + + /** + * Sets this track's muted state without raising any events. + * Only really set by cloning. See MutedChanged for runtime changes. + */ + void SetMuted(bool aMuted) { mMuted = aMuted; } + + virtual void Destroy(); + + /** + * Sets the principal and notifies PrincipalChangeObservers if it changes. + */ + void SetPrincipal(nsIPrincipal* aPrincipal); + + /** + * Creates a new MediaStreamTrack with the same kind, input track, input + * track ID and source as this MediaStreamTrack. + */ + virtual already_AddRefed<MediaStreamTrack> CloneInternal() = 0; + + nsTArray<PrincipalChangeObserver<MediaStreamTrack>*> + mPrincipalChangeObservers; + + nsTArray<WeakPtr<MediaStreamTrackConsumer>> mConsumers; + + // We need this to track our parent object. + nsCOMPtr<nsPIDOMWindowInner> mWindow; + + // The input MediaTrack assigned us by the data producer. + // Owned by the producer. + const RefPtr<mozilla::MediaTrack> mInputTrack; + // The MediaTrack representing this MediaStreamTrack in the MediaTrackGraph. + // Set on construction if we're live. Valid until we end. Owned by us. + RefPtr<ProcessedMediaTrack> mTrack; + // The MediaInputPort connecting mInputTrack to mTrack. Set on construction + // if mInputTrack is non-destroyed and we're live. Valid until we end. Owned + // by us. + RefPtr<MediaInputPort> mPort; + RefPtr<MediaStreamTrackSource> mSource; + const UniquePtr<TrackSink> mSink; + nsCOMPtr<nsIPrincipal> mPrincipal; + nsCOMPtr<nsIPrincipal> mPendingPrincipal; + RefPtr<MTGListener> mMTGListener; + // Keep tracking MediaTrackListener and DirectMediaTrackListener, + // so we can remove them in |Destory|. + nsTArray<RefPtr<MediaTrackListener>> mTrackListeners; + nsTArray<RefPtr<DirectMediaTrackListener>> mDirectTrackListeners; + nsString mID; + MediaStreamTrackState mReadyState; + bool mEnabled; + bool mMuted; + dom::MediaTrackConstraints mConstraints; +}; + +} // namespace dom +} // namespace mozilla + +#endif /* MEDIASTREAMTRACK_H_ */ |