summaryrefslogtreecommitdiffstats
path: root/dom/media/VideoOutput.h
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/VideoOutput.h')
-rw-r--r--dom/media/VideoOutput.h307
1 files changed, 307 insertions, 0 deletions
diff --git a/dom/media/VideoOutput.h b/dom/media/VideoOutput.h
new file mode 100644
index 0000000000..4fa1ff407b
--- /dev/null
+++ b/dom/media/VideoOutput.h
@@ -0,0 +1,307 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef VideoOutput_h
+#define VideoOutput_h
+
+#include "MediaTrackListener.h"
+#include "VideoFrameContainer.h"
+
+namespace mozilla {
+
+static bool SetImageToBlackPixel(layers::PlanarYCbCrImage* aImage) {
+ uint8_t blackPixel[] = {0x10, 0x80, 0x80};
+
+ layers::PlanarYCbCrData data;
+ data.mYChannel = blackPixel;
+ data.mCbChannel = blackPixel + 1;
+ data.mCrChannel = blackPixel + 2;
+ data.mYStride = data.mCbCrStride = 1;
+ data.mPictureRect = gfx::IntRect(0, 0, 1, 1);
+ data.mYUVColorSpace = gfx::YUVColorSpace::BT601;
+ // This could be made FULL once bug 1568745 is complete. A black pixel being
+ // 0x00, 0x80, 0x80
+ data.mColorRange = gfx::ColorRange::LIMITED;
+
+ return NS_SUCCEEDED(aImage->CopyData(data));
+}
+
+class VideoOutput : public DirectMediaTrackListener {
+ protected:
+ typedef layers::Image Image;
+ typedef layers::ImageContainer ImageContainer;
+ typedef layers::ImageContainer::FrameID FrameID;
+ typedef layers::ImageContainer::ProducerID ProducerID;
+
+ virtual ~VideoOutput() = default;
+
+ void DropPastFrames() {
+ TimeStamp now = TimeStamp::Now();
+ size_t nrChunksInPast = 0;
+ for (const auto& idChunkPair : mFrames) {
+ const VideoChunk& chunk = idChunkPair.second;
+ if (chunk.mTimeStamp > now) {
+ break;
+ }
+ ++nrChunksInPast;
+ }
+ if (nrChunksInPast > 1) {
+ // We need to keep one frame that starts in the past, because it only ends
+ // when the next frame starts (which also needs to be in the past for it
+ // to drop).
+ mFrames.RemoveElementsAt(0, nrChunksInPast - 1);
+ }
+ }
+
+ void SendFramesEnsureLocked() {
+ mMutex.AssertCurrentThreadOwns();
+ SendFrames();
+ }
+
+ void SendFrames() {
+ DropPastFrames();
+
+ if (mFrames.IsEmpty()) {
+ return;
+ }
+
+ if (!mEnabled && mDisabledBlackImageSent) {
+ return;
+ }
+
+ // Collect any new frames produced in this iteration.
+ AutoTArray<ImageContainer::NonOwningImage, 16> images;
+ PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
+
+ for (const auto& idChunkPair : mFrames) {
+ ImageContainer::FrameID frameId = idChunkPair.first;
+ const VideoChunk& chunk = idChunkPair.second;
+ const VideoFrame& frame = chunk.mFrame;
+ Image* image = frame.GetImage();
+ if (frame.GetForceBlack() || !mEnabled) {
+ if (!mBlackImage) {
+ RefPtr<Image> blackImage = mVideoFrameContainer->GetImageContainer()
+ ->CreatePlanarYCbCrImage();
+ if (blackImage) {
+ // Sets the image to a single black pixel, which will be scaled to
+ // fill the rendered size.
+ if (SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage())) {
+ mBlackImage = blackImage;
+ }
+ }
+ }
+ if (mBlackImage) {
+ image = mBlackImage;
+ }
+ }
+ if (!image) {
+ // We ignore null images.
+ continue;
+ }
+ images.AppendElement(ImageContainer::NonOwningImage(
+ image, chunk.mTimeStamp, frameId, mProducerID));
+
+ lastPrincipalHandle = chunk.GetPrincipalHandle();
+
+ if (!mEnabled && mBlackImage) {
+ MOZ_ASSERT(images.Length() == 1);
+ mDisabledBlackImageSent = true;
+ break;
+ }
+ }
+
+ if (images.IsEmpty()) {
+ // This could happen if the only images in mFrames are null. We leave the
+ // container at the current frame in this case.
+ mVideoFrameContainer->ClearFutureFrames();
+ return;
+ }
+
+ bool principalHandleChanged =
+ lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+ lastPrincipalHandle != mVideoFrameContainer->GetLastPrincipalHandle();
+
+ if (principalHandleChanged) {
+ mVideoFrameContainer->UpdatePrincipalHandleForFrameID(
+ lastPrincipalHandle, images.LastElement().mFrameID);
+ }
+
+ mVideoFrameContainer->SetCurrentFrames(
+ mFrames[0].second.mFrame.GetIntrinsicSize(), images);
+ mMainThread->Dispatch(NewRunnableMethod("VideoFrameContainer::Invalidate",
+ mVideoFrameContainer,
+ &VideoFrameContainer::Invalidate));
+ }
+
+ FrameID NewFrameID() {
+ mMutex.AssertCurrentThreadOwns();
+ return ++mFrameID;
+ }
+
+ public:
+ VideoOutput(VideoFrameContainer* aContainer, AbstractThread* aMainThread)
+ : mMutex("VideoOutput::mMutex"),
+ mVideoFrameContainer(aContainer),
+ mMainThread(aMainThread) {}
+ void NotifyRealtimeTrackData(MediaTrackGraph* aGraph, TrackTime aTrackOffset,
+ const MediaSegment& aMedia) override {
+ MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
+ const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
+ MutexAutoLock lock(mMutex);
+ for (VideoSegment::ConstChunkIterator i(video); !i.IsEnded(); i.Next()) {
+ if (!mLastFrameTime.IsNull() && i->mTimeStamp < mLastFrameTime) {
+ // Time can go backwards if the source is a captured MediaDecoder and
+ // it seeks, as the previously buffered frames would stretch into the
+ // future. If this happens, we clear the buffered frames and start over.
+ mFrames.ClearAndRetainStorage();
+ }
+ mFrames.AppendElement(std::make_pair(NewFrameID(), *i));
+ mLastFrameTime = i->mTimeStamp;
+ }
+
+ SendFramesEnsureLocked();
+ }
+ void NotifyRemoved(MediaTrackGraph* aGraph) override {
+ // Doesn't need locking by mMutex, since the direct listener is removed from
+ // the track before we get notified.
+ if (mFrames.Length() <= 1) {
+ // The compositor has already received the last frame.
+ mFrames.ClearAndRetainStorage();
+ mVideoFrameContainer->ClearFutureFrames();
+ return;
+ }
+
+ // The compositor has multiple frames. ClearFutureFrames() would only retain
+ // the first as that's normally the current one. We however stop doing
+ // SetCurrentFrames() once we've received the last frame in a track, so
+ // there might be old frames lingering. We'll find the current one and
+ // re-send that.
+ DropPastFrames();
+ mFrames.RemoveLastElements(mFrames.Length() - 1);
+ SendFrames();
+ mFrames.ClearAndRetainStorage();
+ }
+ void NotifyEnded(MediaTrackGraph* aGraph) override {
+ // Doesn't need locking by mMutex, since for the track to end, it must have
+ // been ended by the source, meaning that the source won't append more data.
+ if (mFrames.IsEmpty()) {
+ return;
+ }
+
+ // Re-send only the last one to the compositor.
+ mFrames.RemoveElementsAt(0, mFrames.Length() - 1);
+ SendFrames();
+ mFrames.ClearAndRetainStorage();
+ }
+ void NotifyEnabledStateChanged(MediaTrackGraph* aGraph,
+ bool aEnabled) override {
+ MutexAutoLock lock(mMutex);
+ mEnabled = aEnabled;
+ DropPastFrames();
+ if (mEnabled) {
+ mDisabledBlackImageSent = false;
+ }
+ if (!mEnabled || mFrames.Length() > 1) {
+ // Re-send frames when disabling, as new frames may not arrive. When
+ // enabling we keep them black until new frames arrive, or re-send if we
+ // already have frames in the future. If we're disabling and there are no
+ // frames available yet, we invent one. Unfortunately with a hardcoded
+ // size.
+ //
+ // Since mEnabled will affect whether
+ // frames are real, or black, we assign new FrameIDs whenever we re-send
+ // frames after an mEnabled change.
+ for (auto& idChunkPair : mFrames) {
+ idChunkPair.first = NewFrameID();
+ }
+ if (mFrames.IsEmpty()) {
+ VideoSegment v;
+ v.AppendFrame(nullptr, gfx::IntSize(640, 480), PRINCIPAL_HANDLE_NONE,
+ true, TimeStamp::Now());
+ mFrames.AppendElement(std::make_pair(NewFrameID(), *v.GetLastChunk()));
+ }
+ SendFramesEnsureLocked();
+ }
+ }
+
+ Mutex mMutex MOZ_UNANNOTATED;
+ TimeStamp mLastFrameTime;
+ // Once the frame is forced to black, we initialize mBlackImage for use in any
+ // following forced-black frames.
+ RefPtr<Image> mBlackImage;
+ // True once mBlackImage has been sent due to mEnabled being false.
+ bool mDisabledBlackImageSent = false;
+ bool mEnabled = true;
+ // This array is accessed from both the direct video thread, and the graph
+ // thread. Protected by mMutex.
+ nsTArray<std::pair<ImageContainer::FrameID, VideoChunk>> mFrames;
+ // Accessed from both the direct video thread, and the graph thread. Protected
+ // by mMutex.
+ FrameID mFrameID = 0;
+ const RefPtr<VideoFrameContainer> mVideoFrameContainer;
+ const RefPtr<AbstractThread> mMainThread;
+ const ProducerID mProducerID = ImageContainer::AllocateProducerID();
+};
+
+/**
+ * This listener observes the first video frame to arrive with a non-empty size,
+ * and renders it to its VideoFrameContainer.
+ */
+class FirstFrameVideoOutput : public VideoOutput {
+ public:
+ FirstFrameVideoOutput(VideoFrameContainer* aContainer,
+ AbstractThread* aMainThread)
+ : VideoOutput(aContainer, aMainThread) {
+ MOZ_ASSERT(NS_IsMainThread());
+ }
+
+ // NB that this overrides VideoOutput::NotifyRealtimeTrackData, so we can
+ // filter out all frames but the first one with a real size. This allows us to
+ // later re-use the logic in VideoOutput for rendering that frame.
+ void NotifyRealtimeTrackData(MediaTrackGraph* aGraph, TrackTime aTrackOffset,
+ const MediaSegment& aMedia) override {
+ MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
+
+ if (mInitialSizeFound) {
+ return;
+ }
+
+ const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
+ for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
+ if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0, 0)) {
+ mInitialSizeFound = true;
+
+ mMainThread->Dispatch(NS_NewRunnableFunction(
+ "FirstFrameVideoOutput::FirstFrameRenderedSetter",
+ [self = RefPtr<FirstFrameVideoOutput>(this)] {
+ self->mFirstFrameRendered = true;
+ }));
+
+ // Pick the first frame and run it through the rendering code.
+ VideoSegment segment;
+ segment.AppendFrame(do_AddRef(c->mFrame.GetImage()),
+ c->mFrame.GetIntrinsicSize(),
+ c->mFrame.GetPrincipalHandle(),
+ c->mFrame.GetForceBlack(), c->mTimeStamp);
+ VideoOutput::NotifyRealtimeTrackData(aGraph, aTrackOffset, segment);
+ return;
+ }
+ }
+ }
+
+ // Main thread only.
+ Watchable<bool> mFirstFrameRendered = {
+ false, "FirstFrameVideoOutput::mFirstFrameRendered"};
+
+ private:
+ // Whether a frame with a concrete size has been received. May only be
+ // accessed on the MTG's appending thread. (this is a direct listener so we
+ // get called by whoever is producing this track's data)
+ bool mInitialSizeFound = false;
+};
+
+} // namespace mozilla
+
+#endif // VideoOutput_h