/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "AudioDeviceInfo.h" #include "AudioSink.h" #include "AudioSinkWrapper.h" #include "ImageContainer.h" #include "MockCubeb.h" #include "MockMediaDecoderOwner.h" #include "TimeUnits.h" #include "VideoFrameContainer.h" #include "VideoSink.h" #include "gtest/gtest.h" #include "mozilla/gtest/WaitFor.h" using namespace mozilla; using namespace mozilla::layers; using media::TimeUnit; TEST(TestVideoSink, FrameThrottling) { MockCubeb* cubeb = new MockCubeb(MockCubeb::RunningMode::Manual); CubebUtils::ForceSetCubebContext(cubeb->AsCubebContext()); MediaInfo info; info.EnableAudio(); // to control the advance of time through MockCubeb info.EnableVideo(); MediaQueue audioQueue; auto audioSinkCreator = [&]() { return UniquePtr{new AudioSink(AbstractThread::GetCurrent(), audioQueue, info.mAudio, /*resistFingerprinting*/ false)}; }; RefPtr wrapper = new AudioSinkWrapper( AbstractThread::GetCurrent(), audioQueue, std::move(audioSinkCreator), /*initialVolume*/ 1.0, /*playbackRate*/ 1.0, /*preservesPitch*/ true, /*sinkDevice*/ nullptr); auto owner = std::make_unique(); RefPtr container = new VideoFrameContainer( owner.get(), MakeAndAddRef(ImageUsageType::VideoFrameContainer, #ifdef MOZ_WIDGET_ANDROID // Work around bug 1922144 ImageContainer::SYNCHRONOUS #else ImageContainer::ASYNCHRONOUS #endif )); MediaQueue videoQueue; RefPtr frameStatistics = new FrameStatistics(); RefPtr videoSink = new VideoSink(AbstractThread::GetCurrent(), wrapper, videoQueue, container, *frameStatistics, /*aVQueueSentToCompositerSize*/ 9999); auto initPromise = TakeN(cubeb->StreamInitEvent(), 1); videoSink->Start(TimeUnit::Zero(), info); auto [stream] = WaitFor(initPromise).unwrap()[0]; uint32_t audioRate = stream->SampleRate(); // Enough audio data that it does not underrun, which would stop the clock. size_t audioFrameCount = 1000 * info.mAudio.mRate / audioRate; AlignedAudioBuffer samples(audioFrameCount * info.mAudio.mChannels); RefPtr audioData = new AudioData( /*aOffset*/ 0, /*aTime*/ TimeUnit(0, info.mAudio.mRate), std::move(samples), info.mAudio.mChannels, info.mAudio.mRate); audioQueue.Push(audioData); auto image = container->GetImageContainer()->CreatePlanarYCbCrImage(); static uint8_t pixel[] = {0x00}; PlanarYCbCrData imageData; imageData.mYChannel = imageData.mCbChannel = imageData.mCrChannel = pixel; imageData.mYStride = imageData.mCbCrStride = 1; imageData.mPictureRect = gfx::IntRect(0, 0, 1, 1); image->CopyData(imageData); TimeUnit nextFrameTime = TimeUnit(0, audioRate); auto PushVideoFrame = [&](const gfx::IntSize& aSize, const TimeUnit& aDuration) { static bool isKeyFrame = true; RefPtr frame = VideoData::CreateFromImage(aSize, /*aOffset*/ 0, /*aTime*/ nextFrameTime, aDuration, image, isKeyFrame, /*aTimecode*/ nextFrameTime); frame->mFrameID = container->NewFrameID(); videoQueue.Push(frame); nextFrameTime = frame->GetEndTime(); isKeyFrame = false; }; gfx::IntSize size1{1, 1}; PushVideoFrame(size1, TimeUnit(1, audioRate)); gfx::IntSize size2{1, 2}; PushVideoFrame(size2, TimeUnit(1, audioRate)); // UpdateRenderedVideoFrames() will keep scheduling additional events in // antipication of the audio clock advancing for the second frame, so wait // for only the initial size from the first frame. SpinEventLoopUntil("the intrinsic size receives an initial value"_ns, [&] { return container->CurrentIntrinsicSize().isSome(); }); EXPECT_EQ(container->CurrentIntrinsicSize().value(), size1); // Advance time to expire both frames. stream->ManualDataCallback(nextFrameTime.ToTicksAtRate(audioRate) + 1); // Run UpdateRenderedVideoFramesByTimer(), which is scheduled on TimeStamp's // clock, which we don't control. SpinEventLoopUntil( "the intrinsic size is updated to that of frame 2"_ns, [&] { return container->CurrentIntrinsicSize().value() == size2; }); // The next frame is overdue but has not yet expired. gfx::IntSize size3{1, 3}; PushVideoFrame(size3, TimeUnit(2, audioRate)); gfx::IntSize size4{1, 4}; PushVideoFrame(size4, TimeUnit(1, audioRate)); // Run UpdateRenderedVideoFrames() via OnVideoQueuePushed(). NS_ProcessPendingEvents(nullptr); EXPECT_EQ(container->CurrentIntrinsicSize().value(), size3); EXPECT_EQ(frameStatistics->GetDroppedSinkFrames(), 0u); // Advance time to expire the two frames in the queue and the next three. stream->ManualDataCallback(static_cast( nextFrameTime.ToTicksAtRate(audioRate) + 11 - stream->Position())); // This frame has a longer duration and is late. gfx::IntSize size5{1, 5}; PushVideoFrame(size5, TimeUnit(8, audioRate)); // The most recent frame was late, and so is not rendered yet because it may // be dropped. // // OnVideoQueuePushed() uses TryUpdateRenderedVideoFrames(), which no-ops if // an update is already scheduled. Wait for the update scheduled for // frame 4. SpinEventLoopUntil( "the intrinsic size is updated to that of frame 4"_ns, [&] { return container->CurrentIntrinsicSize().value() == size4; }); // This frame is also late. gfx::IntSize size6{1, 6}; PushVideoFrame(size6, TimeUnit(1, audioRate)); NS_ProcessPendingEvents(nullptr); // One frame was dropped, but the most recent frame was rendered because its // lateness was less than the duration of the dropped frame. EXPECT_EQ(frameStatistics->GetDroppedSinkFrames(), 1u); EXPECT_EQ(container->CurrentIntrinsicSize().value(), size6); gfx::IntSize size7{1, 7}; PushVideoFrame(size7, TimeUnit(1, audioRate)); NS_ProcessPendingEvents(nullptr); // The most recent frame was late, and so is not rendered yet because it may // be dropped. EXPECT_EQ(container->CurrentIntrinsicSize().value(), size6); // On playback pause, the most recent frame is rendered. videoSink->SetPlaying(false); EXPECT_EQ(container->CurrentIntrinsicSize().value(), size7); EXPECT_EQ(frameStatistics->GetDroppedSinkFrames(), 1u); videoSink->Stop(); videoSink->Shutdown(); }