/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "Muxer.h" #include "ContainerWriter.h" namespace mozilla { LazyLogModule gMuxerLog("Muxer"); #define LOG(type, ...) MOZ_LOG(gMuxerLog, type, (__VA_ARGS__)) Muxer::Muxer(UniquePtr aWriter, MediaQueue& aEncodedAudioQueue, MediaQueue& aEncodedVideoQueue) : mEncodedAudioQueue(aEncodedAudioQueue), mEncodedVideoQueue(aEncodedVideoQueue), mWriter(std::move(aWriter)) {} void Muxer::Disconnect() { mAudioPushListener.DisconnectIfExists(); mAudioFinishListener.DisconnectIfExists(); mVideoPushListener.DisconnectIfExists(); mVideoFinishListener.DisconnectIfExists(); } bool Muxer::IsFinished() { return mWriter->IsWritingComplete(); } nsresult Muxer::SetMetadata( const nsTArray>& aMetadata) { MOZ_DIAGNOSTIC_ASSERT(!mMetadataSet); MOZ_DIAGNOSTIC_ASSERT(!mHasAudio); MOZ_DIAGNOSTIC_ASSERT(!mHasVideo); nsresult rv = mWriter->SetMetadata(aMetadata); if (NS_FAILED(rv)) { LOG(LogLevel::Error, "%p Setting metadata failed, tracks=%zu", this, aMetadata.Length()); return rv; } for (const auto& track : aMetadata) { switch (track->GetKind()) { case TrackMetadataBase::METADATA_OPUS: case TrackMetadataBase::METADATA_VORBIS: case TrackMetadataBase::METADATA_AAC: case TrackMetadataBase::METADATA_AMR: case TrackMetadataBase::METADATA_EVRC: MOZ_ASSERT(!mHasAudio, "Only one audio track supported"); mHasAudio = true; break; case TrackMetadataBase::METADATA_VP8: MOZ_ASSERT(!mHasVideo, "Only one video track supported"); mHasVideo = true; break; default: MOZ_CRASH("Unknown codec metadata"); }; } mMetadataSet = true; MOZ_ASSERT(mHasAudio || mHasVideo); LOG(LogLevel::Info, "%p Metadata set; audio=%d, video=%d", this, mHasAudio, mHasVideo); return NS_OK; } nsresult Muxer::GetData(nsTArray>* aOutputBuffers) { MOZ_ASSERT(mHasAudio || mHasVideo); nsresult rv; if (!mMetadataEncoded) { rv = mWriter->GetContainerData(aOutputBuffers, ContainerWriter::GET_HEADER); if (NS_FAILED(rv)) { LOG(LogLevel::Error, "%p Failed getting metadata from writer", this); return rv; } mMetadataEncoded = true; } if (mEncodedAudioQueue.GetSize() == 0 && !mEncodedAudioQueue.IsFinished() && mEncodedVideoQueue.GetSize() == 0 && !mEncodedVideoQueue.IsFinished()) { // Nothing to mux. return NS_OK; } rv = Mux(); if (NS_FAILED(rv)) { LOG(LogLevel::Error, "%p Failed muxing data into writer", this); return rv; } MOZ_ASSERT_IF( mEncodedAudioQueue.IsFinished() && mEncodedVideoQueue.IsFinished(), mEncodedAudioQueue.AtEndOfStream()); MOZ_ASSERT_IF( mEncodedAudioQueue.IsFinished() && mEncodedVideoQueue.IsFinished(), mEncodedVideoQueue.AtEndOfStream()); uint32_t flags = mEncodedAudioQueue.AtEndOfStream() && mEncodedVideoQueue.AtEndOfStream() ? ContainerWriter::FLUSH_NEEDED : 0; if (mEncodedAudioQueue.AtEndOfStream() && mEncodedVideoQueue.AtEndOfStream()) { LOG(LogLevel::Info, "%p All data written", this); } return mWriter->GetContainerData(aOutputBuffers, flags); } nsresult Muxer::Mux() { MOZ_ASSERT(mMetadataSet); MOZ_ASSERT(mHasAudio || mHasVideo); nsTArray> frames; // The times at which we expect our next video and audio frames. These are // based on the time + duration (GetEndTime()) of the last seen frames. // Assumes that the encoders write the correct duration for frames.; media::TimeUnit expectedNextVideoTime; media::TimeUnit expectedNextAudioTime; // Interleave frames until we're out of audio or video while (mEncodedVideoQueue.GetSize() > 0 && mEncodedAudioQueue.GetSize() > 0) { RefPtr videoFrame = mEncodedVideoQueue.PeekFront(); RefPtr audioFrame = mEncodedAudioQueue.PeekFront(); // For any expected time our frames should occur at or after that time. MOZ_ASSERT(videoFrame->mTime >= expectedNextVideoTime); MOZ_ASSERT(audioFrame->mTime >= expectedNextAudioTime); if (videoFrame->mTime <= audioFrame->mTime) { expectedNextVideoTime = videoFrame->GetEndTime(); RefPtr frame = mEncodedVideoQueue.PopFront(); frames.AppendElement(std::move(frame)); } else { expectedNextAudioTime = audioFrame->GetEndTime(); RefPtr frame = mEncodedAudioQueue.PopFront(); frames.AppendElement(std::move(frame)); } } // If we're out of audio we still may be able to add more video... if (mEncodedAudioQueue.GetSize() == 0) { while (mEncodedVideoQueue.GetSize() > 0) { if (!mEncodedAudioQueue.AtEndOfStream() && mEncodedVideoQueue.PeekFront()->mTime > expectedNextAudioTime) { // Audio encoding is not complete and since the video frame comes // after our next audio frame we cannot safely add it. break; } frames.AppendElement(mEncodedVideoQueue.PopFront()); } } // If we're out of video we still may be able to add more audio... if (mEncodedVideoQueue.GetSize() == 0) { while (mEncodedAudioQueue.GetSize() > 0) { if (!mEncodedVideoQueue.AtEndOfStream() && mEncodedAudioQueue.PeekFront()->mTime > expectedNextVideoTime) { // Video encoding is not complete and since the audio frame comes // after our next video frame we cannot safely add it. break; } frames.AppendElement(mEncodedAudioQueue.PopFront()); } } LOG(LogLevel::Debug, "%p Muxed data, remaining-audio=%zu, remaining-video=%zu", this, mEncodedAudioQueue.GetSize(), mEncodedVideoQueue.GetSize()); // If encoding is complete for both encoders we should signal end of stream, // otherwise we keep going. uint32_t flags = mEncodedVideoQueue.AtEndOfStream() && mEncodedAudioQueue.AtEndOfStream() ? ContainerWriter::END_OF_STREAM : 0; nsresult rv = mWriter->WriteEncodedTrack(frames, flags); if (NS_FAILED(rv)) { LOG(LogLevel::Error, "Error! Failed to write muxed data to the container"); } return rv; } } // namespace mozilla #undef LOG