diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-15 03:35:49 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-15 03:35:49 +0000 |
commit | d8bbc7858622b6d9c278469aab701ca0b609cddf (patch) | |
tree | eff41dc61d9f714852212739e6b3738b82a2af87 /dom/media/platforms/ffmpeg | |
parent | Releasing progress-linux version 125.0.3-1~progress7.99u1. (diff) | |
download | firefox-d8bbc7858622b6d9c278469aab701ca0b609cddf.tar.xz firefox-d8bbc7858622b6d9c278469aab701ca0b609cddf.zip |
Merging upstream version 126.0.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/platforms/ffmpeg')
24 files changed, 1337 insertions, 606 deletions
diff --git a/dom/media/platforms/ffmpeg/FFmpegAudioEncoder.cpp b/dom/media/platforms/ffmpeg/FFmpegAudioEncoder.cpp new file mode 100644 index 0000000000..28db667732 --- /dev/null +++ b/dom/media/platforms/ffmpeg/FFmpegAudioEncoder.cpp @@ -0,0 +1,458 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FFmpegAudioEncoder.h" + +#include "FFmpegRuntimeLinker.h" +#include "FFmpegLog.h" +#include "FFmpegUtils.h" +#include "MediaData.h" + +#include "AudioSegment.h" + +namespace mozilla { + +FFmpegAudioEncoder<LIBAV_VER>::FFmpegAudioEncoder( + const FFmpegLibWrapper* aLib, AVCodecID aCodecID, + const RefPtr<TaskQueue>& aTaskQueue, const EncoderConfig& aConfig) + : FFmpegDataEncoder(aLib, aCodecID, aTaskQueue, aConfig) {} + +nsCString FFmpegAudioEncoder<LIBAV_VER>::GetDescriptionName() const { +#ifdef USING_MOZFFVPX + return "ffvpx audio encoder"_ns; +#else + const char* lib = +# if defined(MOZ_FFMPEG) + FFmpegRuntimeLinker::LinkStatusLibraryName(); +# else + "no library: ffmpeg disabled during build"; +# endif + return nsPrintfCString("ffmpeg audio encoder (%s)", lib); +#endif +} + +void FFmpegAudioEncoder<LIBAV_VER>::ResamplerDestroy::operator()( + SpeexResamplerState* aResampler) { + speex_resampler_destroy(aResampler); +} + +nsresult FFmpegAudioEncoder<LIBAV_VER>::InitSpecific() { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + FFMPEG_LOG("FFmpegAudioEncoder::InitInternal"); + + // Initialize the common members of the encoder instance + AVCodec* codec = FFmpegDataEncoder<LIBAV_VER>::InitCommon(); + if (!codec) { + FFMPEG_LOG("FFmpegDataEncoder::InitCommon failed"); + return NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR; + } + + // Find a compatible input rate for the codec, update the encoder config, and + // note the rate at which this instance was configured. + mInputSampleRate = AssertedCast<int>(mConfig.mSampleRate); + if (codec->supported_samplerates) { + // Ensure the sample-rate list is sorted, iterate and either find that the + // sample rate is supported, or pick the same rate just above the audio + // input sample-rate (as to not lose information). If the audio is higher + // than the highest supported sample-rate, down-sample to the highest + // sample-rate supported by the codec. This is the case when encoding high + // samplerate audio to opus. + AutoTArray<int, 16> supportedSampleRates; + IterateZeroTerminated(codec->supported_samplerates, + [&supportedSampleRates](int aRate) mutable { + supportedSampleRates.AppendElement(aRate); + }); + supportedSampleRates.Sort(); + + for (const auto& rate : supportedSampleRates) { + if (mInputSampleRate == rate) { + mConfig.mSampleRate = rate; + break; + } + if (mInputSampleRate < rate) { + // This rate is the smallest supported rate above the content's rate. + mConfig.mSampleRate = rate; + break; + } + if (mInputSampleRate > rate) { + mConfig.mSampleRate = rate; + } + } + } + + if (mConfig.mSampleRate != AssertedCast<uint32_t>(mInputSampleRate)) { + // Need to resample to targetRate + int err; + SpeexResamplerState* resampler = speex_resampler_init( + mConfig.mNumberOfChannels, mInputSampleRate, mConfig.mSampleRate, + SPEEX_RESAMPLER_QUALITY_DEFAULT, &err); + if (!err) { + mResampler.reset(resampler); + } else { + FFMPEG_LOG( + "Error creating resampler in FFmpegAudioEncoder %dHz -> %dHz (%dch)", + mInputSampleRate, mConfig.mSampleRate, mConfig.mNumberOfChannels); + } + } + + // And now the audio-specific part + mCodecContext->sample_rate = AssertedCast<int>(mConfig.mSampleRate); + mCodecContext->channels = AssertedCast<int>(mConfig.mNumberOfChannels); + +#if LIBAVCODEC_VERSION_MAJOR >= 60 + // Gecko's ordering intentionnally matches ffmepg's ordering + mLib->av_channel_layout_default(&mCodecContext->ch_layout, + AssertedCast<int>(mCodecContext->channels)); +#endif + + switch (mConfig.mCodec) { + case CodecType::Opus: + // When using libopus, ffmpeg supports interleaved float and s16 input. + mCodecContext->sample_fmt = AV_SAMPLE_FMT_FLT; + break; + case CodecType::Vorbis: + // When using libvorbis, ffmpeg only supports planar f32 input. + mCodecContext->sample_fmt = AV_SAMPLE_FMT_FLTP; + break; + default: + MOZ_ASSERT_UNREACHABLE("Not supported"); + } + + if (mConfig.mCodec == CodecType::Opus) { + // Default is VBR + if (mConfig.mBitrateMode == BitrateMode::Constant) { + mLib->av_opt_set(mCodecContext->priv_data, "vbr", "off", 0); + } + if (mConfig.mCodecSpecific.isSome()) { + MOZ_ASSERT(mConfig.mCodecSpecific->is<OpusSpecific>()); + const OpusSpecific& specific = mConfig.mCodecSpecific->as<OpusSpecific>(); + // This attribute maps directly to complexity + mCodecContext->compression_level = specific.mComplexity; + FFMPEG_LOG("Opus complexity set to %d", specific.mComplexity); + float frameDurationMs = + AssertedCast<float>(specific.mFrameDuration) / 1000.f; + if (mLib->av_opt_set_double(mCodecContext->priv_data, "frame_duration", + frameDurationMs, 0)) { + FFMPEG_LOG("Error setting the frame duration on Opus encoder"); + return NS_ERROR_FAILURE; + } + FFMPEG_LOG("Opus frame duration set to %0.2f", frameDurationMs); + if (specific.mPacketLossPerc) { + if (mLib->av_opt_set_int( + mCodecContext->priv_data, "packet_loss", + AssertedCast<int64_t>(specific.mPacketLossPerc), 0)) { + FFMPEG_LOG("Error setting the packet loss percentage to %" PRIu64 + " on Opus encoder", + specific.mPacketLossPerc); + return NS_ERROR_FAILURE; + } + FFMPEG_LOGV("Packet loss set to %d%% in Opus encoder", + AssertedCast<int>(specific.mPacketLossPerc)); + } + if (specific.mUseInBandFEC) { + if (mLib->av_opt_set(mCodecContext->priv_data, "fec", "on", 0)) { + FFMPEG_LOG("Error %s FEC on Opus encoder", + specific.mUseInBandFEC ? "enabling" : "disabling"); + return NS_ERROR_FAILURE; + } + FFMPEG_LOGV("In-band FEC enabled for Opus encoder."); + } + if (specific.mUseDTX) { + if (mLib->av_opt_set(mCodecContext->priv_data, "dtx", "on", 0)) { + FFMPEG_LOG("Error %s DTX on Opus encoder", + specific.mUseDTX ? "enabling" : "disabling"); + return NS_ERROR_FAILURE; + } + // DTX packets are a TOC byte, and possibly one byte of length, packets + // 3 bytes and larger are to be returned. + mDtxThreshold = 3; + } + // TODO: format + // https://bugzilla.mozilla.org/show_bug.cgi?id=1876066 + } + } + // Override the time base: always the sample-rate the encoder is running at + mCodecContext->time_base = + AVRational{.num = 1, .den = mCodecContext->sample_rate}; + + MediaResult rv = FinishInitCommon(codec); + if (NS_FAILED(rv)) { + FFMPEG_LOG("FFmpeg encode initialization failure."); + return rv.Code(); + } + + return NS_OK; +} + +// avcodec_send_frame and avcodec_receive_packet were introduced in version 58. +#if LIBAVCODEC_VERSION_MAJOR >= 58 + +Result<MediaDataEncoder::EncodedData, nsresult> +FFmpegAudioEncoder<LIBAV_VER>::EncodeOnePacket(Span<float> aSamples, + media::TimeUnit aPts) { + // Allocate AVFrame. + if (!PrepareFrame()) { + FFMPEG_LOG("failed to allocate frame"); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + uint32_t frameCount = aSamples.Length() / mConfig.mNumberOfChannels; + + // This method assumes that the audio has been packetized appropriately -- + // packets smaller than the packet size are allowed when draining. + MOZ_ASSERT(AssertedCast<int>(frameCount) <= mCodecContext->frame_size); + + mFrame->channels = AssertedCast<int>(mConfig.mNumberOfChannels); + +# if LIBAVCODEC_VERSION_MAJOR >= 60 + int rv = mLib->av_channel_layout_copy(&mFrame->ch_layout, + &mCodecContext->ch_layout); + if (rv < 0) { + FFMPEG_LOG("channel layout copy error: %s", + MakeErrorString(mLib, rv).get()); + return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR); + } +# endif + + mFrame->sample_rate = AssertedCast<int>(mConfig.mSampleRate); + // Not a mistake, nb_samples is per channel in ffmpeg + mFrame->nb_samples = AssertedCast<int>(frameCount); + // Audio is converted below if needed + mFrame->format = mCodecContext->sample_fmt; + // Set presentation timestamp and duration of the AVFrame. +# if LIBAVCODEC_VERSION_MAJOR >= 59 + mFrame->time_base = + AVRational{.num = 1, .den = static_cast<int>(mConfig.mSampleRate)}; +# endif + mFrame->pts = aPts.ToTicksAtRate(mConfig.mSampleRate); + mFrame->pkt_duration = frameCount; +# if LIBAVCODEC_VERSION_MAJOR >= 60 + mFrame->duration = frameCount; +# else + // Save duration in the time_base unit. + mDurationMap.Insert(mFrame->pts, mFrame->pkt_duration); +# endif + + if (int ret = mLib->av_frame_get_buffer(mFrame, 16); ret < 0) { + FFMPEG_LOG("failed to allocate frame data: %s", + MakeErrorString(mLib, ret).get()); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + // Make sure AVFrame is writable. + if (int ret = mLib->av_frame_make_writable(mFrame); ret < 0) { + FFMPEG_LOG("failed to make frame writable: %s", + MakeErrorString(mLib, ret).get()); + return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR); + } + + // The input is always in f32 interleaved for now + if (mCodecContext->sample_fmt == AV_SAMPLE_FMT_FLT) { + PodCopy(reinterpret_cast<float*>(mFrame->data[0]), aSamples.data(), + aSamples.Length()); + } else { + MOZ_ASSERT(mCodecContext->sample_fmt == AV_SAMPLE_FMT_FLTP); + for (uint32_t i = 0; i < mConfig.mNumberOfChannels; i++) { + DeinterleaveAndConvertBuffer(aSamples.data(), mFrame->nb_samples, + mFrame->channels, mFrame->data); + } + } + + // Now send the AVFrame to ffmpeg for encoding, same code for audio and video. + return FFmpegDataEncoder<LIBAV_VER>::EncodeWithModernAPIs(); +} + +Result<MediaDataEncoder::EncodedData, nsresult> FFmpegAudioEncoder< + LIBAV_VER>::EncodeInputWithModernAPIs(RefPtr<const MediaData> aSample) { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + MOZ_ASSERT(mCodecContext); + MOZ_ASSERT(aSample); + + RefPtr<const AudioData> sample(aSample->As<AudioData>()); + + FFMPEG_LOG("Encoding %" PRIu32 " frames of audio at pts: %s", + sample->Frames(), sample->mTime.ToString().get()); + + if ((!mResampler && sample->mRate != mConfig.mSampleRate) || + (mResampler && + sample->mRate != AssertedCast<uint32_t>(mInputSampleRate)) || + sample->mChannels != mConfig.mNumberOfChannels) { + FFMPEG_LOG( + "Rate or sample-rate at the inputof the encoder different from what " + "has been configured initially, erroring out"); + return Result<MediaDataEncoder::EncodedData, nsresult>( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + } + + // ffmpeg expects exactly sized input audio packets most of the time. + // Packetization is performed if needed, and audio packets of the correct size + // are fed to ffmpeg, with timestamps extrapolated the timestamp found on + // the input MediaData. + + if (!mPacketizer) { + media::TimeUnit basePts = media::TimeUnit::Zero(mConfig.mSampleRate); + basePts += sample->mTime; + mPacketizer.emplace(mCodecContext->frame_size, sample->mChannels, + basePts.ToTicksAtRate(mConfig.mSampleRate), + mConfig.mSampleRate); + } + + if (!mFirstPacketPts.IsValid()) { + mFirstPacketPts = sample->mTime; + } + + Span<float> audio = sample->Data(); + + if (mResampler) { + // Ensure that all input frames are consumed each time by oversizing the + // output buffer. + int bufferLengthGuess = std::ceil(2. * static_cast<float>(audio.size()) * + mConfig.mSampleRate / mInputSampleRate); + mTempBuffer.SetLength(bufferLengthGuess); + uint32_t inputFrames = audio.size() / mConfig.mNumberOfChannels; + uint32_t inputFramesProcessed = inputFrames; + uint32_t outputFrames = bufferLengthGuess / mConfig.mNumberOfChannels; + DebugOnly<int> rv = speex_resampler_process_interleaved_float( + mResampler.get(), audio.data(), &inputFramesProcessed, + mTempBuffer.Elements(), &outputFrames); + audio = Span<float>(mTempBuffer.Elements(), + outputFrames * mConfig.mNumberOfChannels); + MOZ_ASSERT(inputFrames == inputFramesProcessed, + "increate the buffer to consume all input each time"); + MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS); + } + + EncodedData output; + MediaResult rv = NS_OK; + + mPacketizer->Input(audio.data(), audio.Length() / mConfig.mNumberOfChannels); + + // Dequeue and encode each packet + while (mPacketizer->PacketsAvailable() && rv.Code() == NS_OK) { + mTempBuffer.SetLength(mCodecContext->frame_size * + mConfig.mNumberOfChannels); + media::TimeUnit pts = mPacketizer->Output(mTempBuffer.Elements()); + auto audio = Span(mTempBuffer.Elements(), mTempBuffer.Length()); + FFMPEG_LOG("Encoding %" PRIu32 " frames, pts: %s", + mPacketizer->PacketSize(), pts.ToString().get()); + auto encodeResult = EncodeOnePacket(audio, pts); + if (encodeResult.isOk()) { + output.AppendElements(std::move(encodeResult.unwrap())); + } else { + return encodeResult; + } + pts += media::TimeUnit(mPacketizer->PacketSize(), mConfig.mSampleRate); + } + return Result<MediaDataEncoder::EncodedData, nsresult>(std::move(output)); +} + +Result<MediaDataEncoder::EncodedData, nsresult> +FFmpegAudioEncoder<LIBAV_VER>::DrainWithModernAPIs() { + // If there's no packetizer, or it's empty, we can proceed immediately. + if (!mPacketizer || mPacketizer->FramesAvailable() == 0) { + return FFmpegDataEncoder<LIBAV_VER>::DrainWithModernAPIs(); + } + EncodedData output; + MediaResult rv = NS_OK; + // Dequeue and encode each packet + mTempBuffer.SetLength(mCodecContext->frame_size * + mPacketizer->ChannelCount()); + uint32_t written; + media::TimeUnit pts = mPacketizer->Drain(mTempBuffer.Elements(), written); + auto audio = + Span(mTempBuffer.Elements(), written * mPacketizer->ChannelCount()); + auto encodeResult = EncodeOnePacket(audio, pts); + if (encodeResult.isOk()) { + auto array = encodeResult.unwrap(); + output.AppendElements(std::move(array)); + } else { + return encodeResult; + } + // Now, drain the encoder + auto drainResult = FFmpegDataEncoder<LIBAV_VER>::DrainWithModernAPIs(); + if (drainResult.isOk()) { + auto array = drainResult.unwrap(); + output.AppendElements(std::move(array)); + } else { + return drainResult; + } + return Result<MediaDataEncoder::EncodedData, nsresult>(std::move(output)); +} +#endif // if LIBAVCODEC_VERSION_MAJOR >= 58 + +RefPtr<MediaRawData> FFmpegAudioEncoder<LIBAV_VER>::ToMediaRawData( + AVPacket* aPacket) { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + MOZ_ASSERT(aPacket); + + if (aPacket->size < mDtxThreshold) { + FFMPEG_LOG( + "DTX enabled and packet is %d bytes (threshold %d), not returning.", + aPacket->size, mDtxThreshold); + return nullptr; + } + + RefPtr<MediaRawData> data = ToMediaRawDataCommon(aPacket); + + data->mTime = media::TimeUnit(aPacket->pts, mConfig.mSampleRate); + data->mTimecode = data->mTime; + data->mDuration = + media::TimeUnit(mCodecContext->frame_size, mConfig.mSampleRate); + + // Handle encoder delay + // Tracked in https://github.com/w3c/webcodecs/issues/626 because not quite + // specced yet. + if (mFirstPacketPts > data->mTime) { + data->mOriginalPresentationWindow = + Some(media::TimeInterval{data->mTime, data->GetEndTime()}); + // Duration is likely to be ajusted when the above spec issue is fixed. For + // now, leave it as-is + // data->mDuration -= (mFirstPacketPts - data->mTime); + // if (data->mDuration.IsNegative()) { + // data->mDuration = media::TimeUnit::Zero(); + // } + data->mTime = mFirstPacketPts; + } + + if (mPacketsDelivered++ == 0) { + // Attach extradata, and the config (including any channel / samplerate + // modification to fit the encoder requirements), if needed. + if (auto r = GetExtraData(aPacket); r.isOk()) { + data->mExtraData = r.unwrap(); + } + data->mConfig = MakeUnique<EncoderConfig>(mConfig); + } + + if (data->mExtraData) { + FFMPEG_LOG( + "FFmpegAudioEncoder out: [%s,%s] (%zu bytes, extradata %zu bytes)", + data->mTime.ToString().get(), data->mDuration.ToString().get(), + data->Size(), data->mExtraData->Length()); + } else { + FFMPEG_LOG("FFmpegAudioEncoder out: [%s,%s] (%zu bytes)", + data->mTime.ToString().get(), data->mDuration.ToString().get(), + data->Size()); + } + + return data; +} + +Result<already_AddRefed<MediaByteBuffer>, nsresult> +FFmpegAudioEncoder<LIBAV_VER>::GetExtraData(AVPacket* /* aPacket */) { + if (!mCodecContext->extradata_size) { + return Err(NS_ERROR_NOT_AVAILABLE); + } + // Create extra data -- they are on the context. + auto extraData = MakeRefPtr<MediaByteBuffer>(); + extraData->SetLength(mCodecContext->extradata_size); + MOZ_ASSERT(extraData); + PodCopy(extraData->Elements(), mCodecContext->extradata, + mCodecContext->extradata_size); + return extraData.forget(); +} + +} // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegAudioEncoder.h b/dom/media/platforms/ffmpeg/FFmpegAudioEncoder.h new file mode 100644 index 0000000000..51b0bfa44e --- /dev/null +++ b/dom/media/platforms/ffmpeg/FFmpegAudioEncoder.h @@ -0,0 +1,70 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGAUDIOENCODER_H_ +#define DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGAUDIOENCODER_H_ + +#include "FFmpegDataEncoder.h" +#include "FFmpegLibWrapper.h" +#include "PlatformEncoderModule.h" +#include "TimedPacketizer.h" + +// This must be the last header included +#include "FFmpegLibs.h" +#include "speex/speex_resampler.h" + +namespace mozilla { + +template <int V> +class FFmpegAudioEncoder : public MediaDataEncoder {}; + +template <> +class FFmpegAudioEncoder<LIBAV_VER> : public FFmpegDataEncoder<LIBAV_VER> { + public: + FFmpegAudioEncoder(const FFmpegLibWrapper* aLib, AVCodecID aCodecID, + const RefPtr<TaskQueue>& aTaskQueue, + const EncoderConfig& aConfig); + + nsCString GetDescriptionName() const override; + + protected: + // Methods only called on mTaskQueue. + virtual nsresult InitSpecific() override; +#if LIBAVCODEC_VERSION_MAJOR >= 58 + Result<EncodedData, nsresult> EncodeOnePacket(Span<float> aSamples, + media::TimeUnit aPts); + Result<EncodedData, nsresult> EncodeInputWithModernAPIs( + RefPtr<const MediaData> aSample) override; + Result<MediaDataEncoder::EncodedData, nsresult> DrainWithModernAPIs() + override; +#endif + virtual RefPtr<MediaRawData> ToMediaRawData(AVPacket* aPacket) override; + Result<already_AddRefed<MediaByteBuffer>, nsresult> GetExtraData( + AVPacket* aPacket) override; + // Most audio codecs (except PCM) require a very specific frame size. + Maybe<TimedPacketizer<float, float>> mPacketizer; + // A temporary buffer kept around for shuffling audio frames, resampling, + // packetization, etc. + nsTArray<float> mTempBuffer; + // The pts of the first packet this encoder has seen, to be able to properly + // mark encoder delay as such. + media::TimeUnit mFirstPacketPts{media::TimeUnit::Invalid()}; + struct ResamplerDestroy { + void operator()(SpeexResamplerState* aResampler); + }; + // Rate at which this instance has been configured, which might be different + // from the rate the underlying encoder is running at. + int mInputSampleRate = 0; + UniquePtr<SpeexResamplerState, ResamplerDestroy> mResampler; + uint64_t mPacketsDelivered = 0; + // Threshold under which a packet isn't returned to the encoder user, + // because it is known to be silent and DTX is enabled. + int mDtxThreshold = 0; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGAUDIOENCODER_H_ diff --git a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp index 1acfc26a4c..30422987cf 100644 --- a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp @@ -17,30 +17,14 @@ #include "mozilla/TaskQueue.h" #include "prsystem.h" #include "VideoUtils.h" +#include "FFmpegUtils.h" + #include "FFmpegLibs.h" namespace mozilla { StaticMutex FFmpegDataDecoder<LIBAV_VER>::sMutex; -static bool IsVideoCodec(AVCodecID aCodecID) { - switch (aCodecID) { - case AV_CODEC_ID_H264: -#if LIBAVCODEC_VERSION_MAJOR >= 54 - case AV_CODEC_ID_VP8: -#endif -#if LIBAVCODEC_VERSION_MAJOR >= 55 - case AV_CODEC_ID_VP9: -#endif -#if LIBAVCODEC_VERSION_MAJOR >= 59 - case AV_CODEC_ID_AV1: -#endif - return true; - default: - return false; - } -} - FFmpegDataDecoder<LIBAV_VER>::FFmpegDataDecoder(FFmpegLibWrapper* aLib, AVCodecID aCodecID) : mLib(aLib), diff --git a/dom/media/platforms/ffmpeg/FFmpegDataEncoder.cpp b/dom/media/platforms/ffmpeg/FFmpegDataEncoder.cpp new file mode 100644 index 0000000000..6b97a48156 --- /dev/null +++ b/dom/media/platforms/ffmpeg/FFmpegDataEncoder.cpp @@ -0,0 +1,495 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FFmpegDataEncoder.h" +#include "PlatformEncoderModule.h" + +#include <utility> + +#include "FFmpegLog.h" +#include "libavutil/error.h" +#include "mozilla/StaticMutex.h" + +#include "FFmpegUtils.h" + +namespace mozilla { + +// TODO: Remove this function and simply use `avcodec_find_encoder` once +// libopenh264 is supported. +static AVCodec* FindEncoderWithPreference(const FFmpegLibWrapper* aLib, + AVCodecID aCodecId) { + MOZ_ASSERT(aLib); + + AVCodec* codec = nullptr; + + // Prioritize libx264 for now since it's the only h264 codec we tested. + if (aCodecId == AV_CODEC_ID_H264) { + codec = aLib->avcodec_find_encoder_by_name("libx264"); + if (codec) { + FFMPEGV_LOG("Prefer libx264 for h264 codec"); + return codec; + } + FFMPEGV_LOG("Fallback to other h264 library. Fingers crossed"); + } + + return aLib->avcodec_find_encoder(aCodecId); +} + +template <> +AVCodecID GetFFmpegEncoderCodecId<LIBAV_VER>(CodecType aCodec) { +#if LIBAVCODEC_VERSION_MAJOR >= 58 + if (aCodec == CodecType::VP8) { + return AV_CODEC_ID_VP8; + } + + if (aCodec == CodecType::VP9) { + return AV_CODEC_ID_VP9; + } + +# if !defined(USING_MOZFFVPX) + if (aCodec == CodecType::H264) { + return AV_CODEC_ID_H264; + } +# endif + + if (aCodec == CodecType::AV1) { + return AV_CODEC_ID_AV1; + } + + if (aCodec == CodecType::Opus) { + return AV_CODEC_ID_OPUS; + } + + if (aCodec == CodecType::Vorbis) { + return AV_CODEC_ID_VORBIS; + } +#endif + return AV_CODEC_ID_NONE; +} + +StaticMutex FFmpegDataEncoder<LIBAV_VER>::sMutex; + +FFmpegDataEncoder<LIBAV_VER>::FFmpegDataEncoder( + const FFmpegLibWrapper* aLib, AVCodecID aCodecID, + const RefPtr<TaskQueue>& aTaskQueue, const EncoderConfig& aConfig) + : mLib(aLib), + mCodecID(aCodecID), + mTaskQueue(aTaskQueue), + mConfig(aConfig), + mCodecName(EmptyCString()), + mCodecContext(nullptr), + mFrame(nullptr), + mVideoCodec(IsVideoCodec(aCodecID)) { + MOZ_ASSERT(mLib); + MOZ_ASSERT(mTaskQueue); +#if LIBAVCODEC_VERSION_MAJOR < 58 + MOZ_CRASH("FFmpegDataEncoder needs ffmpeg 58 at least."); +#endif +}; + +RefPtr<MediaDataEncoder::InitPromise> FFmpegDataEncoder<LIBAV_VER>::Init() { + FFMPEG_LOG("Init"); + return InvokeAsync(mTaskQueue, this, __func__, + &FFmpegDataEncoder::ProcessInit); +} + +RefPtr<MediaDataEncoder::EncodePromise> FFmpegDataEncoder<LIBAV_VER>::Encode( + const MediaData* aSample) { + MOZ_ASSERT(aSample != nullptr); + + FFMPEG_LOG("Encode"); + return InvokeAsync(mTaskQueue, __func__, + [self = RefPtr<FFmpegDataEncoder<LIBAV_VER>>(this), + sample = RefPtr<const MediaData>(aSample)]() { + return self->ProcessEncode(sample); + }); +} + +RefPtr<MediaDataEncoder::ReconfigurationPromise> +FFmpegDataEncoder<LIBAV_VER>::Reconfigure( + const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) { + return InvokeAsync<const RefPtr<const EncoderConfigurationChangeList>>( + mTaskQueue, this, __func__, + &FFmpegDataEncoder<LIBAV_VER>::ProcessReconfigure, aConfigurationChanges); +} + +RefPtr<MediaDataEncoder::EncodePromise> FFmpegDataEncoder<LIBAV_VER>::Drain() { + FFMPEG_LOG("Drain"); + return InvokeAsync(mTaskQueue, this, __func__, + &FFmpegDataEncoder::ProcessDrain); +} + +RefPtr<ShutdownPromise> FFmpegDataEncoder<LIBAV_VER>::Shutdown() { + FFMPEG_LOG("Shutdown"); + return InvokeAsync(mTaskQueue, this, __func__, + &FFmpegDataEncoder::ProcessShutdown); +} + +RefPtr<GenericPromise> FFmpegDataEncoder<LIBAV_VER>::SetBitrate( + uint32_t aBitrate) { + FFMPEG_LOG("SetBitrate"); + return GenericPromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__); +} + +RefPtr<MediaDataEncoder::InitPromise> +FFmpegDataEncoder<LIBAV_VER>::ProcessInit() { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + FFMPEG_LOG("ProcessInit"); + nsresult rv = InitSpecific(); + return NS_FAILED(rv) ? InitPromise::CreateAndReject(rv, __func__) + : InitPromise::CreateAndResolve(true, __func__); +} + +RefPtr<MediaDataEncoder::EncodePromise> +FFmpegDataEncoder<LIBAV_VER>::ProcessEncode(RefPtr<const MediaData> aSample) { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + FFMPEG_LOG("ProcessEncode"); + +#if LIBAVCODEC_VERSION_MAJOR < 58 + // TODO(Bug 1868253): implement encode with avcodec_encode_video2(). + MOZ_CRASH("FFmpegDataEncoder needs ffmpeg 58 at least."); + return EncodePromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__); +#else + + auto rv = EncodeInputWithModernAPIs(std::move(aSample)); + if (rv.isErr()) { + return EncodePromise::CreateAndReject(rv.inspectErr(), __func__); + } + + return EncodePromise::CreateAndResolve(rv.unwrap(), __func__); +#endif +} + +RefPtr<MediaDataEncoder::ReconfigurationPromise> +FFmpegDataEncoder<LIBAV_VER>::ProcessReconfigure( + const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + FFMPEG_LOG("ProcessReconfigure"); + + // Tracked in bug 1869583 -- for now this encoder always reports it cannot be + // reconfigured on the fly + return MediaDataEncoder::ReconfigurationPromise::CreateAndReject( + NS_ERROR_NOT_IMPLEMENTED, __func__); +} + +RefPtr<MediaDataEncoder::EncodePromise> +FFmpegDataEncoder<LIBAV_VER>::ProcessDrain() { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + FFMPEG_LOG("ProcessDrain"); + +#if LIBAVCODEC_VERSION_MAJOR < 58 + MOZ_CRASH("FFmpegDataEncoder needs ffmpeg 58 at least."); + return EncodePromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__); +#else + auto rv = DrainWithModernAPIs(); + if (rv.isErr()) { + return EncodePromise::CreateAndReject(rv.inspectErr(), __func__); + } + return EncodePromise::CreateAndResolve(rv.unwrap(), __func__); +#endif +} + +RefPtr<ShutdownPromise> FFmpegDataEncoder<LIBAV_VER>::ProcessShutdown() { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + FFMPEG_LOG("ProcessShutdown"); + + ShutdownInternal(); + + // Don't shut mTaskQueue down since it's owned by others. + return ShutdownPromise::CreateAndResolve(true, __func__); +} + +AVCodec* FFmpegDataEncoder<LIBAV_VER>::InitCommon() { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + FFMPEG_LOG("FFmpegDataEncoder::InitCommon"); + + AVCodec* codec = FindEncoderWithPreference(mLib, mCodecID); + if (!codec) { + FFMPEG_LOG("failed to find ffmpeg encoder for codec id %d", mCodecID); + return nullptr; + } + FFMPEG_LOG("found codec: %s", codec->name); + mCodecName = codec->name; + + ForceEnablingFFmpegDebugLogs(); + + MOZ_ASSERT(!mCodecContext); + if (!(mCodecContext = mLib->avcodec_alloc_context3(codec))) { + FFMPEG_LOG("failed to allocate ffmpeg context for codec %s", codec->name); + return nullptr; + } + + return codec; +} + +MediaResult FFmpegDataEncoder<LIBAV_VER>::FinishInitCommon(AVCodec* aCodec) { + mCodecContext->bit_rate = static_cast<FFmpegBitRate>(mConfig.mBitrate); +#if LIBAVCODEC_VERSION_MAJOR >= 60 + mCodecContext->flags |= AV_CODEC_FLAG_FRAME_DURATION; +#endif + + AVDictionary* options = nullptr; + if (int ret = OpenCodecContext(aCodec, &options); ret < 0) { + FFMPEG_LOG("failed to open %s avcodec: %s", aCodec->name, + MakeErrorString(mLib, ret).get()); + return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("avcodec_open2 error")); + } + mLib->av_dict_free(&options); + + return MediaResult(NS_OK); +} + +void FFmpegDataEncoder<LIBAV_VER>::ShutdownInternal() { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + FFMPEG_LOG("ShutdownInternal"); + + DestroyFrame(); + + if (mCodecContext) { + CloseCodecContext(); + mLib->av_freep(&mCodecContext); + mCodecContext = nullptr; + } +} + +int FFmpegDataEncoder<LIBAV_VER>::OpenCodecContext(const AVCodec* aCodec, + AVDictionary** aOptions) { + MOZ_ASSERT(mCodecContext); + + StaticMutexAutoLock mon(sMutex); + return mLib->avcodec_open2(mCodecContext, aCodec, aOptions); +} + +void FFmpegDataEncoder<LIBAV_VER>::CloseCodecContext() { + MOZ_ASSERT(mCodecContext); + + StaticMutexAutoLock mon(sMutex); + mLib->avcodec_close(mCodecContext); +} + +bool FFmpegDataEncoder<LIBAV_VER>::PrepareFrame() { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + // TODO: Merge the duplicate part with FFmpegDataDecoder's PrepareFrame. +#if LIBAVCODEC_VERSION_MAJOR >= 55 + if (mFrame) { + mLib->av_frame_unref(mFrame); + } else { + mFrame = mLib->av_frame_alloc(); + } +#elif LIBAVCODEC_VERSION_MAJOR == 54 + if (mFrame) { + mLib->avcodec_get_frame_defaults(mFrame); + } else { + mFrame = mLib->avcodec_alloc_frame(); + } +#else + mLib->av_freep(&mFrame); + mFrame = mLib->avcodec_alloc_frame(); +#endif + return !!mFrame; +} + +void FFmpegDataEncoder<LIBAV_VER>::DestroyFrame() { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + if (mFrame) { +#if LIBAVCODEC_VERSION_MAJOR >= 55 + mLib->av_frame_unref(mFrame); + mLib->av_frame_free(&mFrame); +#elif LIBAVCODEC_VERSION_MAJOR == 54 + mLib->avcodec_free_frame(&mFrame); +#else + mLib->av_freep(&mFrame); +#endif + mFrame = nullptr; + } +} + +// avcodec_send_frame and avcodec_receive_packet were introduced in version 58. +#if LIBAVCODEC_VERSION_MAJOR >= 58 +Result<MediaDataEncoder::EncodedData, nsresult> +FFmpegDataEncoder<LIBAV_VER>::EncodeWithModernAPIs() { + // Initialize AVPacket. + AVPacket* pkt = mLib->av_packet_alloc(); + + if (!pkt) { + FFMPEG_LOG("failed to allocate packet"); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + auto freePacket = MakeScopeExit([this, &pkt] { mLib->av_packet_free(&pkt); }); + + // Send frame and receive packets. + if (int ret = mLib->avcodec_send_frame(mCodecContext, mFrame); ret < 0) { + // In theory, avcodec_send_frame could sent -EAGAIN to signal its internal + // buffers is full. In practice this can't happen as we only feed one frame + // at a time, and we immediately call avcodec_receive_packet right after. + // TODO: Create a NS_ERROR_DOM_MEDIA_ENCODE_ERR in ErrorList.py? + FFMPEG_LOG("avcodec_send_frame error: %s", + MakeErrorString(mLib, ret).get()); + return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR); + } + + EncodedData output; + while (true) { + int ret = mLib->avcodec_receive_packet(mCodecContext, pkt); + if (ret == AVERROR(EAGAIN)) { + // The encoder is asking for more inputs. + FFMPEG_LOG("encoder is asking for more input!"); + break; + } + + if (ret < 0) { + // AVERROR_EOF is returned when the encoder has been fully flushed, but it + // shouldn't happen here. + FFMPEG_LOG("avcodec_receive_packet error: %s", + MakeErrorString(mLib, ret).get()); + return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR); + } + + RefPtr<MediaRawData> d = ToMediaRawData(pkt); + mLib->av_packet_unref(pkt); + if (!d) { + // This can happen if e.g. DTX is enabled + FFMPEG_LOG("No encoded packet output"); + continue; + } + output.AppendElement(std::move(d)); + } + + FFMPEG_LOG("Got %zu encoded data", output.Length()); + return std::move(output); +} + +Result<MediaDataEncoder::EncodedData, nsresult> +FFmpegDataEncoder<LIBAV_VER>::DrainWithModernAPIs() { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + MOZ_ASSERT(mCodecContext); + + // TODO: Create a Result<EncodedData, nsresult> EncodeWithModernAPIs(AVFrame + // *aFrame) to merge the duplicate code below with EncodeWithModernAPIs above. + + // Initialize AVPacket. + AVPacket* pkt = mLib->av_packet_alloc(); + if (!pkt) { + FFMPEG_LOG("failed to allocate packet"); + return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR); + } + auto freePacket = MakeScopeExit([this, &pkt] { mLib->av_packet_free(&pkt); }); + + // Enter draining mode by sending NULL to the avcodec_send_frame(). Note that + // this can leave the encoder in a permanent EOF state after draining. As a + // result, the encoder is unable to continue encoding. A new + // AVCodecContext/encoder creation is required if users need to encode after + // draining. + // + // TODO: Use `avcodec_flush_buffers` to drain the pending packets if + // AV_CODEC_CAP_ENCODER_FLUSH is set in mCodecContext->codec->capabilities. + if (int ret = mLib->avcodec_send_frame(mCodecContext, nullptr); ret < 0) { + if (ret == AVERROR_EOF) { + // The encoder has been flushed. Drain can be called multiple time. + FFMPEG_LOG("encoder has been flushed!"); + return EncodedData(); + } + + FFMPEG_LOG("avcodec_send_frame error: %s", + MakeErrorString(mLib, ret).get()); + return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR); + } + + EncodedData output; + while (true) { + int ret = mLib->avcodec_receive_packet(mCodecContext, pkt); + if (ret == AVERROR_EOF) { + FFMPEG_LOG("encoder has no more output packet!"); + break; + } + + if (ret < 0) { + // avcodec_receive_packet should not result in a -EAGAIN once it's in + // draining mode. + FFMPEG_LOG("avcodec_receive_packet error: %s", + MakeErrorString(mLib, ret).get()); + return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR); + } + + RefPtr<MediaRawData> d = ToMediaRawData(pkt); + mLib->av_packet_unref(pkt); + if (!d) { + FFMPEG_LOG("failed to create a MediaRawData from the AVPacket"); + return Err(NS_ERROR_DOM_MEDIA_FATAL_ERR); + } + output.AppendElement(std::move(d)); + } + + FFMPEG_LOG("Encoding successful, %zu packets", output.Length()); + + // TODO: Evaluate a better solution (Bug 1869466) + // TODO: Only re-create AVCodecContext when avcodec_flush_buffers is + // unavailable. + ShutdownInternal(); + nsresult r = InitSpecific(); + return NS_FAILED(r) ? Result<MediaDataEncoder::EncodedData, nsresult>( + NS_ERROR_DOM_MEDIA_FATAL_ERR) + : Result<MediaDataEncoder::EncodedData, nsresult>( + std::move(output)); +} +#endif // LIBAVCODEC_VERSION_MAJOR >= 58 + +RefPtr<MediaRawData> FFmpegDataEncoder<LIBAV_VER>::ToMediaRawDataCommon( + AVPacket* aPacket) { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + MOZ_ASSERT(aPacket); + + // Copy frame data from AVPacket. + auto data = MakeRefPtr<MediaRawData>(); + UniquePtr<MediaRawDataWriter> writer(data->CreateWriter()); + if (!writer->Append(aPacket->data, static_cast<size_t>(aPacket->size))) { + FFMPEG_LOG("fail to allocate MediaRawData buffer"); + return nullptr; // OOM + } + + data->mKeyframe = (aPacket->flags & AV_PKT_FLAG_KEY) != 0; + // TODO(bug 1869560): The unit of pts, dts, and duration is time_base, which + // is recommended to be the reciprocal of the frame rate, but we set it to + // microsecond for now. + data->mTime = media::TimeUnit::FromMicroseconds(aPacket->pts); +#if LIBAVCODEC_VERSION_MAJOR >= 60 + data->mDuration = media::TimeUnit::FromMicroseconds(aPacket->duration); +#else + int64_t duration; + if (mDurationMap.Find(aPacket->pts, duration)) { + data->mDuration = media::TimeUnit::FromMicroseconds(duration); + } else { + data->mDuration = media::TimeUnit::FromMicroseconds(aPacket->duration); + } +#endif + data->mTimecode = media::TimeUnit::FromMicroseconds(aPacket->dts); + + if (auto r = GetExtraData(aPacket); r.isOk()) { + data->mExtraData = r.unwrap(); + } + + return data; +} +void FFmpegDataEncoder<LIBAV_VER>::ForceEnablingFFmpegDebugLogs() { +#if DEBUG + if (!getenv("MOZ_AV_LOG_LEVEL") && + MOZ_LOG_TEST(sFFmpegVideoLog, LogLevel::Debug)) { + mLib->av_log_set_level(AV_LOG_DEBUG); + } +#endif // DEBUG +} + +} // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegDataEncoder.h b/dom/media/platforms/ffmpeg/FFmpegDataEncoder.h new file mode 100644 index 0000000000..de80ed36ca --- /dev/null +++ b/dom/media/platforms/ffmpeg/FFmpegDataEncoder.h @@ -0,0 +1,107 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGDATAENCODER_H_ +#define DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGDATAENCODER_H_ + +#include "FFmpegLibWrapper.h" +#include "PlatformEncoderModule.h" +#include "SimpleMap.h" +#include "mozilla/ThreadSafety.h" + +// This must be the last header included +#include "FFmpegLibs.h" + +namespace mozilla { + +template <int V> +AVCodecID GetFFmpegEncoderCodecId(CodecType aCodec); + +template <> +AVCodecID GetFFmpegEncoderCodecId<LIBAV_VER>(CodecType aCodec); + +template <int V> +class FFmpegDataEncoder : public MediaDataEncoder {}; + +template <> +class FFmpegDataEncoder<LIBAV_VER> : public MediaDataEncoder { + using DurationMap = SimpleMap<int64_t>; + + public: + FFmpegDataEncoder(const FFmpegLibWrapper* aLib, AVCodecID aCodecID, + const RefPtr<TaskQueue>& aTaskQueue, + const EncoderConfig& aConfig); + + /* MediaDataEncoder Methods */ + // All methods run on the task queue, except for GetDescriptionName. + RefPtr<InitPromise> Init() override; + RefPtr<EncodePromise> Encode(const MediaData* aSample) override; + RefPtr<ReconfigurationPromise> Reconfigure( + const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) + override; + RefPtr<EncodePromise> Drain() override; + RefPtr<ShutdownPromise> Shutdown() override; + RefPtr<GenericPromise> SetBitrate(uint32_t aBitRate) override; + + protected: + // Methods only called on mTaskQueue. + RefPtr<InitPromise> ProcessInit(); + RefPtr<EncodePromise> ProcessEncode(RefPtr<const MediaData> aSample); + RefPtr<ReconfigurationPromise> ProcessReconfigure( + const RefPtr<const EncoderConfigurationChangeList>& + aConfigurationChanges); + RefPtr<EncodePromise> ProcessDrain(); + RefPtr<ShutdownPromise> ProcessShutdown(); + // Initialize the audio or video-specific members of an encoder instance. + virtual nsresult InitSpecific() = 0; + // nullptr in case of failure. This is to be called by the + // audio/video-specific InitInternal methods in the sub-class, and initializes + // the common members. + AVCodec* InitCommon(); + MediaResult FinishInitCommon(AVCodec* aCodec); + void ShutdownInternal(); + int OpenCodecContext(const AVCodec* aCodec, AVDictionary** aOptions) + MOZ_EXCLUDES(sMutex); + void CloseCodecContext() MOZ_EXCLUDES(sMutex); + bool PrepareFrame(); + void DestroyFrame(); +#if LIBAVCODEC_VERSION_MAJOR >= 58 + virtual Result<EncodedData, nsresult> EncodeInputWithModernAPIs( + RefPtr<const MediaData> aSample) = 0; + Result<EncodedData, nsresult> EncodeWithModernAPIs(); + virtual Result<EncodedData, nsresult> DrainWithModernAPIs(); +#endif + // Convert an AVPacket to a MediaRawData. This can return nullptr if a packet + // has been processed by the encoder, but is not to be returned to the caller, + // because DTX is enabled. + virtual RefPtr<MediaRawData> ToMediaRawData(AVPacket* aPacket) = 0; + RefPtr<MediaRawData> ToMediaRawDataCommon(AVPacket* aPacket); + virtual Result<already_AddRefed<MediaByteBuffer>, nsresult> GetExtraData( + AVPacket* aPacket) = 0; + void ForceEnablingFFmpegDebugLogs(); + + // This refers to a static FFmpegLibWrapper, so raw pointer is adequate. + const FFmpegLibWrapper* mLib; + const AVCodecID mCodecID; + const RefPtr<TaskQueue> mTaskQueue; + + // set in constructor, modified when parameters change + EncoderConfig mConfig; + + // mTaskQueue only. + nsCString mCodecName; + AVCodecContext* mCodecContext; + AVFrame* mFrame; + DurationMap mDurationMap; + + // Provide critical-section for open/close mCodecContext. + static StaticMutex sMutex; + const bool mVideoCodec; +}; + +} // namespace mozilla + +#endif /* DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGDATAENCODER_H_ */ diff --git a/dom/media/platforms/ffmpeg/FFmpegEncoderModule.cpp b/dom/media/platforms/ffmpeg/FFmpegEncoderModule.cpp index 42c54a48ed..b6e734268d 100644 --- a/dom/media/platforms/ffmpeg/FFmpegEncoderModule.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegEncoderModule.cpp @@ -7,6 +7,7 @@ #include "FFmpegEncoderModule.h" #include "FFmpegLog.h" +#include "FFmpegAudioEncoder.h" #include "FFmpegVideoEncoder.h" // This must be the last header included @@ -44,6 +45,23 @@ already_AddRefed<MediaDataEncoder> FFmpegEncoderModule<V>::CreateVideoEncoder( return encoder.forget(); } +template <int V> +already_AddRefed<MediaDataEncoder> FFmpegEncoderModule<V>::CreateAudioEncoder( + const EncoderConfig& aConfig, const RefPtr<TaskQueue>& aTaskQueue) const { + AVCodecID codecId = GetFFmpegEncoderCodecId<V>(aConfig.mCodec); + if (codecId == AV_CODEC_ID_NONE) { + FFMPEGV_LOG("No ffmpeg encoder for %s", GetCodecTypeString(aConfig.mCodec)); + return nullptr; + } + + RefPtr<MediaDataEncoder> encoder = + new FFmpegAudioEncoder<V>(mLib, codecId, aTaskQueue, aConfig); + FFMPEGA_LOG("ffmpeg %s encoder: %s has been created", + GetCodecTypeString(aConfig.mCodec), + encoder->GetDescriptionName().get()); + return encoder.forget(); +} + template class FFmpegEncoderModule<LIBAV_VER>; } // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegEncoderModule.h b/dom/media/platforms/ffmpeg/FFmpegEncoderModule.h index 1c9e94b78f..6d0e4b1c30 100644 --- a/dom/media/platforms/ffmpeg/FFmpegEncoderModule.h +++ b/dom/media/platforms/ffmpeg/FFmpegEncoderModule.h @@ -30,6 +30,10 @@ class FFmpegEncoderModule final : public PlatformEncoderModule { const EncoderConfig& aConfig, const RefPtr<TaskQueue>& aTaskQueue) const override; + already_AddRefed<MediaDataEncoder> CreateAudioEncoder( + const EncoderConfig& aConfig, + const RefPtr<TaskQueue>& aTaskQueue) const override; + protected: explicit FFmpegEncoderModule(FFmpegLibWrapper* aLib) : mLib(aLib) { MOZ_ASSERT(mLib); diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp index bfb3105a57..5fd6102a34 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp @@ -200,6 +200,7 @@ FFmpegLibWrapper::LinkResult FFmpegLibWrapper::Link() { AV_FUNC(av_image_get_buffer_size, AV_FUNC_AVUTIL_ALL) AV_FUNC_OPTION(av_channel_layout_default, AV_FUNC_AVUTIL_60) AV_FUNC_OPTION(av_channel_layout_from_mask, AV_FUNC_AVUTIL_60) + AV_FUNC_OPTION(av_channel_layout_copy, AV_FUNC_AVUTIL_60) AV_FUNC_OPTION(av_buffer_get_opaque, (AV_FUNC_AVUTIL_56 | AV_FUNC_AVUTIL_57 | AV_FUNC_AVUTIL_58 | AV_FUNC_AVUTIL_59 | AV_FUNC_AVUTIL_60)) @@ -218,6 +219,8 @@ FFmpegLibWrapper::LinkResult FFmpegLibWrapper::Link() { AV_FUNC(av_dict_set, AV_FUNC_AVUTIL_ALL) AV_FUNC(av_dict_free, AV_FUNC_AVUTIL_ALL) AV_FUNC(av_opt_set, AV_FUNC_AVUTIL_ALL) + AV_FUNC(av_opt_set_double, AV_FUNC_AVUTIL_ALL) + AV_FUNC(av_opt_set_int, AV_FUNC_AVUTIL_ALL) #ifdef MOZ_WIDGET_GTK AV_FUNC_OPTION_SILENT(avcodec_get_hw_config, diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h index eacbba286a..226b4fc8cb 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h +++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h @@ -161,11 +161,16 @@ struct MOZ_ONLY_USED_TO_AVOID_STATIC_CONSTRUCTORS FFmpegLibWrapper { int nb_channels); void (*av_channel_layout_from_mask)(AVChannelLayout* ch_layout, uint64_t mask); + int (*av_channel_layout_copy)(AVChannelLayout* dst, AVChannelLayout* src); int (*av_dict_set)(AVDictionary** pm, const char* key, const char* value, int flags); void (*av_dict_free)(AVDictionary** m); int (*av_opt_set)(void* obj, const char* name, const char* val, int search_flags); + int (*av_opt_set_double)(void* obj, const char* name, double val, + int search_flags); + int (*av_opt_set_int)(void* obj, const char* name, int64_t val, + int search_flags); // libavutil v55 and later only AVFrame* (*av_frame_alloc)(); diff --git a/dom/media/platforms/ffmpeg/FFmpegLog.h b/dom/media/platforms/ffmpeg/FFmpegLog.h index 45ea700936..676c5e4ba1 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLog.h +++ b/dom/media/platforms/ffmpeg/FFmpegLog.h @@ -19,6 +19,9 @@ static mozilla::LazyLogModule sFFmpegAudioLog("FFmpegAudio"); # define FFMPEGV_LOG(str, ...) \ MOZ_LOG(sFFmpegVideoLog, mozilla::LogLevel::Debug, \ ("FFVPX: " str, ##__VA_ARGS__)) +# define FFMPEGA_LOG(str, ...) \ + MOZ_LOG(sFFmpegAudioLog, mozilla::LogLevel::Debug, \ + ("FFVPX: " str, ##__VA_ARGS__)) # define FFMPEGP_LOG(str, ...) \ MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, ("FFVPX: " str, ##__VA_ARGS__)) #else @@ -28,11 +31,15 @@ static mozilla::LazyLogModule sFFmpegAudioLog("FFmpegAudio"); # define FFMPEGV_LOG(str, ...) \ MOZ_LOG(sFFmpegVideoLog, mozilla::LogLevel::Debug, \ ("FFMPEG: " str, ##__VA_ARGS__)) +# define FFMPEGA_LOG(str, ...) \ + MOZ_LOG(sFFmpegAudioLog, mozilla::LogLevel::Debug, \ + ("FFMPEG: " str, ##__VA_ARGS__)) # define FFMPEGP_LOG(str, ...) \ MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, ("FFMPEG: " str, ##__VA_ARGS__)) #endif -#define FFMPEG_LOGV(...) \ - MOZ_LOG(sFFmpegVideoLog, mozilla::LogLevel::Verbose, (__VA_ARGS__)) +#define FFMPEG_LOGV(...) \ + MOZ_LOG(mVideoCodec ? sFFmpegVideoLog : sFFmpegAudioLog, \ + mozilla::LogLevel::Verbose, (__VA_ARGS__)) #endif // __FFmpegLog_h__ diff --git a/dom/media/platforms/ffmpeg/FFmpegUtils.cpp b/dom/media/platforms/ffmpeg/FFmpegUtils.cpp new file mode 100644 index 0000000000..e209306133 --- /dev/null +++ b/dom/media/platforms/ffmpeg/FFmpegUtils.cpp @@ -0,0 +1,23 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FFmpegUtils.h" + +#include "FFmpegLibWrapper.h" +#include "mozilla/Assertions.h" +#include "nsString.h" + +namespace mozilla { + +nsCString MakeErrorString(const FFmpegLibWrapper* aLib, int aErrNum) { + MOZ_ASSERT(aLib); + + char errStr[FFmpegErrorMaxStringSize]; + aLib->av_strerror(aErrNum, errStr, FFmpegErrorMaxStringSize); + return nsCString(errStr); +} + +} // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegUtils.h b/dom/media/platforms/ffmpeg/FFmpegUtils.h new file mode 100644 index 0000000000..fe588ed14c --- /dev/null +++ b/dom/media/platforms/ffmpeg/FFmpegUtils.h @@ -0,0 +1,56 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGUTILS_H_ +#define DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGUTILS_H_ + +#include <cstddef> +#include "nsStringFwd.h" +#include "FFmpegLibWrapper.h" + +// This must be the last header included +#include "FFmpegLibs.h" + +namespace mozilla { + +#if LIBAVCODEC_VERSION_MAJOR >= 57 +using FFmpegBitRate = int64_t; +constexpr size_t FFmpegErrorMaxStringSize = AV_ERROR_MAX_STRING_SIZE; +#else +using FFmpegBitRate = int; +constexpr size_t FFmpegErrorMaxStringSize = 64; +#endif + +nsCString MakeErrorString(const FFmpegLibWrapper* aLib, int aErrNum); + +template <typename T, typename F> +void IterateZeroTerminated(const T& aList, F&& aLambda) { + for (size_t i = 0; aList[i] != 0; i++) { + aLambda(aList[i]); + } +} + +inline bool IsVideoCodec(AVCodecID aCodecID) { + switch (aCodecID) { + case AV_CODEC_ID_H264: +#if LIBAVCODEC_VERSION_MAJOR >= 54 + case AV_CODEC_ID_VP8: +#endif +#if LIBAVCODEC_VERSION_MAJOR >= 55 + case AV_CODEC_ID_VP9: +#endif +#if LIBAVCODEC_VERSION_MAJOR >= 59 + case AV_CODEC_ID_AV1: +#endif + return true; + default: + return false; + } +} + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGUTILS_H_ diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp index 040b2e72a1..3fe46938fd 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp @@ -46,6 +46,7 @@ # define AV_PIX_FMT_YUV444P10LE PIX_FMT_YUV444P10LE # define AV_PIX_FMT_GBRP PIX_FMT_GBRP # define AV_PIX_FMT_NONE PIX_FMT_NONE +# define AV_PIX_FMT_VAAPI_VLD PIX_FMT_VAAPI_VLD #endif #if LIBAVCODEC_VERSION_MAJOR > 58 # define AV_PIX_FMT_VAAPI_VLD AV_PIX_FMT_VAAPI @@ -618,6 +619,9 @@ static gfx::ColorDepth GetColorDepth(const AVPixelFormat& aFormat) { case AV_PIX_FMT_YUV444P12LE: return gfx::ColorDepth::COLOR_12; #endif + case AV_PIX_FMT_VAAPI_VLD: + // Placeholder, it could be deeper colors + return gfx::ColorDepth::COLOR_8; default: MOZ_ASSERT_UNREACHABLE("Not supported format?"); return gfx::ColorDepth::COLOR_8; @@ -662,7 +666,7 @@ static int GetVideoBufferWrapper(struct AVCodecContext* aCodecContext, static void ReleaseVideoBufferWrapper(void* opaque, uint8_t* data) { if (opaque) { - FFMPEG_LOGV("ReleaseVideoBufferWrapper: PlanarYCbCrImage=%p", opaque); + FFMPEGV_LOG("ReleaseVideoBufferWrapper: PlanarYCbCrImage=%p", opaque); RefPtr<ImageBufferWrapper> image = static_cast<ImageBufferWrapper*>(opaque); image->ReleaseBuffer(); } @@ -1199,6 +1203,8 @@ MediaResult FFmpegVideoDecoder<LIBAV_VER>::DoDecode( return Some(DecodeStage::YUV444P); case AV_PIX_FMT_GBRP: return Some(DecodeStage::GBRP); + case AV_PIX_FMT_VAAPI_VLD: + return Some(DecodeStage::VAAPI_SURFACE); default: return Nothing(); } diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp index a3cfdf1b1d..9d1dbcf80f 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.cpp @@ -8,32 +8,21 @@ #include "BufferReader.h" #include "FFmpegLog.h" -#include "FFmpegRuntimeLinker.h" +#include "FFmpegUtils.h" #include "H264.h" #include "ImageContainer.h" #include "libavutil/error.h" #include "libavutil/pixfmt.h" -#include "mozilla/CheckedInt.h" -#include "mozilla/PodOperations.h" -#include "mozilla/StaticMutex.h" -#include "mozilla/dom/ImageBitmapBinding.h" #include "mozilla/dom/ImageUtils.h" #include "nsPrintfCString.h" #include "ImageToI420.h" #include "libyuv.h" +#include "FFmpegRuntimeLinker.h" // The ffmpeg namespace is introduced to avoid the PixelFormat's name conflicts // with MediaDataEncoder::PixelFormat in MediaDataEncoder class scope. namespace ffmpeg { -#if LIBAVCODEC_VERSION_MAJOR >= 57 -using FFmpegBitRate = int64_t; -constexpr size_t FFmpegErrorMaxStringSize = AV_ERROR_MAX_STRING_SIZE; -#else -using FFmpegBitRate = int; -constexpr size_t FFmpegErrorMaxStringSize = 64; -#endif - // TODO: WebCodecs' I420A should map to MediaDataEncoder::PixelFormat and then // to AV_PIX_FMT_YUVA420P here. #if LIBAVCODEC_VERSION_MAJOR < 54 @@ -166,9 +155,9 @@ struct VPXSVCSetting { nsTArray<uint32_t> mTargetBitrates; }; -static Maybe<VPXSVCSetting> GetVPXSVCSetting( - const MediaDataEncoder::ScalabilityMode& aMode, uint32_t aBitPerSec) { - if (aMode == MediaDataEncoder::ScalabilityMode::None) { +static Maybe<VPXSVCSetting> GetVPXSVCSetting(const ScalabilityMode& aMode, + uint32_t aBitPerSec) { + if (aMode == ScalabilityMode::None) { return Nothing(); } @@ -183,7 +172,7 @@ static Maybe<VPXSVCSetting> GetVPXSVCSetting( nsTArray<uint8_t> layerIds; nsTArray<uint8_t> rateDecimators; nsTArray<uint32_t> bitrates; - if (aMode == MediaDataEncoder::ScalabilityMode::L1T2) { + if (aMode == ScalabilityMode::L1T2) { // Two temporal layers. 0-1... // // Frame pattern: @@ -208,7 +197,7 @@ static Maybe<VPXSVCSetting> GetVPXSVCSetting( bitrates.AppendElement(kbps * 3 / 5); bitrates.AppendElement(kbps); } else { - MOZ_ASSERT(aMode == MediaDataEncoder::ScalabilityMode::L1T3); + MOZ_ASSERT(aMode == ScalabilityMode::L1T3); // Three temporal layers. 0-2-1-2... // // Frame pattern: @@ -245,59 +234,6 @@ static Maybe<VPXSVCSetting> GetVPXSVCSetting( std::move(rateDecimators), std::move(bitrates)}); } -static nsCString MakeErrorString(const FFmpegLibWrapper* aLib, int aErrNum) { - MOZ_ASSERT(aLib); - - char errStr[ffmpeg::FFmpegErrorMaxStringSize]; - aLib->av_strerror(aErrNum, errStr, ffmpeg::FFmpegErrorMaxStringSize); - return nsCString(errStr); -} - -// TODO: Remove this function and simply use `avcodec_find_encoder` once -// libopenh264 is supported. -static AVCodec* FindEncoderWithPreference(const FFmpegLibWrapper* aLib, - AVCodecID aCodecId) { - MOZ_ASSERT(aLib); - - AVCodec* codec = nullptr; - - // Prioritize libx264 for now since it's the only h264 codec we tested. - if (aCodecId == AV_CODEC_ID_H264) { - codec = aLib->avcodec_find_encoder_by_name("libx264"); - if (codec) { - FFMPEGV_LOG("Prefer libx264 for h264 codec"); - return codec; - } - } - - FFMPEGV_LOG("Fallback to other h264 library. Fingers crossed"); - return aLib->avcodec_find_encoder(aCodecId); -} - -template <> -AVCodecID GetFFmpegEncoderCodecId<LIBAV_VER>(CodecType aCodec) { -#if LIBAVCODEC_VERSION_MAJOR >= 58 - if (aCodec == CodecType::VP8) { - return AV_CODEC_ID_VP8; - } - - if (aCodec == CodecType::VP9) { - return AV_CODEC_ID_VP9; - } - -# if !defined(USING_MOZFFVPX) - if (aCodec == CodecType::H264) { - return AV_CODEC_ID_H264; - } -# endif - - if (aCodec == CodecType::AV1) { - return AV_CODEC_ID_AV1; - } -#endif - return AV_CODEC_ID_NONE; -} - uint8_t FFmpegVideoEncoder<LIBAV_VER>::SVCInfo::UpdateTemporalLayerId() { MOZ_ASSERT(!mTemporalLayerIds.IsEmpty()); @@ -306,70 +242,10 @@ uint8_t FFmpegVideoEncoder<LIBAV_VER>::SVCInfo::UpdateTemporalLayerId() { return static_cast<uint8_t>(mTemporalLayerIds[currentIndex]); } -StaticMutex FFmpegVideoEncoder<LIBAV_VER>::sMutex; - FFmpegVideoEncoder<LIBAV_VER>::FFmpegVideoEncoder( const FFmpegLibWrapper* aLib, AVCodecID aCodecID, const RefPtr<TaskQueue>& aTaskQueue, const EncoderConfig& aConfig) - : mLib(aLib), - mCodecID(aCodecID), - mTaskQueue(aTaskQueue), - mConfig(aConfig), - mCodecName(EmptyCString()), - mCodecContext(nullptr), - mFrame(nullptr), - mSVCInfo(Nothing()) { - MOZ_ASSERT(mLib); - MOZ_ASSERT(mTaskQueue); -#if LIBAVCODEC_VERSION_MAJOR < 58 - MOZ_CRASH("FFmpegVideoEncoder needs ffmpeg 58 at least."); -#endif -}; - -RefPtr<MediaDataEncoder::InitPromise> FFmpegVideoEncoder<LIBAV_VER>::Init() { - FFMPEGV_LOG("Init"); - return InvokeAsync(mTaskQueue, this, __func__, - &FFmpegVideoEncoder::ProcessInit); -} - -RefPtr<MediaDataEncoder::EncodePromise> FFmpegVideoEncoder<LIBAV_VER>::Encode( - const MediaData* aSample) { - MOZ_ASSERT(aSample != nullptr); - - FFMPEGV_LOG("Encode"); - return InvokeAsync(mTaskQueue, __func__, - [self = RefPtr<FFmpegVideoEncoder<LIBAV_VER>>(this), - sample = RefPtr<const MediaData>(aSample)]() { - return self->ProcessEncode(std::move(sample)); - }); -} - -RefPtr<MediaDataEncoder::ReconfigurationPromise> -FFmpegVideoEncoder<LIBAV_VER>::Reconfigure( - const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) { - return InvokeAsync<const RefPtr<const EncoderConfigurationChangeList>>( - mTaskQueue, this, __func__, - &FFmpegVideoEncoder<LIBAV_VER>::ProcessReconfigure, - aConfigurationChanges); -} - -RefPtr<MediaDataEncoder::EncodePromise> FFmpegVideoEncoder<LIBAV_VER>::Drain() { - FFMPEGV_LOG("Drain"); - return InvokeAsync(mTaskQueue, this, __func__, - &FFmpegVideoEncoder::ProcessDrain); -} - -RefPtr<ShutdownPromise> FFmpegVideoEncoder<LIBAV_VER>::Shutdown() { - FFMPEGV_LOG("Shutdown"); - return InvokeAsync(mTaskQueue, this, __func__, - &FFmpegVideoEncoder::ProcessShutdown); -} - -RefPtr<GenericPromise> FFmpegVideoEncoder<LIBAV_VER>::SetBitrate( - uint32_t aBitrate) { - FFMPEGV_LOG("SetBitrate"); - return GenericPromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__); -} + : FFmpegDataEncoder(aLib, aCodecID, aTaskQueue, aConfig) {} nsCString FFmpegVideoEncoder<LIBAV_VER>::GetDescriptionName() const { #ifdef USING_MOZFFVPX @@ -385,112 +261,23 @@ nsCString FFmpegVideoEncoder<LIBAV_VER>::GetDescriptionName() const { #endif } -RefPtr<MediaDataEncoder::InitPromise> -FFmpegVideoEncoder<LIBAV_VER>::ProcessInit() { +nsresult FFmpegVideoEncoder<LIBAV_VER>::InitSpecific() { MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); - FFMPEGV_LOG("ProcessInit"); - MediaResult r = InitInternal(); - return NS_FAILED(r) - ? InitPromise::CreateAndReject(r, __func__) - : InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); -} - -RefPtr<MediaDataEncoder::EncodePromise> -FFmpegVideoEncoder<LIBAV_VER>::ProcessEncode(RefPtr<const MediaData> aSample) { - MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + FFMPEGV_LOG("FFmpegVideoEncoder::InitSpecific"); - FFMPEGV_LOG("ProcessEncode"); - -#if LIBAVCODEC_VERSION_MAJOR < 58 - // TODO(Bug 1868253): implement encode with avcodec_encode_video2(). - MOZ_CRASH("FFmpegVideoEncoder needs ffmpeg 58 at least."); - return EncodePromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__); -#else - RefPtr<const VideoData> sample(aSample->As<const VideoData>()); - MOZ_ASSERT(sample); - - return EncodeWithModernAPIs(sample); -#endif -} - -RefPtr<MediaDataEncoder::ReconfigurationPromise> -FFmpegVideoEncoder<LIBAV_VER>::ProcessReconfigure( - const RefPtr<const EncoderConfigurationChangeList> aConfigurationChanges) { - MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); - - FFMPEGV_LOG("ProcessReconfigure"); - - // Tracked in bug 1869583 -- for now this encoder always reports it cannot be - // reconfigured on the fly - return MediaDataEncoder::ReconfigurationPromise::CreateAndReject( - NS_ERROR_NOT_IMPLEMENTED, __func__); -} - -RefPtr<MediaDataEncoder::EncodePromise> -FFmpegVideoEncoder<LIBAV_VER>::ProcessDrain() { - MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); - - FFMPEGV_LOG("ProcessDrain"); - -#if LIBAVCODEC_VERSION_MAJOR < 58 - MOZ_CRASH("FFmpegVideoEncoder needs ffmpeg 58 at least."); - return EncodePromise::CreateAndReject(NS_ERROR_NOT_IMPLEMENTED, __func__); -#else - return DrainWithModernAPIs(); -#endif -} - -RefPtr<ShutdownPromise> FFmpegVideoEncoder<LIBAV_VER>::ProcessShutdown() { - MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); - - FFMPEGV_LOG("ProcessShutdown"); - - ShutdownInternal(); - - // Don't shut mTaskQueue down since it's owned by others. - return ShutdownPromise::CreateAndResolve(true, __func__); -} - -MediaResult FFmpegVideoEncoder<LIBAV_VER>::InitInternal() { - MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); - - FFMPEGV_LOG("InitInternal"); - - if (mCodecID == AV_CODEC_ID_H264) { - // H264Specific is required to get the format (avcc vs annexb). - if (!mConfig.mCodecSpecific || - !mConfig.mCodecSpecific->is<H264Specific>()) { - return MediaResult( - NS_ERROR_DOM_MEDIA_FATAL_ERR, - RESULT_DETAIL("Unable to get H264 necessary encoding info")); - } - } - - AVCodec* codec = FindEncoderWithPreference(mLib, mCodecID); + // Initialize the common members of the encoder instance + AVCodec* codec = FFmpegDataEncoder<LIBAV_VER>::InitCommon(); if (!codec) { - FFMPEGV_LOG("failed to find ffmpeg encoder for codec id %d", mCodecID); - return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, - RESULT_DETAIL("Unable to find codec")); + FFMPEGV_LOG("FFmpegDataEncoder::InitCommon failed"); + return NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR; } - FFMPEGV_LOG("find codec: %s", codec->name); - mCodecName = codec->name; - ForceEnablingFFmpegDebugLogs(); - - MOZ_ASSERT(!mCodecContext); - if (!(mCodecContext = mLib->avcodec_alloc_context3(codec))) { - FFMPEGV_LOG("failed to allocate ffmpeg context for codec %s", codec->name); - return MediaResult(NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("Failed to initialize ffmpeg context")); - } - - // Set up AVCodecContext. + // And now the video-specific part mCodecContext->pix_fmt = ffmpeg::FFMPEG_PIX_FMT_YUV420P; - mCodecContext->bit_rate = - static_cast<ffmpeg::FFmpegBitRate>(mConfig.mBitrate); mCodecContext->width = static_cast<int>(mConfig.mSize.width); mCodecContext->height = static_cast<int>(mConfig.mSize.height); + mCodecContext->gop_size = static_cast<int>(mConfig.mKeyframeInterval); // TODO(bug 1869560): The recommended time_base is the reciprocal of the frame // rate, but we set it to microsecond for now. mCodecContext->time_base = @@ -500,12 +287,13 @@ MediaResult FFmpegVideoEncoder<LIBAV_VER>::InitInternal() { mCodecContext->framerate = AVRational{.num = static_cast<int>(mConfig.mFramerate), .den = 1}; #endif + #if LIBAVCODEC_VERSION_MAJOR >= 60 mCodecContext->flags |= AV_CODEC_FLAG_FRAME_DURATION; #endif mCodecContext->gop_size = static_cast<int>(mConfig.mKeyframeInterval); - if (mConfig.mUsage == MediaDataEncoder::Usage::Realtime) { + if (mConfig.mUsage == Usage::Realtime) { mLib->av_opt_set(mCodecContext->priv_data, "deadline", "realtime", 0); // Explicitly ask encoder do not keep in flight at any one time for // lookahead purposes. @@ -578,14 +366,11 @@ MediaResult FFmpegVideoEncoder<LIBAV_VER>::InitInternal() { // encoder. mCodecContext->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL; - AVDictionary* options = nullptr; - if (int ret = OpenCodecContext(codec, &options); ret < 0) { - FFMPEGV_LOG("failed to open %s avcodec: %s", codec->name, - MakeErrorString(mLib, ret).get()); - return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, - RESULT_DETAIL("Unable to open avcodec")); + MediaResult rv = FinishInitCommon(codec); + if (NS_FAILED(rv)) { + FFMPEGV_LOG("FFmpeg video encoder initialization failure."); + return rv; } - mLib->av_dict_free(&options); FFMPEGV_LOG("%s has been initialized with format: %s, bitrate: %" PRIi64 ", width: %d, height: %d, time_base: %d/%d%s", @@ -595,74 +380,7 @@ MediaResult FFmpegVideoEncoder<LIBAV_VER>::InitInternal() { mCodecContext->time_base.num, mCodecContext->time_base.den, h264Log.IsEmpty() ? "" : h264Log.get()); - return MediaResult(NS_OK); -} - -void FFmpegVideoEncoder<LIBAV_VER>::ShutdownInternal() { - MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); - - FFMPEGV_LOG("ShutdownInternal"); - - DestroyFrame(); - - if (mCodecContext) { - CloseCodecContext(); - mLib->av_freep(&mCodecContext); - mCodecContext = nullptr; - } -} - -int FFmpegVideoEncoder<LIBAV_VER>::OpenCodecContext(const AVCodec* aCodec, - AVDictionary** aOptions) { - MOZ_ASSERT(mCodecContext); - - StaticMutexAutoLock mon(sMutex); - return mLib->avcodec_open2(mCodecContext, aCodec, aOptions); -} - -void FFmpegVideoEncoder<LIBAV_VER>::CloseCodecContext() { - MOZ_ASSERT(mCodecContext); - - StaticMutexAutoLock mon(sMutex); - mLib->avcodec_close(mCodecContext); -} - -bool FFmpegVideoEncoder<LIBAV_VER>::PrepareFrame() { - MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); - - // TODO: Merge the duplicate part with FFmpegDataDecoder's PrepareFrame. -#if LIBAVCODEC_VERSION_MAJOR >= 55 - if (mFrame) { - mLib->av_frame_unref(mFrame); - } else { - mFrame = mLib->av_frame_alloc(); - } -#elif LIBAVCODEC_VERSION_MAJOR == 54 - if (mFrame) { - mLib->avcodec_get_frame_defaults(mFrame); - } else { - mFrame = mLib->avcodec_alloc_frame(); - } -#else - mLib->av_freep(&mFrame); - mFrame = mLib->avcodec_alloc_frame(); -#endif - return !!mFrame; -} - -void FFmpegVideoEncoder<LIBAV_VER>::DestroyFrame() { - MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); - if (mFrame) { -#if LIBAVCODEC_VERSION_MAJOR >= 55 - mLib->av_frame_unref(mFrame); - mLib->av_frame_free(&mFrame); -#elif LIBAVCODEC_VERSION_MAJOR == 54 - mLib->avcodec_free_frame(&mFrame); -#else - mLib->av_freep(&mFrame); -#endif - mFrame = nullptr; - } + return NS_OK; } bool FFmpegVideoEncoder<LIBAV_VER>::ScaleInputFrame() { @@ -709,71 +427,62 @@ bool FFmpegVideoEncoder<LIBAV_VER>::ScaleInputFrame() { // avcodec_send_frame and avcodec_receive_packet were introduced in version 58. #if LIBAVCODEC_VERSION_MAJOR >= 58 -RefPtr<MediaDataEncoder::EncodePromise> FFmpegVideoEncoder< - LIBAV_VER>::EncodeWithModernAPIs(RefPtr<const VideoData> aSample) { +Result<MediaDataEncoder::EncodedData, nsresult> FFmpegVideoEncoder< + LIBAV_VER>::EncodeInputWithModernAPIs(RefPtr<const MediaData> aSample) { MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); MOZ_ASSERT(mCodecContext); MOZ_ASSERT(aSample); + RefPtr<const VideoData> sample(aSample->As<VideoData>()); + // Validate input. - if (!aSample->mImage) { + if (!sample->mImage) { FFMPEGV_LOG("No image"); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_ILLEGAL_INPUT, - RESULT_DETAIL("No image in sample")), - __func__); - } else if (aSample->mImage->GetSize().IsEmpty()) { + return Result<MediaDataEncoder::EncodedData, nsresult>( + NS_ERROR_DOM_MEDIA_FATAL_ERR); + } + if (sample->mImage->GetSize().IsEmpty()) { FFMPEGV_LOG("image width or height is invalid"); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_ILLEGAL_INPUT, - RESULT_DETAIL("Invalid image size")), - __func__); + return Result<MediaDataEncoder::EncodedData, nsresult>( + NS_ERROR_DOM_MEDIA_FATAL_ERR); } // Allocate AVFrame. if (!PrepareFrame()) { FFMPEGV_LOG("failed to allocate frame"); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("Unable to allocate frame")), - __func__); + return Result<MediaDataEncoder::EncodedData, nsresult>( + NS_ERROR_DOM_MEDIA_FATAL_ERR); } // Set AVFrame properties for its internal data allocation. For now, we always // convert into ffmpeg's buffer. mFrame->format = ffmpeg::FFMPEG_PIX_FMT_YUV420P; - mFrame->width = static_cast<int>(aSample->mImage->GetSize().width); - mFrame->height = static_cast<int>(aSample->mImage->GetSize().height); + mFrame->width = static_cast<int>(sample->mImage->GetSize().width); + mFrame->height = static_cast<int>(sample->mImage->GetSize().height); // Allocate AVFrame data. if (int ret = mLib->av_frame_get_buffer(mFrame, 0); ret < 0) { FFMPEGV_LOG("failed to allocate frame data: %s", MakeErrorString(mLib, ret).get()); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("Unable to allocate frame data")), - __func__); + return Result<MediaDataEncoder::EncodedData, nsresult>( + NS_ERROR_DOM_MEDIA_FATAL_ERR); } // Make sure AVFrame is writable. if (int ret = mLib->av_frame_make_writable(mFrame); ret < 0) { FFMPEGV_LOG("failed to make frame writable: %s", MakeErrorString(mLib, ret).get()); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_NOT_AVAILABLE, - RESULT_DETAIL("Unable to make frame writable")), - __func__); + return Result<MediaDataEncoder::EncodedData, nsresult>( + NS_ERROR_DOM_MEDIA_FATAL_ERR); } nsresult rv = ConvertToI420( - aSample->mImage, mFrame->data[0], mFrame->linesize[0], mFrame->data[1], + sample->mImage, mFrame->data[0], mFrame->linesize[0], mFrame->data[1], mFrame->linesize[1], mFrame->data[2], mFrame->linesize[2]); if (NS_FAILED(rv)) { FFMPEGV_LOG("Conversion error!"); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_ILLEGAL_INPUT, - RESULT_DETAIL("libyuv conversion error")), - __func__); + return Result<MediaDataEncoder::EncodedData, nsresult>( + NS_ERROR_DOM_MEDIA_FATAL_ERR); } // Scale the YUV input frame if needed -- the encoded frame will have the @@ -781,10 +490,8 @@ RefPtr<MediaDataEncoder::EncodePromise> FFmpegVideoEncoder< if (mFrame->width != mConfig.mSize.Width() || mFrame->height != mConfig.mSize.Height()) { if (!ScaleInputFrame()) { - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("libyuv scaling error")), - __func__); + return Result<MediaDataEncoder::EncodedData, nsresult>( + NS_ERROR_DOM_MEDIA_FATAL_ERR); } } @@ -805,193 +512,17 @@ RefPtr<MediaDataEncoder::EncodePromise> FFmpegVideoEncoder< # endif mFrame->pkt_duration = aSample->mDuration.ToMicroseconds(); - // Initialize AVPacket. - AVPacket* pkt = mLib->av_packet_alloc(); - - if (!pkt) { - FFMPEGV_LOG("failed to allocate packet"); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("Unable to allocate packet")), - __func__); - } - - auto freePacket = MakeScopeExit([this, &pkt] { mLib->av_packet_free(&pkt); }); - - // Send frame and receive packets. - - if (int ret = mLib->avcodec_send_frame(mCodecContext, mFrame); ret < 0) { - // In theory, avcodec_send_frame could sent -EAGAIN to signal its internal - // buffers is full. In practice this can't happen as we only feed one frame - // at a time, and we immediately call avcodec_receive_packet right after. - // TODO: Create a NS_ERROR_DOM_MEDIA_ENCODE_ERR in ErrorList.py? - FFMPEGV_LOG("avcodec_send_frame error: %s", - MakeErrorString(mLib, ret).get()); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, - RESULT_DETAIL("avcodec_send_frame error")), - __func__); - } - - EncodedData output; - while (true) { - int ret = mLib->avcodec_receive_packet(mCodecContext, pkt); - if (ret == AVERROR(EAGAIN)) { - // The encoder is asking for more inputs. - FFMPEGV_LOG("encoder is asking for more input!"); - break; - } - - if (ret < 0) { - // AVERROR_EOF is returned when the encoder has been fully flushed, but it - // shouldn't happen here. - FFMPEGV_LOG("avcodec_receive_packet error: %s", - MakeErrorString(mLib, ret).get()); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, - RESULT_DETAIL("avcodec_receive_packet error")), - __func__); - } - - RefPtr<MediaRawData> d = ToMediaRawData(pkt); - mLib->av_packet_unref(pkt); - if (!d) { - FFMPEGV_LOG("failed to create a MediaRawData from the AVPacket"); - return EncodePromise::CreateAndReject( - MediaResult( - NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("Unable to get MediaRawData from AVPacket")), - __func__); - } - output.AppendElement(std::move(d)); - } - - FFMPEGV_LOG("get %zu encoded data", output.Length()); - return EncodePromise::CreateAndResolve(std::move(output), __func__); + // Now send the AVFrame to ffmpeg for encoding, same code for audio and video. + return FFmpegDataEncoder<LIBAV_VER>::EncodeWithModernAPIs(); } - -RefPtr<MediaDataEncoder::EncodePromise> -FFmpegVideoEncoder<LIBAV_VER>::DrainWithModernAPIs() { - MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); - MOZ_ASSERT(mCodecContext); - - // TODO: Create a Result<EncodedData, nsresult> EncodeWithModernAPIs(AVFrame - // *aFrame) to merge the duplicate code below with EncodeWithModernAPIs above. - - // Initialize AVPacket. - AVPacket* pkt = mLib->av_packet_alloc(); - if (!pkt) { - FFMPEGV_LOG("failed to allocate packet"); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("Unable to allocate packet")), - __func__); - } - auto freePacket = MakeScopeExit([this, &pkt] { mLib->av_packet_free(&pkt); }); - - // Enter draining mode by sending NULL to the avcodec_send_frame(). Note that - // this can leave the encoder in a permanent EOF state after draining. As a - // result, the encoder is unable to continue encoding. A new - // AVCodecContext/encoder creation is required if users need to encode after - // draining. - // - // TODO: Use `avcodec_flush_buffers` to drain the pending packets if - // AV_CODEC_CAP_ENCODER_FLUSH is set in mCodecContext->codec->capabilities. - if (int ret = mLib->avcodec_send_frame(mCodecContext, nullptr); ret < 0) { - if (ret == AVERROR_EOF) { - // The encoder has been flushed. Drain can be called multiple time. - FFMPEGV_LOG("encoder has been flushed!"); - return EncodePromise::CreateAndResolve(EncodedData(), __func__); - } - - FFMPEGV_LOG("avcodec_send_frame error: %s", - MakeErrorString(mLib, ret).get()); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, - RESULT_DETAIL("avcodec_send_frame error")), - __func__); - } - - EncodedData output; - while (true) { - int ret = mLib->avcodec_receive_packet(mCodecContext, pkt); - if (ret == AVERROR_EOF) { - FFMPEGV_LOG("encoder has no more output packet!"); - break; - } - - if (ret < 0) { - // avcodec_receive_packet should not result in a -EAGAIN once it's in - // draining mode. - FFMPEGV_LOG("avcodec_receive_packet error: %s", - MakeErrorString(mLib, ret).get()); - return EncodePromise::CreateAndReject( - MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, - RESULT_DETAIL("avcodec_receive_packet error")), - __func__); - } - - RefPtr<MediaRawData> d = ToMediaRawData(pkt); - mLib->av_packet_unref(pkt); - if (!d) { - FFMPEGV_LOG("failed to create a MediaRawData from the AVPacket"); - return EncodePromise::CreateAndReject( - MediaResult( - NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("Unable to get MediaRawData from AVPacket")), - __func__); - } - output.AppendElement(std::move(d)); - } - - FFMPEGV_LOG("get %zu encoded data", output.Length()); - - // TODO: Evaluate a better solution (Bug 1869466) - // TODO: Only re-create AVCodecContext when avcodec_flush_buffers is - // unavailable. - ShutdownInternal(); - MediaResult r = InitInternal(); - return NS_FAILED(r) - ? EncodePromise::CreateAndReject(r, __func__) - : EncodePromise::CreateAndResolve(std::move(output), __func__); -} -#endif +#endif // if LIBAVCODEC_VERSION_MAJOR >= 58 RefPtr<MediaRawData> FFmpegVideoEncoder<LIBAV_VER>::ToMediaRawData( AVPacket* aPacket) { MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); MOZ_ASSERT(aPacket); - // TODO: Do we need to check AV_PKT_FLAG_CORRUPT? - - // Copy frame data from AVPacket. - auto data = MakeRefPtr<MediaRawData>(); - UniquePtr<MediaRawDataWriter> writer(data->CreateWriter()); - if (!writer->Append(aPacket->data, static_cast<size_t>(aPacket->size))) { - FFMPEGV_LOG("fail to allocate MediaRawData buffer"); - return nullptr; // OOM - } - - data->mKeyframe = (aPacket->flags & AV_PKT_FLAG_KEY) != 0; - // TODO(bug 1869560): The unit of pts, dts, and duration is time_base, which - // is recommended to be the reciprocal of the frame rate, but we set it to - // microsecond for now. - data->mTime = media::TimeUnit::FromMicroseconds(aPacket->pts); -#if LIBAVCODEC_VERSION_MAJOR >= 60 - data->mDuration = media::TimeUnit::FromMicroseconds(aPacket->duration); -#else - int64_t duration; - if (mDurationMap.Find(aPacket->pts, duration)) { - data->mDuration = media::TimeUnit::FromMicroseconds(duration); - } else { - data->mDuration = media::TimeUnit::FromMicroseconds(aPacket->duration); - } -#endif - data->mTimecode = media::TimeUnit::FromMicroseconds(aPacket->dts); - - if (auto r = GetExtraData(aPacket); r.isOk()) { - data->mExtraData = r.unwrap(); - } + RefPtr<MediaRawData> data = ToMediaRawDataCommon(aPacket); // TODO: Is it possible to retrieve temporal layer id from underlying codec // instead? diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h index 07c433ddd7..0ee5f52aec 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h +++ b/dom/media/platforms/ffmpeg/FFmpegVideoEncoder.h @@ -7,10 +7,10 @@ #ifndef DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGVIDEOENCODER_H_ #define DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGVIDEOENCODER_H_ +#include "FFmpegDataEncoder.h" #include "FFmpegLibWrapper.h" #include "PlatformEncoderModule.h" #include "SimpleMap.h" -#include "mozilla/ThreadSafety.h" // This must be the last header included #include "FFmpegLibs.h" @@ -18,17 +18,10 @@ namespace mozilla { template <int V> -AVCodecID GetFFmpegEncoderCodecId(CodecType aCodec); - -template <> -AVCodecID GetFFmpegEncoderCodecId<LIBAV_VER>(CodecType aCodec); - -template <int V> class FFmpegVideoEncoder : public MediaDataEncoder {}; -// TODO: Bug 1860925: FFmpegDataEncoder template <> -class FFmpegVideoEncoder<LIBAV_VER> final : public MediaDataEncoder { +class FFmpegVideoEncoder<LIBAV_VER> : public FFmpegDataEncoder<LIBAV_VER> { using DurationMap = SimpleMap<int64_t>; public: @@ -36,44 +29,19 @@ class FFmpegVideoEncoder<LIBAV_VER> final : public MediaDataEncoder { const RefPtr<TaskQueue>& aTaskQueue, const EncoderConfig& aConfig); - /* MediaDataEncoder Methods */ - // All methods run on the task queue, except for GetDescriptionName. - RefPtr<InitPromise> Init() override; - RefPtr<EncodePromise> Encode(const MediaData* aSample) override; - RefPtr<ReconfigurationPromise> Reconfigure( - const RefPtr<const EncoderConfigurationChangeList>& aConfigurationChanges) - override; - RefPtr<EncodePromise> Drain() override; - RefPtr<ShutdownPromise> Shutdown() override; - RefPtr<GenericPromise> SetBitrate(uint32_t aBitRate) override; nsCString GetDescriptionName() const override; - private: - ~FFmpegVideoEncoder() = default; - + protected: // Methods only called on mTaskQueue. - RefPtr<InitPromise> ProcessInit(); - RefPtr<EncodePromise> ProcessEncode(RefPtr<const MediaData> aSample); - RefPtr<ReconfigurationPromise> ProcessReconfigure( - const RefPtr<const EncoderConfigurationChangeList> aConfigurationChanges); - RefPtr<EncodePromise> ProcessDrain(); - RefPtr<ShutdownPromise> ProcessShutdown(); - MediaResult InitInternal(); - void ShutdownInternal(); - // TODO: Share these with FFmpegDataDecoder. - int OpenCodecContext(const AVCodec* aCodec, AVDictionary** aOptions) - MOZ_EXCLUDES(sMutex); - void CloseCodecContext() MOZ_EXCLUDES(sMutex); - bool PrepareFrame(); - void DestroyFrame(); - bool ScaleInputFrame(); + virtual nsresult InitSpecific() override; #if LIBAVCODEC_VERSION_MAJOR >= 58 - RefPtr<EncodePromise> EncodeWithModernAPIs(RefPtr<const VideoData> aSample); - RefPtr<EncodePromise> DrainWithModernAPIs(); + Result<EncodedData, nsresult> EncodeInputWithModernAPIs( + RefPtr<const MediaData> aSample) override; #endif - RefPtr<MediaRawData> ToMediaRawData(AVPacket* aPacket); + bool ScaleInputFrame(); + virtual RefPtr<MediaRawData> ToMediaRawData(AVPacket* aPacket) override; Result<already_AddRefed<MediaByteBuffer>, nsresult> GetExtraData( - AVPacket* aPacket); + AVPacket* aPacket) override; void ForceEnablingFFmpegDebugLogs(); struct SVCSettings { nsTArray<uint8_t> mTemporalLayerIds; @@ -88,21 +56,6 @@ class FFmpegVideoEncoder<LIBAV_VER> final : public MediaDataEncoder { nsTArray<std::pair<nsCString, nsCString>> mSettingKeyValuePairs; }; H264Settings GetH264Settings(const H264Specific& aH264Specific); - - // This refers to a static FFmpegLibWrapper, so raw pointer is adequate. - const FFmpegLibWrapper* mLib; - const AVCodecID mCodecID; - const RefPtr<TaskQueue> mTaskQueue; - - // set in constructor, modified when parameters change - EncoderConfig mConfig; - - // mTaskQueue only. - nsCString mCodecName; - AVCodecContext* mCodecContext; - AVFrame* mFrame; - DurationMap mDurationMap; - struct SVCInfo { explicit SVCInfo(nsTArray<uint8_t>&& aTemporalLayerIds) : mTemporalLayerIds(std::move(aTemporalLayerIds)), mNextIndex(0) {} @@ -111,13 +64,9 @@ class FFmpegVideoEncoder<LIBAV_VER> final : public MediaDataEncoder { // Return the current temporal layer id and update the next. uint8_t UpdateTemporalLayerId(); }; - Maybe<SVCInfo> mSVCInfo; - - // Provide critical-section for open/close mCodecContext. - // TODO: Merge this with FFmpegDataDecoder's one. - static StaticMutex sMutex; + Maybe<SVCInfo> mSVCInfo{}; }; } // namespace mozilla -#endif /* DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGVIDEOENCODER_H_ */ +#endif // DOM_MEDIA_PLATFORMS_FFMPEG_FFMPEGVIDEOENCODER_H_ diff --git a/dom/media/platforms/ffmpeg/ffmpeg57/moz.build b/dom/media/platforms/ffmpeg/ffmpeg57/moz.build index f26edcdc7f..db48b36f6b 100644 --- a/dom/media/platforms/ffmpeg/ffmpeg57/moz.build +++ b/dom/media/platforms/ffmpeg/ffmpeg57/moz.build @@ -6,7 +6,9 @@ UNIFIED_SOURCES += [ '../FFmpegAudioDecoder.cpp', + '../FFmpegAudioEncoder.cpp', '../FFmpegDataDecoder.cpp', + "../FFmpegDataEncoder.cpp", '../FFmpegDecoderModule.cpp', '../FFmpegEncoderModule.cpp', '../FFmpegVideoDecoder.cpp', diff --git a/dom/media/platforms/ffmpeg/ffmpeg58/moz.build b/dom/media/platforms/ffmpeg/ffmpeg58/moz.build index a22bf98abd..12e48c44f0 100644 --- a/dom/media/platforms/ffmpeg/ffmpeg58/moz.build +++ b/dom/media/platforms/ffmpeg/ffmpeg58/moz.build @@ -6,7 +6,9 @@ UNIFIED_SOURCES += [ '../FFmpegAudioDecoder.cpp', + '../FFmpegAudioEncoder.cpp', '../FFmpegDataDecoder.cpp', + "../FFmpegDataEncoder.cpp", '../FFmpegDecoderModule.cpp', '../FFmpegEncoderModule.cpp', '../FFmpegVideoDecoder.cpp', diff --git a/dom/media/platforms/ffmpeg/ffmpeg59/moz.build b/dom/media/platforms/ffmpeg/ffmpeg59/moz.build index e0c6c10ecd..c4f7b89951 100644 --- a/dom/media/platforms/ffmpeg/ffmpeg59/moz.build +++ b/dom/media/platforms/ffmpeg/ffmpeg59/moz.build @@ -6,7 +6,9 @@ UNIFIED_SOURCES += [ "../FFmpegAudioDecoder.cpp", + '../FFmpegAudioEncoder.cpp', "../FFmpegDataDecoder.cpp", + "../FFmpegDataEncoder.cpp", "../FFmpegDecoderModule.cpp", "../FFmpegEncoderModule.cpp", "../FFmpegVideoDecoder.cpp", diff --git a/dom/media/platforms/ffmpeg/ffmpeg60/moz.build b/dom/media/platforms/ffmpeg/ffmpeg60/moz.build index e0c6c10ecd..c4f7b89951 100644 --- a/dom/media/platforms/ffmpeg/ffmpeg60/moz.build +++ b/dom/media/platforms/ffmpeg/ffmpeg60/moz.build @@ -6,7 +6,9 @@ UNIFIED_SOURCES += [ "../FFmpegAudioDecoder.cpp", + '../FFmpegAudioEncoder.cpp', "../FFmpegDataDecoder.cpp", + "../FFmpegDataEncoder.cpp", "../FFmpegDecoderModule.cpp", "../FFmpegEncoderModule.cpp", "../FFmpegVideoDecoder.cpp", diff --git a/dom/media/platforms/ffmpeg/ffvpx/moz.build b/dom/media/platforms/ffmpeg/ffvpx/moz.build index 97a224b08b..bc72b6d1a7 100644 --- a/dom/media/platforms/ffmpeg/ffvpx/moz.build +++ b/dom/media/platforms/ffmpeg/ffvpx/moz.build @@ -11,9 +11,12 @@ EXPORTS += [ UNIFIED_SOURCES += [ "../FFmpegAudioDecoder.cpp", + "../FFmpegAudioEncoder.cpp", "../FFmpegDataDecoder.cpp", + "../FFmpegDataEncoder.cpp", "../FFmpegDecoderModule.cpp", "../FFmpegEncoderModule.cpp", + "../FFmpegUtils.cpp", "../FFmpegVideoDecoder.cpp", "../FFmpegVideoEncoder.cpp", ] diff --git a/dom/media/platforms/ffmpeg/libav53/moz.build b/dom/media/platforms/ffmpeg/libav53/moz.build index 06b226e1f1..81b8b8dcc6 100644 --- a/dom/media/platforms/ffmpeg/libav53/moz.build +++ b/dom/media/platforms/ffmpeg/libav53/moz.build @@ -6,7 +6,9 @@ UNIFIED_SOURCES += [ '../FFmpegAudioDecoder.cpp', + '../FFmpegAudioEncoder.cpp', '../FFmpegDataDecoder.cpp', + "../FFmpegDataEncoder.cpp", '../FFmpegDecoderModule.cpp', '../FFmpegEncoderModule.cpp', '../FFmpegVideoDecoder.cpp', diff --git a/dom/media/platforms/ffmpeg/libav54/moz.build b/dom/media/platforms/ffmpeg/libav54/moz.build index 06b226e1f1..81b8b8dcc6 100644 --- a/dom/media/platforms/ffmpeg/libav54/moz.build +++ b/dom/media/platforms/ffmpeg/libav54/moz.build @@ -6,7 +6,9 @@ UNIFIED_SOURCES += [ '../FFmpegAudioDecoder.cpp', + '../FFmpegAudioEncoder.cpp', '../FFmpegDataDecoder.cpp', + "../FFmpegDataEncoder.cpp", '../FFmpegDecoderModule.cpp', '../FFmpegEncoderModule.cpp', '../FFmpegVideoDecoder.cpp', diff --git a/dom/media/platforms/ffmpeg/libav55/moz.build b/dom/media/platforms/ffmpeg/libav55/moz.build index af2d4f1831..2c3d89b9b3 100644 --- a/dom/media/platforms/ffmpeg/libav55/moz.build +++ b/dom/media/platforms/ffmpeg/libav55/moz.build @@ -6,7 +6,9 @@ UNIFIED_SOURCES += [ '../FFmpegAudioDecoder.cpp', + '../FFmpegAudioEncoder.cpp', '../FFmpegDataDecoder.cpp', + "../FFmpegDataEncoder.cpp", '../FFmpegDecoderModule.cpp', '../FFmpegEncoderModule.cpp', '../FFmpegVideoDecoder.cpp', diff --git a/dom/media/platforms/ffmpeg/moz.build b/dom/media/platforms/ffmpeg/moz.build index f519b30cec..ac78eee289 100644 --- a/dom/media/platforms/ffmpeg/moz.build +++ b/dom/media/platforms/ffmpeg/moz.build @@ -18,9 +18,7 @@ DIRS += [ "ffmpeg60", ] -UNIFIED_SOURCES += [ - "FFmpegRuntimeLinker.cpp", -] +UNIFIED_SOURCES += ["FFmpegRuntimeLinker.cpp"] if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk": include("/ipc/chromium/chromium-config.mozbuild") |