diff options
Diffstat (limited to 'third_party/libwebrtc/modules/video_coding/codecs/h264')
14 files changed, 2445 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/DEPS b/third_party/libwebrtc/modules/video_coding/codecs/h264/DEPS new file mode 100644 index 0000000000..4e110917d8 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/DEPS @@ -0,0 +1,5 @@ +include_rules = [ + "+third_party/ffmpeg", + "+third_party/openh264", + "+media/base", +] diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/OWNERS b/third_party/libwebrtc/modules/video_coding/codecs/h264/OWNERS new file mode 100644 index 0000000000..4b06c4e32b --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/OWNERS @@ -0,0 +1,2 @@ +sprang@webrtc.org +ssilkin@webrtc.org diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264.cc b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264.cc new file mode 100644 index 0000000000..23580d7a4a --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264.cc @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "modules/video_coding/codecs/h264/include/h264.h" + +#include <memory> +#include <string> + +#include "absl/container/inlined_vector.h" +#include "absl/types/optional.h" +#include "api/video_codecs/sdp_video_format.h" +#include "media/base/media_constants.h" +#include "rtc_base/trace_event.h" + +#if defined(WEBRTC_USE_H264) +#include "modules/video_coding/codecs/h264/h264_decoder_impl.h" +#include "modules/video_coding/codecs/h264/h264_encoder_impl.h" +#endif + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +namespace { + +#if defined(WEBRTC_USE_H264) +bool g_rtc_use_h264 = true; +#endif + +// If H.264 OpenH264/FFmpeg codec is supported. +bool IsH264CodecSupported() { +#if defined(WEBRTC_USE_H264) + return g_rtc_use_h264; +#else + return false; +#endif +} + +constexpr ScalabilityMode kSupportedScalabilityModes[] = { + ScalabilityMode::kL1T1, ScalabilityMode::kL1T2, ScalabilityMode::kL1T3}; + +} // namespace + +SdpVideoFormat CreateH264Format(H264Profile profile, + H264Level level, + const std::string& packetization_mode, + bool add_scalability_modes) { + const absl::optional<std::string> profile_string = + H264ProfileLevelIdToString(H264ProfileLevelId(profile, level)); + RTC_CHECK(profile_string); + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> scalability_modes; + if (add_scalability_modes) { + for (const auto scalability_mode : kSupportedScalabilityModes) { + scalability_modes.push_back(scalability_mode); + } + } + return SdpVideoFormat( + cricket::kH264CodecName, + {{cricket::kH264FmtpProfileLevelId, *profile_string}, + {cricket::kH264FmtpLevelAsymmetryAllowed, "1"}, + {cricket::kH264FmtpPacketizationMode, packetization_mode}}, + scalability_modes); +} + +void DisableRtcUseH264() { +#if defined(WEBRTC_USE_H264) + g_rtc_use_h264 = false; +#endif +} + +std::vector<SdpVideoFormat> SupportedH264Codecs(bool add_scalability_modes) { + TRACE_EVENT0("webrtc", __func__); + if (!IsH264CodecSupported()) + return std::vector<SdpVideoFormat>(); + // We only support encoding Constrained Baseline Profile (CBP), but the + // decoder supports more profiles. We can list all profiles here that are + // supported by the decoder and that are also supersets of CBP, i.e. the + // decoder for that profile is required to be able to decode CBP. This means + // we can encode and send CBP even though we negotiated a potentially + // higher profile. See the H264 spec for more information. + // + // We support both packetization modes 0 (mandatory) and 1 (optional, + // preferred). + return {CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, + "1", add_scalability_modes), + CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, + "0", add_scalability_modes), + CreateH264Format(H264Profile::kProfileConstrainedBaseline, + H264Level::kLevel3_1, "1", add_scalability_modes), + CreateH264Format(H264Profile::kProfileConstrainedBaseline, + H264Level::kLevel3_1, "0", add_scalability_modes), + CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "1", + add_scalability_modes), + CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "0", + add_scalability_modes)}; +} + +std::vector<SdpVideoFormat> SupportedH264DecoderCodecs() { + TRACE_EVENT0("webrtc", __func__); + if (!IsH264CodecSupported()) + return std::vector<SdpVideoFormat>(); + + std::vector<SdpVideoFormat> supportedCodecs = SupportedH264Codecs(); + + // OpenH264 doesn't yet support High Predictive 4:4:4 encoding but it does + // support decoding. + supportedCodecs.push_back(CreateH264Format( + H264Profile::kProfilePredictiveHigh444, H264Level::kLevel3_1, "1")); + supportedCodecs.push_back(CreateH264Format( + H264Profile::kProfilePredictiveHigh444, H264Level::kLevel3_1, "0")); + + return supportedCodecs; +} + +std::unique_ptr<H264Encoder> H264Encoder::Create( + const cricket::VideoCodec& codec) { + RTC_DCHECK(H264Encoder::IsSupported()); +#if defined(WEBRTC_USE_H264) + RTC_CHECK(g_rtc_use_h264); + RTC_LOG(LS_INFO) << "Creating H264EncoderImpl."; + return std::make_unique<H264EncoderImpl>(codec); +#else + RTC_DCHECK_NOTREACHED(); + return nullptr; +#endif +} + +bool H264Encoder::IsSupported() { + return IsH264CodecSupported(); +} + +bool H264Encoder::SupportsScalabilityMode(ScalabilityMode scalability_mode) { + for (const auto& entry : kSupportedScalabilityModes) { + if (entry == scalability_mode) { + return true; + } + } + return false; +} + +std::unique_ptr<H264Decoder> H264Decoder::Create() { + RTC_DCHECK(H264Decoder::IsSupported()); +#if defined(WEBRTC_USE_H264) + RTC_CHECK(g_rtc_use_h264); + RTC_LOG(LS_INFO) << "Creating H264DecoderImpl."; + return std::make_unique<H264DecoderImpl>(); +#else + RTC_DCHECK_NOTREACHED(); + return nullptr; +#endif +} + +bool H264Decoder::IsSupported() { + return IsH264CodecSupported(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_color_space.cc b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_color_space.cc new file mode 100644 index 0000000000..59921263e3 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_color_space.cc @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Everything declared/defined in this header is only required when WebRTC is +// build with H264 support, please do not move anything out of the +// #ifdef unless needed and tested. +#ifdef WEBRTC_USE_H264 + +#include "modules/video_coding/codecs/h264/h264_color_space.h" + +namespace webrtc { + +ColorSpace ExtractH264ColorSpace(AVCodecContext* codec) { + ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kUnspecified; + switch (codec->color_primaries) { + case AVCOL_PRI_BT709: + primaries = ColorSpace::PrimaryID::kBT709; + break; + case AVCOL_PRI_BT470M: + primaries = ColorSpace::PrimaryID::kBT470M; + break; + case AVCOL_PRI_BT470BG: + primaries = ColorSpace::PrimaryID::kBT470BG; + break; + case AVCOL_PRI_SMPTE170M: + primaries = ColorSpace::PrimaryID::kSMPTE170M; + break; + case AVCOL_PRI_SMPTE240M: + primaries = ColorSpace::PrimaryID::kSMPTE240M; + break; + case AVCOL_PRI_FILM: + primaries = ColorSpace::PrimaryID::kFILM; + break; + case AVCOL_PRI_BT2020: + primaries = ColorSpace::PrimaryID::kBT2020; + break; + case AVCOL_PRI_SMPTE428: + primaries = ColorSpace::PrimaryID::kSMPTEST428; + break; + case AVCOL_PRI_SMPTE431: + primaries = ColorSpace::PrimaryID::kSMPTEST431; + break; + case AVCOL_PRI_SMPTE432: + primaries = ColorSpace::PrimaryID::kSMPTEST432; + break; + case AVCOL_PRI_JEDEC_P22: + primaries = ColorSpace::PrimaryID::kJEDECP22; + break; + case AVCOL_PRI_RESERVED0: + case AVCOL_PRI_UNSPECIFIED: + case AVCOL_PRI_RESERVED: + default: + break; + } + + ColorSpace::TransferID transfer = ColorSpace::TransferID::kUnspecified; + switch (codec->color_trc) { + case AVCOL_TRC_BT709: + transfer = ColorSpace::TransferID::kBT709; + break; + case AVCOL_TRC_GAMMA22: + transfer = ColorSpace::TransferID::kGAMMA22; + break; + case AVCOL_TRC_GAMMA28: + transfer = ColorSpace::TransferID::kGAMMA28; + break; + case AVCOL_TRC_SMPTE170M: + transfer = ColorSpace::TransferID::kSMPTE170M; + break; + case AVCOL_TRC_SMPTE240M: + transfer = ColorSpace::TransferID::kSMPTE240M; + break; + case AVCOL_TRC_LINEAR: + transfer = ColorSpace::TransferID::kLINEAR; + break; + case AVCOL_TRC_LOG: + transfer = ColorSpace::TransferID::kLOG; + break; + case AVCOL_TRC_LOG_SQRT: + transfer = ColorSpace::TransferID::kLOG_SQRT; + break; + case AVCOL_TRC_IEC61966_2_4: + transfer = ColorSpace::TransferID::kIEC61966_2_4; + break; + case AVCOL_TRC_BT1361_ECG: + transfer = ColorSpace::TransferID::kBT1361_ECG; + break; + case AVCOL_TRC_IEC61966_2_1: + transfer = ColorSpace::TransferID::kIEC61966_2_1; + break; + case AVCOL_TRC_BT2020_10: + transfer = ColorSpace::TransferID::kBT2020_10; + break; + case AVCOL_TRC_BT2020_12: + transfer = ColorSpace::TransferID::kBT2020_12; + break; + case AVCOL_TRC_SMPTE2084: + transfer = ColorSpace::TransferID::kSMPTEST2084; + break; + case AVCOL_TRC_SMPTE428: + transfer = ColorSpace::TransferID::kSMPTEST428; + break; + case AVCOL_TRC_ARIB_STD_B67: + transfer = ColorSpace::TransferID::kARIB_STD_B67; + break; + case AVCOL_TRC_RESERVED0: + case AVCOL_TRC_UNSPECIFIED: + case AVCOL_TRC_RESERVED: + default: + break; + } + + ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kUnspecified; + switch (codec->colorspace) { + case AVCOL_SPC_RGB: + matrix = ColorSpace::MatrixID::kRGB; + break; + case AVCOL_SPC_BT709: + matrix = ColorSpace::MatrixID::kBT709; + break; + case AVCOL_SPC_FCC: + matrix = ColorSpace::MatrixID::kFCC; + break; + case AVCOL_SPC_BT470BG: + matrix = ColorSpace::MatrixID::kBT470BG; + break; + case AVCOL_SPC_SMPTE170M: + matrix = ColorSpace::MatrixID::kSMPTE170M; + break; + case AVCOL_SPC_SMPTE240M: + matrix = ColorSpace::MatrixID::kSMPTE240M; + break; + case AVCOL_SPC_YCGCO: + matrix = ColorSpace::MatrixID::kYCOCG; + break; + case AVCOL_SPC_BT2020_NCL: + matrix = ColorSpace::MatrixID::kBT2020_NCL; + break; + case AVCOL_SPC_BT2020_CL: + matrix = ColorSpace::MatrixID::kBT2020_CL; + break; + case AVCOL_SPC_SMPTE2085: + matrix = ColorSpace::MatrixID::kSMPTE2085; + break; + case AVCOL_SPC_CHROMA_DERIVED_NCL: + case AVCOL_SPC_CHROMA_DERIVED_CL: + case AVCOL_SPC_ICTCP: + case AVCOL_SPC_UNSPECIFIED: + case AVCOL_SPC_RESERVED: + default: + break; + } + + ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid; + switch (codec->color_range) { + case AVCOL_RANGE_MPEG: + range = ColorSpace::RangeID::kLimited; + break; + case AVCOL_RANGE_JPEG: + range = ColorSpace::RangeID::kFull; + break; + case AVCOL_RANGE_UNSPECIFIED: + default: + break; + } + return ColorSpace(primaries, transfer, matrix, range); +} + +} // namespace webrtc + +#endif // WEBRTC_USE_H264 diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_color_space.h b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_color_space.h new file mode 100644 index 0000000000..392ccaf563 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_color_space.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_ +#define MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_ + +// Everything declared in this header is only required when WebRTC is +// build with H264 support, please do not move anything out of the +// #ifdef unless needed and tested. +#ifdef WEBRTC_USE_H264 + +#if defined(WEBRTC_WIN) && !defined(__clang__) +#error "See: bugs.webrtc.org/9213#c13." +#endif + +#include "api/video/color_space.h" + +extern "C" { +#include "third_party/ffmpeg/libavcodec/avcodec.h" +} // extern "C" + +namespace webrtc { + +// Helper class for extracting color space information from H264 stream. +ColorSpace ExtractH264ColorSpace(AVCodecContext* codec); + +} // namespace webrtc + +#endif // WEBRTC_USE_H264 + +#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc new file mode 100644 index 0000000000..f67718cb23 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc @@ -0,0 +1,657 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +// Everything declared/defined in this header is only required when WebRTC is +// build with H264 support, please do not move anything out of the +// #ifdef unless needed and tested. +#ifdef WEBRTC_USE_H264 + +#include "modules/video_coding/codecs/h264/h264_decoder_impl.h" + +#include <algorithm> +#include <limits> +#include <memory> + +extern "C" { +#include "third_party/ffmpeg/libavcodec/avcodec.h" +#include "third_party/ffmpeg/libavformat/avformat.h" +#include "third_party/ffmpeg/libavutil/imgutils.h" +} // extern "C" + +#include "api/video/color_space.h" +#include "api/video/i010_buffer.h" +#include "api/video/i420_buffer.h" +#include "common_video/include/video_frame_buffer.h" +#include "modules/video_coding/codecs/h264/h264_color_space.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +constexpr std::array<AVPixelFormat, 9> kPixelFormatsSupported = { + AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P, + AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P, + AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_YUV422P10LE, AV_PIX_FMT_YUV444P10LE}; +const size_t kYPlaneIndex = 0; +const size_t kUPlaneIndex = 1; +const size_t kVPlaneIndex = 2; + +// Used by histograms. Values of entries should not be changed. +enum H264DecoderImplEvent { + kH264DecoderEventInit = 0, + kH264DecoderEventError = 1, + kH264DecoderEventMax = 16, +}; + +struct ScopedPtrAVFreePacket { + void operator()(AVPacket* packet) { av_packet_free(&packet); } +}; +typedef std::unique_ptr<AVPacket, ScopedPtrAVFreePacket> ScopedAVPacket; + +ScopedAVPacket MakeScopedAVPacket() { + ScopedAVPacket packet(av_packet_alloc()); + return packet; +} + +} // namespace + +int H264DecoderImpl::AVGetBuffer2(AVCodecContext* context, + AVFrame* av_frame, + int flags) { + // Set in `Configure`. + H264DecoderImpl* decoder = static_cast<H264DecoderImpl*>(context->opaque); + // DCHECK values set in `Configure`. + RTC_DCHECK(decoder); + // Necessary capability to be allowed to provide our own buffers. + RTC_DCHECK(context->codec->capabilities | AV_CODEC_CAP_DR1); + + auto pixelFormatSupported = std::find_if( + kPixelFormatsSupported.begin(), kPixelFormatsSupported.end(), + [context](AVPixelFormat format) { return context->pix_fmt == format; }); + + RTC_CHECK(pixelFormatSupported != kPixelFormatsSupported.end()); + + // `av_frame->width` and `av_frame->height` are set by FFmpeg. These are the + // actual image's dimensions and may be different from `context->width` and + // `context->coded_width` due to reordering. + int width = av_frame->width; + int height = av_frame->height; + // See `lowres`, if used the decoder scales the image by 1/2^(lowres). This + // has implications on which resolutions are valid, but we don't use it. + RTC_CHECK_EQ(context->lowres, 0); + // Adjust the `width` and `height` to values acceptable by the decoder. + // Without this, FFmpeg may overflow the buffer. If modified, `width` and/or + // `height` are larger than the actual image and the image has to be cropped + // (top-left corner) after decoding to avoid visible borders to the right and + // bottom of the actual image. + avcodec_align_dimensions(context, &width, &height); + + RTC_CHECK_GE(width, 0); + RTC_CHECK_GE(height, 0); + int ret = av_image_check_size(static_cast<unsigned int>(width), + static_cast<unsigned int>(height), 0, nullptr); + if (ret < 0) { + RTC_LOG(LS_ERROR) << "Invalid picture size " << width << "x" << height; + decoder->ReportError(); + return ret; + } + + // The video frame is stored in `frame_buffer`. `av_frame` is FFmpeg's version + // of a video frame and will be set up to reference `frame_buffer`'s data. + + // FFmpeg expects the initial allocation to be zero-initialized according to + // http://crbug.com/390941. Our pool is set up to zero-initialize new buffers. + // TODO(https://crbug.com/390941): Delete that feature from the video pool, + // instead add an explicit call to InitializeData here. + rtc::scoped_refptr<PlanarYuvBuffer> frame_buffer; + rtc::scoped_refptr<I444Buffer> i444_buffer; + rtc::scoped_refptr<I420Buffer> i420_buffer; + rtc::scoped_refptr<I422Buffer> i422_buffer; + rtc::scoped_refptr<I010Buffer> i010_buffer; + rtc::scoped_refptr<I210Buffer> i210_buffer; + rtc::scoped_refptr<I410Buffer> i410_buffer; + int bytes_per_pixel = 1; + switch (context->pix_fmt) { + case AV_PIX_FMT_YUV420P: + case AV_PIX_FMT_YUVJ420P: + i420_buffer = + decoder->ffmpeg_buffer_pool_.CreateI420Buffer(width, height); + // Set `av_frame` members as required by FFmpeg. + av_frame->data[kYPlaneIndex] = i420_buffer->MutableDataY(); + av_frame->linesize[kYPlaneIndex] = i420_buffer->StrideY(); + av_frame->data[kUPlaneIndex] = i420_buffer->MutableDataU(); + av_frame->linesize[kUPlaneIndex] = i420_buffer->StrideU(); + av_frame->data[kVPlaneIndex] = i420_buffer->MutableDataV(); + av_frame->linesize[kVPlaneIndex] = i420_buffer->StrideV(); + RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data); + frame_buffer = i420_buffer; + break; + case AV_PIX_FMT_YUV444P: + case AV_PIX_FMT_YUVJ444P: + i444_buffer = + decoder->ffmpeg_buffer_pool_.CreateI444Buffer(width, height); + // Set `av_frame` members as required by FFmpeg. + av_frame->data[kYPlaneIndex] = i444_buffer->MutableDataY(); + av_frame->linesize[kYPlaneIndex] = i444_buffer->StrideY(); + av_frame->data[kUPlaneIndex] = i444_buffer->MutableDataU(); + av_frame->linesize[kUPlaneIndex] = i444_buffer->StrideU(); + av_frame->data[kVPlaneIndex] = i444_buffer->MutableDataV(); + av_frame->linesize[kVPlaneIndex] = i444_buffer->StrideV(); + frame_buffer = i444_buffer; + break; + case AV_PIX_FMT_YUV422P: + case AV_PIX_FMT_YUVJ422P: + i422_buffer = + decoder->ffmpeg_buffer_pool_.CreateI422Buffer(width, height); + // Set `av_frame` members as required by FFmpeg. + av_frame->data[kYPlaneIndex] = i422_buffer->MutableDataY(); + av_frame->linesize[kYPlaneIndex] = i422_buffer->StrideY(); + av_frame->data[kUPlaneIndex] = i422_buffer->MutableDataU(); + av_frame->linesize[kUPlaneIndex] = i422_buffer->StrideU(); + av_frame->data[kVPlaneIndex] = i422_buffer->MutableDataV(); + av_frame->linesize[kVPlaneIndex] = i422_buffer->StrideV(); + frame_buffer = i422_buffer; + break; + case AV_PIX_FMT_YUV420P10LE: + i010_buffer = + decoder->ffmpeg_buffer_pool_.CreateI010Buffer(width, height); + // Set `av_frame` members as required by FFmpeg. + av_frame->data[kYPlaneIndex] = + reinterpret_cast<uint8_t*>(i010_buffer->MutableDataY()); + av_frame->linesize[kYPlaneIndex] = i010_buffer->StrideY() * 2; + av_frame->data[kUPlaneIndex] = + reinterpret_cast<uint8_t*>(i010_buffer->MutableDataU()); + av_frame->linesize[kUPlaneIndex] = i010_buffer->StrideU() * 2; + av_frame->data[kVPlaneIndex] = + reinterpret_cast<uint8_t*>(i010_buffer->MutableDataV()); + av_frame->linesize[kVPlaneIndex] = i010_buffer->StrideV() * 2; + frame_buffer = i010_buffer; + bytes_per_pixel = 2; + break; + case AV_PIX_FMT_YUV422P10LE: + i210_buffer = + decoder->ffmpeg_buffer_pool_.CreateI210Buffer(width, height); + // Set `av_frame` members as required by FFmpeg. + av_frame->data[kYPlaneIndex] = + reinterpret_cast<uint8_t*>(i210_buffer->MutableDataY()); + av_frame->linesize[kYPlaneIndex] = i210_buffer->StrideY() * 2; + av_frame->data[kUPlaneIndex] = + reinterpret_cast<uint8_t*>(i210_buffer->MutableDataU()); + av_frame->linesize[kUPlaneIndex] = i210_buffer->StrideU() * 2; + av_frame->data[kVPlaneIndex] = + reinterpret_cast<uint8_t*>(i210_buffer->MutableDataV()); + av_frame->linesize[kVPlaneIndex] = i210_buffer->StrideV() * 2; + frame_buffer = i210_buffer; + bytes_per_pixel = 2; + break; + case AV_PIX_FMT_YUV444P10LE: + i410_buffer = + decoder->ffmpeg_buffer_pool_.CreateI410Buffer(width, height); + // Set `av_frame` members as required by FFmpeg. + av_frame->data[kYPlaneIndex] = + reinterpret_cast<uint8_t*>(i410_buffer->MutableDataY()); + av_frame->linesize[kYPlaneIndex] = i410_buffer->StrideY() * 2; + av_frame->data[kUPlaneIndex] = + reinterpret_cast<uint8_t*>(i410_buffer->MutableDataU()); + av_frame->linesize[kUPlaneIndex] = i410_buffer->StrideU() * 2; + av_frame->data[kVPlaneIndex] = + reinterpret_cast<uint8_t*>(i410_buffer->MutableDataV()); + av_frame->linesize[kVPlaneIndex] = i410_buffer->StrideV() * 2; + frame_buffer = i410_buffer; + bytes_per_pixel = 2; + break; + default: + RTC_LOG(LS_ERROR) << "Unsupported buffer type " << context->pix_fmt + << ". Check supported supported pixel formats!"; + decoder->ReportError(); + return -1; + } + + int y_size = width * height * bytes_per_pixel; + int uv_size = frame_buffer->ChromaWidth() * frame_buffer->ChromaHeight() * + bytes_per_pixel; + // DCHECK that we have a continuous buffer as is required. + RTC_DCHECK_EQ(av_frame->data[kUPlaneIndex], + av_frame->data[kYPlaneIndex] + y_size); + RTC_DCHECK_EQ(av_frame->data[kVPlaneIndex], + av_frame->data[kUPlaneIndex] + uv_size); + int total_size = y_size + 2 * uv_size; + + av_frame->format = context->pix_fmt; + av_frame->reordered_opaque = context->reordered_opaque; + + // Create a VideoFrame object, to keep a reference to the buffer. + // TODO(nisse): The VideoFrame's timestamp and rotation info is not used. + // Refactor to do not use a VideoFrame object at all. + av_frame->buf[0] = av_buffer_create( + av_frame->data[kYPlaneIndex], total_size, AVFreeBuffer2, + static_cast<void*>( + std::make_unique<VideoFrame>(VideoFrame::Builder() + .set_video_frame_buffer(frame_buffer) + .set_rotation(kVideoRotation_0) + .set_timestamp_us(0) + .build()) + .release()), + 0); + RTC_CHECK(av_frame->buf[0]); + return 0; +} + +void H264DecoderImpl::AVFreeBuffer2(void* opaque, uint8_t* data) { + // The buffer pool recycles the buffer used by `video_frame` when there are no + // more references to it. `video_frame` is a thin buffer holder and is not + // recycled. + VideoFrame* video_frame = static_cast<VideoFrame*>(opaque); + delete video_frame; +} + +H264DecoderImpl::H264DecoderImpl() + : ffmpeg_buffer_pool_(true), + decoded_image_callback_(nullptr), + has_reported_init_(false), + has_reported_error_(false) {} + +H264DecoderImpl::~H264DecoderImpl() { + Release(); +} + +bool H264DecoderImpl::Configure(const Settings& settings) { + ReportInit(); + if (settings.codec_type() != kVideoCodecH264) { + ReportError(); + return false; + } + + // Release necessary in case of re-initializing. + int32_t ret = Release(); + if (ret != WEBRTC_VIDEO_CODEC_OK) { + ReportError(); + return false; + } + RTC_DCHECK(!av_context_); + + // Initialize AVCodecContext. + av_context_.reset(avcodec_alloc_context3(nullptr)); + + av_context_->codec_type = AVMEDIA_TYPE_VIDEO; + av_context_->codec_id = AV_CODEC_ID_H264; + const RenderResolution& resolution = settings.max_render_resolution(); + if (resolution.Valid()) { + av_context_->coded_width = resolution.Width(); + av_context_->coded_height = resolution.Height(); + } + av_context_->extradata = nullptr; + av_context_->extradata_size = 0; + + // If this is ever increased, look at `av_context_->thread_safe_callbacks` and + // make it possible to disable the thread checker in the frame buffer pool. + av_context_->thread_count = 1; + av_context_->thread_type = FF_THREAD_SLICE; + + // Function used by FFmpeg to get buffers to store decoded frames in. + av_context_->get_buffer2 = AVGetBuffer2; + // `get_buffer2` is called with the context, there `opaque` can be used to get + // a pointer `this`. + av_context_->opaque = this; + + const AVCodec* codec = avcodec_find_decoder(av_context_->codec_id); + if (!codec) { + // This is an indication that FFmpeg has not been initialized or it has not + // been compiled/initialized with the correct set of codecs. + RTC_LOG(LS_ERROR) << "FFmpeg H.264 decoder not found."; + Release(); + ReportError(); + return false; + } + int res = avcodec_open2(av_context_.get(), codec, nullptr); + if (res < 0) { + RTC_LOG(LS_ERROR) << "avcodec_open2 error: " << res; + Release(); + ReportError(); + return false; + } + + av_frame_.reset(av_frame_alloc()); + + if (absl::optional<int> buffer_pool_size = settings.buffer_pool_size()) { + if (!ffmpeg_buffer_pool_.Resize(*buffer_pool_size)) { + return false; + } + } + return true; +} + +int32_t H264DecoderImpl::Release() { + av_context_.reset(); + av_frame_.reset(); + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t H264DecoderImpl::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + decoded_image_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t H264DecoderImpl::Decode(const EncodedImage& input_image, + bool /*missing_frames*/, + int64_t /*render_time_ms*/) { + if (!IsInitialized()) { + ReportError(); + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (!decoded_image_callback_) { + RTC_LOG(LS_WARNING) + << "Configure() has been called, but a callback function " + "has not been set with RegisterDecodeCompleteCallback()"; + ReportError(); + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (!input_image.data() || !input_image.size()) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + ScopedAVPacket packet = MakeScopedAVPacket(); + if (!packet) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + // packet.data has a non-const type, but isn't modified by + // avcodec_send_packet. + packet->data = const_cast<uint8_t*>(input_image.data()); + if (input_image.size() > + static_cast<size_t>(std::numeric_limits<int>::max())) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + packet->size = static_cast<int>(input_image.size()); + int64_t frame_timestamp_us = input_image.ntp_time_ms_ * 1000; // ms -> μs + av_context_->reordered_opaque = frame_timestamp_us; + + int result = avcodec_send_packet(av_context_.get(), packet.get()); + + if (result < 0) { + RTC_LOG(LS_ERROR) << "avcodec_send_packet error: " << result; + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + result = avcodec_receive_frame(av_context_.get(), av_frame_.get()); + if (result < 0) { + RTC_LOG(LS_ERROR) << "avcodec_receive_frame error: " << result; + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // We don't expect reordering. Decoded frame timestamp should match + // the input one. + RTC_DCHECK_EQ(av_frame_->reordered_opaque, frame_timestamp_us); + + // TODO(sakal): Maybe it is possible to get QP directly from FFmpeg. + h264_bitstream_parser_.ParseBitstream(input_image); + absl::optional<int> qp = h264_bitstream_parser_.GetLastSliceQp(); + + // Obtain the `video_frame` containing the decoded image. + VideoFrame* input_frame = + static_cast<VideoFrame*>(av_buffer_get_opaque(av_frame_->buf[0])); + RTC_DCHECK(input_frame); + rtc::scoped_refptr<VideoFrameBuffer> frame_buffer = + input_frame->video_frame_buffer(); + + // Instantiate Planar YUV buffer according to video frame buffer type + const webrtc::PlanarYuvBuffer* planar_yuv_buffer = nullptr; + const webrtc::PlanarYuv8Buffer* planar_yuv8_buffer = nullptr; + const webrtc::PlanarYuv16BBuffer* planar_yuv16_buffer = nullptr; + VideoFrameBuffer::Type video_frame_buffer_type = frame_buffer->type(); + switch (video_frame_buffer_type) { + case VideoFrameBuffer::Type::kI420: + planar_yuv_buffer = frame_buffer->GetI420(); + planar_yuv8_buffer = + reinterpret_cast<const webrtc::PlanarYuv8Buffer*>(planar_yuv_buffer); + break; + case VideoFrameBuffer::Type::kI444: + planar_yuv_buffer = frame_buffer->GetI444(); + planar_yuv8_buffer = + reinterpret_cast<const webrtc::PlanarYuv8Buffer*>(planar_yuv_buffer); + break; + case VideoFrameBuffer::Type::kI422: + planar_yuv_buffer = frame_buffer->GetI422(); + planar_yuv8_buffer = + reinterpret_cast<const webrtc::PlanarYuv8Buffer*>(planar_yuv_buffer); + break; + case VideoFrameBuffer::Type::kI010: + planar_yuv_buffer = frame_buffer->GetI010(); + planar_yuv16_buffer = reinterpret_cast<const webrtc::PlanarYuv16BBuffer*>( + planar_yuv_buffer); + break; + case VideoFrameBuffer::Type::kI210: + planar_yuv_buffer = frame_buffer->GetI210(); + planar_yuv16_buffer = reinterpret_cast<const webrtc::PlanarYuv16BBuffer*>( + planar_yuv_buffer); + break; + case VideoFrameBuffer::Type::kI410: + planar_yuv_buffer = frame_buffer->GetI410(); + planar_yuv16_buffer = reinterpret_cast<const webrtc::PlanarYuv16BBuffer*>( + planar_yuv_buffer); + break; + default: + // If this code is changed to allow other video frame buffer type, + // make sure that the code below which wraps I420/I422/I444 buffer and + // code which converts to NV12 is changed + // to work with new video frame buffer type + + RTC_LOG(LS_ERROR) << "frame_buffer type: " + << static_cast<int32_t>(video_frame_buffer_type) + << " is not supported!"; + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // When needed, FFmpeg applies cropping by moving plane pointers and adjusting + // frame width/height. Ensure that cropped buffers lie within the allocated + // memory. + RTC_DCHECK_LE(av_frame_->width, planar_yuv_buffer->width()); + RTC_DCHECK_LE(av_frame_->height, planar_yuv_buffer->height()); + switch (video_frame_buffer_type) { + case VideoFrameBuffer::Type::kI420: + case VideoFrameBuffer::Type::kI444: + case VideoFrameBuffer::Type::kI422: { + RTC_DCHECK_GE(av_frame_->data[kYPlaneIndex], planar_yuv8_buffer->DataY()); + RTC_DCHECK_LE( + av_frame_->data[kYPlaneIndex] + + av_frame_->linesize[kYPlaneIndex] * av_frame_->height, + planar_yuv8_buffer->DataY() + + planar_yuv8_buffer->StrideY() * planar_yuv8_buffer->height()); + RTC_DCHECK_GE(av_frame_->data[kUPlaneIndex], planar_yuv8_buffer->DataU()); + RTC_DCHECK_LE( + av_frame_->data[kUPlaneIndex] + + av_frame_->linesize[kUPlaneIndex] * + planar_yuv8_buffer->ChromaHeight(), + planar_yuv8_buffer->DataU() + planar_yuv8_buffer->StrideU() * + planar_yuv8_buffer->ChromaHeight()); + RTC_DCHECK_GE(av_frame_->data[kVPlaneIndex], planar_yuv8_buffer->DataV()); + RTC_DCHECK_LE( + av_frame_->data[kVPlaneIndex] + + av_frame_->linesize[kVPlaneIndex] * + planar_yuv8_buffer->ChromaHeight(), + planar_yuv8_buffer->DataV() + planar_yuv8_buffer->StrideV() * + planar_yuv8_buffer->ChromaHeight()); + break; + } + case VideoFrameBuffer::Type::kI010: + case VideoFrameBuffer::Type::kI210: + case VideoFrameBuffer::Type::kI410: { + RTC_DCHECK_GE( + av_frame_->data[kYPlaneIndex], + reinterpret_cast<const uint8_t*>(planar_yuv16_buffer->DataY())); + RTC_DCHECK_LE( + av_frame_->data[kYPlaneIndex] + + av_frame_->linesize[kYPlaneIndex] * av_frame_->height, + reinterpret_cast<const uint8_t*>(planar_yuv16_buffer->DataY()) + + planar_yuv16_buffer->StrideY() * 2 * + planar_yuv16_buffer->height()); + RTC_DCHECK_GE( + av_frame_->data[kUPlaneIndex], + reinterpret_cast<const uint8_t*>(planar_yuv16_buffer->DataU())); + RTC_DCHECK_LE( + av_frame_->data[kUPlaneIndex] + + av_frame_->linesize[kUPlaneIndex] * + planar_yuv16_buffer->ChromaHeight(), + reinterpret_cast<const uint8_t*>(planar_yuv16_buffer->DataU()) + + planar_yuv16_buffer->StrideU() * 2 * + planar_yuv16_buffer->ChromaHeight()); + RTC_DCHECK_GE( + av_frame_->data[kVPlaneIndex], + reinterpret_cast<const uint8_t*>(planar_yuv16_buffer->DataV())); + RTC_DCHECK_LE( + av_frame_->data[kVPlaneIndex] + + av_frame_->linesize[kVPlaneIndex] * + planar_yuv16_buffer->ChromaHeight(), + reinterpret_cast<const uint8_t*>(planar_yuv16_buffer->DataV()) + + planar_yuv16_buffer->StrideV() * 2 * + planar_yuv16_buffer->ChromaHeight()); + break; + } + default: + RTC_LOG(LS_ERROR) << "frame_buffer type: " + << static_cast<int32_t>(video_frame_buffer_type) + << " is not supported!"; + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + rtc::scoped_refptr<webrtc::VideoFrameBuffer> cropped_buffer; + switch (video_frame_buffer_type) { + case VideoFrameBuffer::Type::kI420: + cropped_buffer = WrapI420Buffer( + av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex], + av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex], + av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex], + av_frame_->linesize[kVPlaneIndex], + // To keep reference alive. + [frame_buffer] {}); + break; + case VideoFrameBuffer::Type::kI444: + cropped_buffer = WrapI444Buffer( + av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex], + av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex], + av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex], + av_frame_->linesize[kVPlaneIndex], + // To keep reference alive. + [frame_buffer] {}); + break; + case VideoFrameBuffer::Type::kI422: + cropped_buffer = WrapI422Buffer( + av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex], + av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex], + av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex], + av_frame_->linesize[kVPlaneIndex], + // To keep reference alive. + [frame_buffer] {}); + break; + case VideoFrameBuffer::Type::kI010: + cropped_buffer = WrapI010Buffer( + av_frame_->width, av_frame_->height, + reinterpret_cast<const uint16_t*>(av_frame_->data[kYPlaneIndex]), + av_frame_->linesize[kYPlaneIndex] / 2, + reinterpret_cast<const uint16_t*>(av_frame_->data[kUPlaneIndex]), + av_frame_->linesize[kUPlaneIndex] / 2, + reinterpret_cast<const uint16_t*>(av_frame_->data[kVPlaneIndex]), + av_frame_->linesize[kVPlaneIndex] / 2, + // To keep reference alive. + [frame_buffer] {}); + break; + case VideoFrameBuffer::Type::kI210: + cropped_buffer = WrapI210Buffer( + av_frame_->width, av_frame_->height, + reinterpret_cast<const uint16_t*>(av_frame_->data[kYPlaneIndex]), + av_frame_->linesize[kYPlaneIndex] / 2, + reinterpret_cast<const uint16_t*>(av_frame_->data[kUPlaneIndex]), + av_frame_->linesize[kUPlaneIndex] / 2, + reinterpret_cast<const uint16_t*>(av_frame_->data[kVPlaneIndex]), + av_frame_->linesize[kVPlaneIndex] / 2, + // To keep reference alive. + [frame_buffer] {}); + break; + case VideoFrameBuffer::Type::kI410: + cropped_buffer = WrapI410Buffer( + av_frame_->width, av_frame_->height, + reinterpret_cast<const uint16_t*>(av_frame_->data[kYPlaneIndex]), + av_frame_->linesize[kYPlaneIndex] / 2, + reinterpret_cast<const uint16_t*>(av_frame_->data[kUPlaneIndex]), + av_frame_->linesize[kUPlaneIndex] / 2, + reinterpret_cast<const uint16_t*>(av_frame_->data[kVPlaneIndex]), + av_frame_->linesize[kVPlaneIndex] / 2, + // To keep reference alive. + [frame_buffer] {}); + break; + default: + RTC_LOG(LS_ERROR) << "frame_buffer type: " + << static_cast<int32_t>(video_frame_buffer_type) + << " is not supported!"; + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Pass on color space from input frame if explicitly specified. + const ColorSpace& color_space = + input_image.ColorSpace() ? *input_image.ColorSpace() + : ExtractH264ColorSpace(av_context_.get()); + + VideoFrame decoded_frame = VideoFrame::Builder() + .set_video_frame_buffer(cropped_buffer) + .set_timestamp_rtp(input_image.Timestamp()) + .set_color_space(color_space) + .build(); + + // Return decoded frame. + // TODO(nisse): Timestamp and rotation are all zero here. Change decoder + // interface to pass a VideoFrameBuffer instead of a VideoFrame? + decoded_image_callback_->Decoded(decoded_frame, absl::nullopt, qp); + + // Stop referencing it, possibly freeing `input_frame`. + av_frame_unref(av_frame_.get()); + input_frame = nullptr; + + return WEBRTC_VIDEO_CODEC_OK; +} + +const char* H264DecoderImpl::ImplementationName() const { + return "FFmpeg"; +} + +bool H264DecoderImpl::IsInitialized() const { + return av_context_ != nullptr; +} + +void H264DecoderImpl::ReportInit() { + if (has_reported_init_) + return; + RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DecoderImpl.Event", + kH264DecoderEventInit, kH264DecoderEventMax); + has_reported_init_ = true; +} + +void H264DecoderImpl::ReportError() { + if (has_reported_error_) + return; + RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DecoderImpl.Event", + kH264DecoderEventError, kH264DecoderEventMax); + has_reported_error_ = true; +} + +} // namespace webrtc + +#endif // WEBRTC_USE_H264 diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.h b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.h new file mode 100644 index 0000000000..97d091cf4b --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_H264_H264_DECODER_IMPL_H_ +#define MODULES_VIDEO_CODING_CODECS_H264_H264_DECODER_IMPL_H_ + +// Everything declared in this header is only required when WebRTC is +// build with H264 support, please do not move anything out of the +// #ifdef unless needed and tested. +#ifdef WEBRTC_USE_H264 + +#if defined(WEBRTC_WIN) && !defined(__clang__) +#error "See: bugs.webrtc.org/9213#c13." +#endif + +#include <memory> + +#include "modules/video_coding/codecs/h264/include/h264.h" + +// CAVEAT: According to ffmpeg docs for avcodec_send_packet, ffmpeg requires a +// few extra padding bytes after the end of input. And in addition, docs for +// AV_INPUT_BUFFER_PADDING_SIZE says "If the first 23 bits of the additional +// bytes are not 0, then damaged MPEG bitstreams could cause overread and +// segfault." +// +// WebRTC doesn't ensure any such padding, and REQUIRES ffmpeg to be compiled +// with CONFIG_SAFE_BITSTREAM_READER, which is intended to eliminate +// out-of-bounds reads. ffmpeg docs doesn't say explicitly what effects this +// flag has on the h.264 decoder or avcodec_send_packet, though, so this is in +// some way depending on undocumented behavior. If any problems turn up, we may +// have to add an extra copy operation, to enforce padding before buffers are +// passed to ffmpeg. + +extern "C" { +#include "third_party/ffmpeg/libavcodec/avcodec.h" +} // extern "C" + +#include "common_video/h264/h264_bitstream_parser.h" +#include "common_video/include/video_frame_buffer_pool.h" + +namespace webrtc { + +struct AVCodecContextDeleter { + void operator()(AVCodecContext* ptr) const { avcodec_free_context(&ptr); } +}; +struct AVFrameDeleter { + void operator()(AVFrame* ptr) const { av_frame_free(&ptr); } +}; + +class H264DecoderImpl : public H264Decoder { + public: + H264DecoderImpl(); + ~H264DecoderImpl() override; + + bool Configure(const Settings& settings) override; + int32_t Release() override; + + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + + // `missing_frames`, `fragmentation` and `render_time_ms` are ignored. + int32_t Decode(const EncodedImage& input_image, + bool /*missing_frames*/, + int64_t render_time_ms = -1) override; + + const char* ImplementationName() const override; + + private: + // Called by FFmpeg when it needs a frame buffer to store decoded frames in. + // The `VideoFrame` returned by FFmpeg at `Decode` originate from here. Their + // buffers are reference counted and freed by FFmpeg using `AVFreeBuffer2`. + static int AVGetBuffer2(AVCodecContext* context, + AVFrame* av_frame, + int flags); + // Called by FFmpeg when it is done with a video frame, see `AVGetBuffer2`. + static void AVFreeBuffer2(void* opaque, uint8_t* data); + + bool IsInitialized() const; + + // Reports statistics with histograms. + void ReportInit(); + void ReportError(); + + // Used by ffmpeg via `AVGetBuffer2()` to allocate I420 images. + VideoFrameBufferPool ffmpeg_buffer_pool_; + std::unique_ptr<AVCodecContext, AVCodecContextDeleter> av_context_; + std::unique_ptr<AVFrame, AVFrameDeleter> av_frame_; + + DecodedImageCallback* decoded_image_callback_; + + bool has_reported_init_; + bool has_reported_error_; + + webrtc::H264BitstreamParser h264_bitstream_parser_; +}; + +} // namespace webrtc + +#endif // WEBRTC_USE_H264 + +#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_DECODER_IMPL_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc new file mode 100644 index 0000000000..b8055ac85f --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -0,0 +1,713 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +// Everything declared/defined in this header is only required when WebRTC is +// build with H264 support, please do not move anything out of the +// #ifdef unless needed and tested. +#ifdef WEBRTC_USE_H264 + +#include "modules/video_coding/codecs/h264/h264_encoder_impl.h" + +#include <algorithm> +#include <limits> +#include <string> + +#include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/video/video_codec_constants.h" +#include "api/video_codecs/scalability_mode.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "modules/video_coding/utility/simulcast_utility.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/metrics.h" +#include "third_party/libyuv/include/libyuv/convert.h" +#include "third_party/libyuv/include/libyuv/scale.h" +#include "third_party/openh264/src/codec/api/wels/codec_api.h" +#include "third_party/openh264/src/codec/api/wels/codec_app_def.h" +#include "third_party/openh264/src/codec/api/wels/codec_def.h" +#include "third_party/openh264/src/codec/api/wels/codec_ver.h" + +namespace webrtc { + +namespace { + +const bool kOpenH264EncoderDetailedLogging = false; + +// QP scaling thresholds. +static const int kLowH264QpThreshold = 24; +static const int kHighH264QpThreshold = 37; + +// Used by histograms. Values of entries should not be changed. +enum H264EncoderImplEvent { + kH264EncoderEventInit = 0, + kH264EncoderEventError = 1, + kH264EncoderEventMax = 16, +}; + +int NumberOfThreads(int width, int height, int number_of_cores) { + // TODO(hbos): In Chromium, multiple threads do not work with sandbox on Mac, + // see crbug.com/583348. Until further investigated, only use one thread. + // if (width * height >= 1920 * 1080 && number_of_cores > 8) { + // return 8; // 8 threads for 1080p on high perf machines. + // } else if (width * height > 1280 * 960 && number_of_cores >= 6) { + // return 3; // 3 threads for 1080p. + // } else if (width * height > 640 * 480 && number_of_cores >= 3) { + // return 2; // 2 threads for qHD/HD. + // } else { + // return 1; // 1 thread for VGA or less. + // } + // TODO(sprang): Also check sSliceArgument.uiSliceNum om GetEncoderPrams(), + // before enabling multithreading here. + return 1; +} + +VideoFrameType ConvertToVideoFrameType(EVideoFrameType type) { + switch (type) { + case videoFrameTypeIDR: + return VideoFrameType::kVideoFrameKey; + case videoFrameTypeSkip: + case videoFrameTypeI: + case videoFrameTypeP: + case videoFrameTypeIPMixed: + return VideoFrameType::kVideoFrameDelta; + case videoFrameTypeInvalid: + break; + } + RTC_DCHECK_NOTREACHED() << "Unexpected/invalid frame type: " << type; + return VideoFrameType::kEmptyFrame; +} + +absl::optional<ScalabilityMode> ScalabilityModeFromTemporalLayers( + int num_temporal_layers) { + switch (num_temporal_layers) { + case 0: + break; + case 1: + return ScalabilityMode::kL1T1; + case 2: + return ScalabilityMode::kL1T2; + case 3: + return ScalabilityMode::kL1T3; + default: + RTC_DCHECK_NOTREACHED(); + } + return absl::nullopt; +} + +} // namespace + +// Helper method used by H264EncoderImpl::Encode. +// Copies the encoded bytes from `info` to `encoded_image`. The +// `encoded_image->_buffer` may be deleted and reallocated if a bigger buffer is +// required. +// +// After OpenH264 encoding, the encoded bytes are stored in `info` spread out +// over a number of layers and "NAL units". Each NAL unit is a fragment starting +// with the four-byte start code {0,0,0,1}. All of this data (including the +// start codes) is copied to the `encoded_image->_buffer`. +static void RtpFragmentize(EncodedImage* encoded_image, SFrameBSInfo* info) { + // Calculate minimum buffer size required to hold encoded data. + size_t required_capacity = 0; + size_t fragments_count = 0; + for (int layer = 0; layer < info->iLayerNum; ++layer) { + const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; + for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) { + RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0); + // Ensure `required_capacity` will not overflow. + RTC_CHECK_LE(layerInfo.pNalLengthInByte[nal], + std::numeric_limits<size_t>::max() - required_capacity); + required_capacity += layerInfo.pNalLengthInByte[nal]; + } + } + auto buffer = EncodedImageBuffer::Create(required_capacity); + encoded_image->SetEncodedData(buffer); + + // Iterate layers and NAL units, note each NAL unit as a fragment and copy + // the data to `encoded_image->_buffer`. + const uint8_t start_code[4] = {0, 0, 0, 1}; + size_t frag = 0; + encoded_image->set_size(0); + for (int layer = 0; layer < info->iLayerNum; ++layer) { + const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; + // Iterate NAL units making up this layer, noting fragments. + size_t layer_len = 0; + for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) { + // Because the sum of all layer lengths, `required_capacity`, fits in a + // `size_t`, we know that any indices in-between will not overflow. + RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4); + RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 0], start_code[0]); + RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 1], start_code[1]); + RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 2], start_code[2]); + RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len + 3], start_code[3]); + layer_len += layerInfo.pNalLengthInByte[nal]; + } + // Copy the entire layer's data (including start codes). + memcpy(buffer->data() + encoded_image->size(), layerInfo.pBsBuf, layer_len); + encoded_image->set_size(encoded_image->size() + layer_len); + } +} + +H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec) + : packetization_mode_(H264PacketizationMode::SingleNalUnit), + max_payload_size_(0), + number_of_cores_(0), + encoded_image_callback_(nullptr), + has_reported_init_(false), + has_reported_error_(false) { + RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName)); + std::string packetization_mode_string; + if (codec.GetParam(cricket::kH264FmtpPacketizationMode, + &packetization_mode_string) && + packetization_mode_string == "1") { + packetization_mode_ = H264PacketizationMode::NonInterleaved; + } + downscaled_buffers_.reserve(kMaxSimulcastStreams - 1); + encoded_images_.reserve(kMaxSimulcastStreams); + encoders_.reserve(kMaxSimulcastStreams); + configurations_.reserve(kMaxSimulcastStreams); + tl0sync_limit_.reserve(kMaxSimulcastStreams); + svc_controllers_.reserve(kMaxSimulcastStreams); +} + +H264EncoderImpl::~H264EncoderImpl() { + Release(); +} + +int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst, + const VideoEncoder::Settings& settings) { + ReportInit(); + if (!inst || inst->codecType != kVideoCodecH264) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->maxFramerate == 0) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->width < 1 || inst->height < 1) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + int32_t release_ret = Release(); + if (release_ret != WEBRTC_VIDEO_CODEC_OK) { + ReportError(); + return release_ret; + } + + int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst); + bool doing_simulcast = (number_of_streams > 1); + + if (doing_simulcast && + !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) { + return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; + } + downscaled_buffers_.resize(number_of_streams - 1); + encoded_images_.resize(number_of_streams); + encoders_.resize(number_of_streams); + pictures_.resize(number_of_streams); + svc_controllers_.resize(number_of_streams); + scalability_modes_.resize(number_of_streams); + configurations_.resize(number_of_streams); + tl0sync_limit_.resize(number_of_streams); + + number_of_cores_ = settings.number_of_cores; + max_payload_size_ = settings.max_payload_size; + codec_ = *inst; + + // Code expects simulcastStream resolutions to be correct, make sure they are + // filled even when there are no simulcast layers. + if (codec_.numberOfSimulcastStreams == 0) { + codec_.simulcastStream[0].width = codec_.width; + codec_.simulcastStream[0].height = codec_.height; + } + + for (int i = 0, idx = number_of_streams - 1; i < number_of_streams; + ++i, --idx) { + ISVCEncoder* openh264_encoder; + // Create encoder. + if (WelsCreateSVCEncoder(&openh264_encoder) != 0) { + // Failed to create encoder. + RTC_LOG(LS_ERROR) << "Failed to create OpenH264 encoder"; + RTC_DCHECK(!openh264_encoder); + Release(); + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_DCHECK(openh264_encoder); + if (kOpenH264EncoderDetailedLogging) { + int trace_level = WELS_LOG_DETAIL; + openh264_encoder->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level); + } + // else WELS_LOG_DEFAULT is used by default. + + // Store h264 encoder. + encoders_[i] = openh264_encoder; + + // Set internal settings from codec_settings + configurations_[i].simulcast_idx = idx; + configurations_[i].sending = false; + configurations_[i].width = codec_.simulcastStream[idx].width; + configurations_[i].height = codec_.simulcastStream[idx].height; + configurations_[i].max_frame_rate = static_cast<float>(codec_.maxFramerate); + configurations_[i].frame_dropping_on = codec_.GetFrameDropEnabled(); + configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval; + configurations_[i].num_temporal_layers = + std::max(codec_.H264()->numberOfTemporalLayers, + codec_.simulcastStream[idx].numberOfTemporalLayers); + + // Create downscaled image buffers. + if (i > 0) { + downscaled_buffers_[i - 1] = I420Buffer::Create( + configurations_[i].width, configurations_[i].height, + configurations_[i].width, configurations_[i].width / 2, + configurations_[i].width / 2); + } + + // Codec_settings uses kbits/second; encoder uses bits/second. + configurations_[i].max_bps = codec_.maxBitrate * 1000; + configurations_[i].target_bps = codec_.startBitrate * 1000; + + // Create encoder parameters based on the layer configuration. + SEncParamExt encoder_params = CreateEncoderParams(i); + + // Initialize. + if (openh264_encoder->InitializeExt(&encoder_params) != 0) { + RTC_LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder"; + Release(); + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + // TODO(pbos): Base init params on these values before submitting. + int video_format = EVideoFormatType::videoFormatI420; + openh264_encoder->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format); + + // Initialize encoded image. Default buffer size: size of unencoded data. + + const size_t new_capacity = + CalcBufferSize(VideoType::kI420, codec_.simulcastStream[idx].width, + codec_.simulcastStream[idx].height); + encoded_images_[i].SetEncodedData(EncodedImageBuffer::Create(new_capacity)); + encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width; + encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height; + encoded_images_[i].set_size(0); + + tl0sync_limit_[i] = configurations_[i].num_temporal_layers; + scalability_modes_[i] = ScalabilityModeFromTemporalLayers( + configurations_[i].num_temporal_layers); + if (scalability_modes_[i].has_value()) { + svc_controllers_[i] = CreateScalabilityStructure(*scalability_modes_[i]); + if (svc_controllers_[i] == nullptr) { + RTC_LOG(LS_ERROR) << "Failed to create scalability structure"; + Release(); + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + } + + SimulcastRateAllocator init_allocator(codec_); + VideoBitrateAllocation allocation = + init_allocator.Allocate(VideoBitrateAllocationParameters( + DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate)); + SetRates(RateControlParameters(allocation, codec_.maxFramerate)); + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t H264EncoderImpl::Release() { + while (!encoders_.empty()) { + ISVCEncoder* openh264_encoder = encoders_.back(); + if (openh264_encoder) { + RTC_CHECK_EQ(0, openh264_encoder->Uninitialize()); + WelsDestroySVCEncoder(openh264_encoder); + } + encoders_.pop_back(); + } + downscaled_buffers_.clear(); + configurations_.clear(); + encoded_images_.clear(); + pictures_.clear(); + tl0sync_limit_.clear(); + svc_controllers_.clear(); + scalability_modes_.clear(); + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t H264EncoderImpl::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + encoded_image_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +void H264EncoderImpl::SetRates(const RateControlParameters& parameters) { + if (encoders_.empty()) { + RTC_LOG(LS_WARNING) << "SetRates() while uninitialized."; + return; + } + + if (parameters.framerate_fps < 1.0) { + RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps; + return; + } + + if (parameters.bitrate.get_sum_bps() == 0) { + // Encoder paused, turn off all encoding. + for (size_t i = 0; i < configurations_.size(); ++i) { + configurations_[i].SetStreamState(false); + } + return; + } + + codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps); + + size_t stream_idx = encoders_.size() - 1; + for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) { + // Update layer config. + configurations_[i].target_bps = + parameters.bitrate.GetSpatialLayerSum(stream_idx); + configurations_[i].max_frame_rate = parameters.framerate_fps; + + if (configurations_[i].target_bps) { + configurations_[i].SetStreamState(true); + + // Update h264 encoder. + SBitrateInfo target_bitrate; + memset(&target_bitrate, 0, sizeof(SBitrateInfo)); + target_bitrate.iLayer = SPATIAL_LAYER_ALL, + target_bitrate.iBitrate = configurations_[i].target_bps; + encoders_[i]->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate); + encoders_[i]->SetOption(ENCODER_OPTION_FRAME_RATE, + &configurations_[i].max_frame_rate); + } else { + configurations_[i].SetStreamState(false); + } + } +} + +int32_t H264EncoderImpl::Encode( + const VideoFrame& input_frame, + const std::vector<VideoFrameType>* frame_types) { + if (encoders_.empty()) { + ReportError(); + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (!encoded_image_callback_) { + RTC_LOG(LS_WARNING) + << "InitEncode() has been called, but a callback function " + "has not been set with RegisterEncodeCompleteCallback()"; + ReportError(); + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + rtc::scoped_refptr<I420BufferInterface> frame_buffer = + input_frame.video_frame_buffer()->ToI420(); + if (!frame_buffer) { + RTC_LOG(LS_ERROR) << "Failed to convert " + << VideoFrameBufferTypeToString( + input_frame.video_frame_buffer()->type()) + << " image to I420. Can't encode frame."; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + RTC_CHECK(frame_buffer->type() == VideoFrameBuffer::Type::kI420 || + frame_buffer->type() == VideoFrameBuffer::Type::kI420A); + + bool is_keyframe_needed = false; + for (size_t i = 0; i < configurations_.size(); ++i) { + if (configurations_[i].key_frame_request && configurations_[i].sending) { + // This is legacy behavior, generating a keyframe on all layers + // when generating one for a layer that became active for the first time + // or after being disabled. + is_keyframe_needed = true; + break; + } + } + + RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width()); + RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height()); + + // Encode image for each layer. + for (size_t i = 0; i < encoders_.size(); ++i) { + // EncodeFrame input. + pictures_[i] = {0}; + pictures_[i].iPicWidth = configurations_[i].width; + pictures_[i].iPicHeight = configurations_[i].height; + pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420; + pictures_[i].uiTimeStamp = input_frame.ntp_time_ms(); + // Downscale images on second and ongoing layers. + if (i == 0) { + pictures_[i].iStride[0] = frame_buffer->StrideY(); + pictures_[i].iStride[1] = frame_buffer->StrideU(); + pictures_[i].iStride[2] = frame_buffer->StrideV(); + pictures_[i].pData[0] = const_cast<uint8_t*>(frame_buffer->DataY()); + pictures_[i].pData[1] = const_cast<uint8_t*>(frame_buffer->DataU()); + pictures_[i].pData[2] = const_cast<uint8_t*>(frame_buffer->DataV()); + } else { + pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY(); + pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU(); + pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV(); + pictures_[i].pData[0] = + const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataY()); + pictures_[i].pData[1] = + const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataU()); + pictures_[i].pData[2] = + const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataV()); + // Scale the image down a number of times by downsampling factor. + libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0], + pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1], + pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2], + configurations_[i - 1].width, + configurations_[i - 1].height, pictures_[i].pData[0], + pictures_[i].iStride[0], pictures_[i].pData[1], + pictures_[i].iStride[1], pictures_[i].pData[2], + pictures_[i].iStride[2], configurations_[i].width, + configurations_[i].height, libyuv::kFilterBox); + } + + if (!configurations_[i].sending) { + continue; + } + if (frame_types != nullptr && i < frame_types->size()) { + // Skip frame? + if ((*frame_types)[i] == VideoFrameType::kEmptyFrame) { + continue; + } + } + // Send a key frame either when this layer is configured to require one + // or we have explicitly been asked to. + const size_t simulcast_idx = + static_cast<size_t>(configurations_[i].simulcast_idx); + bool send_key_frame = + is_keyframe_needed || + (frame_types && simulcast_idx < frame_types->size() && + (*frame_types)[simulcast_idx] == VideoFrameType::kVideoFrameKey); + if (send_key_frame) { + // API doc says ForceIntraFrame(false) does nothing, but calling this + // function forces a key frame regardless of the `bIDR` argument's value. + // (If every frame is a key frame we get lag/delays.) + encoders_[i]->ForceIntraFrame(true); + configurations_[i].key_frame_request = false; + } + // EncodeFrame output. + SFrameBSInfo info; + memset(&info, 0, sizeof(SFrameBSInfo)); + + std::vector<ScalableVideoController::LayerFrameConfig> layer_frames; + if (svc_controllers_[i]) { + layer_frames = svc_controllers_[i]->NextFrameConfig(send_key_frame); + RTC_CHECK_EQ(layer_frames.size(), 1); + } + + // Encode! + int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info); + if (enc_ret != 0) { + RTC_LOG(LS_ERROR) + << "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret + << "."; + ReportError(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + encoded_images_[i]._encodedWidth = configurations_[i].width; + encoded_images_[i]._encodedHeight = configurations_[i].height; + encoded_images_[i].SetTimestamp(input_frame.timestamp()); + encoded_images_[i].SetColorSpace(input_frame.color_space()); + encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType); + encoded_images_[i].SetSpatialIndex(configurations_[i].simulcast_idx); + + // Split encoded image up into fragments. This also updates + // `encoded_image_`. + RtpFragmentize(&encoded_images_[i], &info); + + // Encoder can skip frames to save bandwidth in which case + // `encoded_images_[i]._length` == 0. + if (encoded_images_[i].size() > 0) { + // Parse QP. + h264_bitstream_parser_.ParseBitstream(encoded_images_[i]); + encoded_images_[i].qp_ = + h264_bitstream_parser_.GetLastSliceQp().value_or(-1); + + // Deliver encoded image. + CodecSpecificInfo codec_specific; + codec_specific.codecType = kVideoCodecH264; + codec_specific.codecSpecific.H264.packetization_mode = + packetization_mode_; + codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx; + codec_specific.codecSpecific.H264.idr_frame = + info.eFrameType == videoFrameTypeIDR; + codec_specific.codecSpecific.H264.base_layer_sync = false; + if (configurations_[i].num_temporal_layers > 1) { + const uint8_t tid = info.sLayerInfo[0].uiTemporalId; + codec_specific.codecSpecific.H264.temporal_idx = tid; + codec_specific.codecSpecific.H264.base_layer_sync = + tid > 0 && tid < tl0sync_limit_[i]; + if (svc_controllers_[i]) { + if (layer_frames[0].TemporalId() != tid) { + RTC_LOG(LS_WARNING) + << "Encoder produced a frame for layer S" << (i + 1) << "T" + << tid + 1 << " that wasn't requested."; + continue; + } + encoded_images_[i].SetTemporalIndex(tid); + } + if (codec_specific.codecSpecific.H264.base_layer_sync) { + tl0sync_limit_[i] = tid; + } + if (tid == 0) { + tl0sync_limit_[i] = configurations_[i].num_temporal_layers; + } + } + if (svc_controllers_[i]) { + codec_specific.generic_frame_info = + svc_controllers_[i]->OnEncodeDone(layer_frames[0]); + if (send_key_frame && codec_specific.generic_frame_info.has_value()) { + codec_specific.template_structure = + svc_controllers_[i]->DependencyStructure(); + } + codec_specific.scalability_mode = scalability_modes_[i]; + } + encoded_image_callback_->OnEncodedImage(encoded_images_[i], + &codec_specific); + } + } + return WEBRTC_VIDEO_CODEC_OK; +} + +// Initialization parameters. +// There are two ways to initialize. There is SEncParamBase (cleared with +// memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt +// which is a superset of SEncParamBase (cleared with GetDefaultParams) used +// in InitializeExt. +SEncParamExt H264EncoderImpl::CreateEncoderParams(size_t i) const { + SEncParamExt encoder_params; + encoders_[i]->GetDefaultParams(&encoder_params); + if (codec_.mode == VideoCodecMode::kRealtimeVideo) { + encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME; + } else if (codec_.mode == VideoCodecMode::kScreensharing) { + encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME; + } else { + RTC_DCHECK_NOTREACHED(); + } + encoder_params.iPicWidth = configurations_[i].width; + encoder_params.iPicHeight = configurations_[i].height; + encoder_params.iTargetBitrate = configurations_[i].target_bps; + // Keep unspecified. WebRTC's max codec bitrate is not the same setting + // as OpenH264's iMaxBitrate. More details in https://crbug.com/webrtc/11543 + encoder_params.iMaxBitrate = UNSPECIFIED_BIT_RATE; + // Rate Control mode + encoder_params.iRCMode = RC_BITRATE_MODE; + encoder_params.fMaxFrameRate = configurations_[i].max_frame_rate; + + // The following parameters are extension parameters (they're in SEncParamExt, + // not in SEncParamBase). + encoder_params.bEnableFrameSkip = configurations_[i].frame_dropping_on; + // `uiIntraPeriod` - multiple of GOP size + // `keyFrameInterval` - number of frames + encoder_params.uiIntraPeriod = configurations_[i].key_frame_interval; + // Reuse SPS id if possible. This helps to avoid reset of chromium HW decoder + // on each key-frame. + // Note that WebRTC resets encoder on resolution change which makes all + // EParameterSetStrategy modes except INCREASING_ID (default) essentially + // equivalent to CONSTANT_ID. + encoder_params.eSpsPpsIdStrategy = SPS_LISTING; + encoder_params.uiMaxNalSize = 0; + // Threading model: use auto. + // 0: auto (dynamic imp. internal encoder) + // 1: single thread (default value) + // >1: number of threads + encoder_params.iMultipleThreadIdc = NumberOfThreads( + encoder_params.iPicWidth, encoder_params.iPicHeight, number_of_cores_); + // The base spatial layer 0 is the only one we use. + encoder_params.sSpatialLayers[0].iVideoWidth = encoder_params.iPicWidth; + encoder_params.sSpatialLayers[0].iVideoHeight = encoder_params.iPicHeight; + encoder_params.sSpatialLayers[0].fFrameRate = encoder_params.fMaxFrameRate; + encoder_params.sSpatialLayers[0].iSpatialBitrate = + encoder_params.iTargetBitrate; + encoder_params.sSpatialLayers[0].iMaxSpatialBitrate = + encoder_params.iMaxBitrate; + encoder_params.iTemporalLayerNum = configurations_[i].num_temporal_layers; + if (encoder_params.iTemporalLayerNum > 1) { + // iNumRefFrame specifies total number of reference buffers to allocate. + // For N temporal layers we need at least (N - 1) buffers to store last + // encoded frames of all reference temporal layers. + // Note that there is no API in OpenH264 encoder to specify exact set of + // references to be used to prediction of a given frame. Encoder can + // theoretically use all available reference buffers. + encoder_params.iNumRefFrame = encoder_params.iTemporalLayerNum - 1; + } + RTC_LOG(LS_INFO) << "OpenH264 version is " << OPENH264_MAJOR << "." + << OPENH264_MINOR; + switch (packetization_mode_) { + case H264PacketizationMode::SingleNalUnit: + // Limit the size of the packets produced. + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = + SM_SIZELIMITED_SLICE; + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint = + static_cast<unsigned int>(max_payload_size_); + RTC_LOG(LS_INFO) << "Encoder is configured with NALU constraint: " + << max_payload_size_ << " bytes"; + break; + case H264PacketizationMode::NonInterleaved: + // When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto + // design it with cpu core number. + // TODO(sprang): Set to 0 when we understand why the rate controller borks + // when uiSliceNum > 1. + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = + SM_FIXEDSLCNUM_SLICE; + break; + } + return encoder_params; +} + +void H264EncoderImpl::ReportInit() { + if (has_reported_init_) + return; + RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event", + kH264EncoderEventInit, kH264EncoderEventMax); + has_reported_init_ = true; +} + +void H264EncoderImpl::ReportError() { + if (has_reported_error_) + return; + RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264EncoderImpl.Event", + kH264EncoderEventError, kH264EncoderEventMax); + has_reported_error_ = true; +} + +VideoEncoder::EncoderInfo H264EncoderImpl::GetEncoderInfo() const { + EncoderInfo info; + info.supports_native_handle = false; + info.implementation_name = "OpenH264"; + info.scaling_settings = + VideoEncoder::ScalingSettings(kLowH264QpThreshold, kHighH264QpThreshold); + info.is_hardware_accelerated = false; + info.supports_simulcast = true; + info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420}; + return info; +} + +void H264EncoderImpl::LayerConfig::SetStreamState(bool send_stream) { + if (send_stream && !sending) { + // Need a key frame if we have not sent this stream before. + key_frame_request = true; + } + sending = send_stream; +} + +} // namespace webrtc + +#endif // WEBRTC_USE_H264 diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h new file mode 100644 index 0000000000..f02521f0dc --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_ +#define MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_ + +// Everything declared in this header is only required when WebRTC is +// build with H264 support, please do not move anything out of the +// #ifdef unless needed and tested. +#ifdef WEBRTC_USE_H264 + +#if defined(WEBRTC_WIN) && !defined(__clang__) +#error "See: bugs.webrtc.org/9213#c13." +#endif + +#include <memory> +#include <vector> + +#include "absl/container/inlined_vector.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_codec_constants.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/h264/h264_bitstream_parser.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "modules/video_coding/svc/scalable_video_controller.h" +#include "modules/video_coding/utility/quality_scaler.h" +#include "third_party/openh264/src/codec/api/wels/codec_app_def.h" + +class ISVCEncoder; + +namespace webrtc { + +class H264EncoderImpl : public H264Encoder { + public: + struct LayerConfig { + int simulcast_idx = 0; + int width = -1; + int height = -1; + bool sending = true; + bool key_frame_request = false; + float max_frame_rate = 0; + uint32_t target_bps = 0; + uint32_t max_bps = 0; + bool frame_dropping_on = false; + int key_frame_interval = 0; + int num_temporal_layers = 1; + + void SetStreamState(bool send_stream); + }; + + public: + explicit H264EncoderImpl(const cricket::VideoCodec& codec); + ~H264EncoderImpl() override; + + // `settings.max_payload_size` is ignored. + // The following members of `codec_settings` are used. The rest are ignored. + // - codecType (must be kVideoCodecH264) + // - targetBitrate + // - maxFramerate + // - width + // - height + int32_t InitEncode(const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) override; + int32_t Release() override; + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + void SetRates(const RateControlParameters& parameters) override; + + // The result of encoding - an EncodedImage and CodecSpecificInfo - are + // passed to the encode complete callback. + int32_t Encode(const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) override; + + EncoderInfo GetEncoderInfo() const override; + + // Exposed for testing. + H264PacketizationMode PacketizationModeForTesting() const { + return packetization_mode_; + } + + private: + SEncParamExt CreateEncoderParams(size_t i) const; + + webrtc::H264BitstreamParser h264_bitstream_parser_; + // Reports statistics with histograms. + void ReportInit(); + void ReportError(); + + std::vector<ISVCEncoder*> encoders_; + std::vector<SSourcePicture> pictures_; + std::vector<rtc::scoped_refptr<I420Buffer>> downscaled_buffers_; + std::vector<LayerConfig> configurations_; + std::vector<EncodedImage> encoded_images_; + std::vector<std::unique_ptr<ScalableVideoController>> svc_controllers_; + absl::InlinedVector<absl::optional<ScalabilityMode>, kMaxSimulcastStreams> + scalability_modes_; + + VideoCodec codec_; + H264PacketizationMode packetization_mode_; + size_t max_payload_size_; + int32_t number_of_cores_; + EncodedImageCallback* encoded_image_callback_; + + bool has_reported_init_; + bool has_reported_error_; + + std::vector<uint8_t> tl0sync_limit_; +}; + +} // namespace webrtc + +#endif // WEBRTC_USE_H264 + +#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_ENCODER_IMPL_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc new file mode 100644 index 0000000000..52d26955ab --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "modules/video_coding/codecs/h264/h264_encoder_impl.h" + +#include "api/video_codecs/video_encoder.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +const int kMaxPayloadSize = 1024; +const int kNumCores = 1; + +const VideoEncoder::Capabilities kCapabilities(false); +const VideoEncoder::Settings kSettings(kCapabilities, + kNumCores, + kMaxPayloadSize); + +void SetDefaultSettings(VideoCodec* codec_settings) { + codec_settings->codecType = kVideoCodecH264; + codec_settings->maxFramerate = 60; + codec_settings->width = 640; + codec_settings->height = 480; + // If frame dropping is false, we get a warning that bitrate can't + // be controlled for RC_QUALITY_MODE; RC_BITRATE_MODE and RC_TIMESTAMP_MODE + codec_settings->SetFrameDropEnabled(true); + codec_settings->startBitrate = 2000; + codec_settings->maxBitrate = 4000; +} + +TEST(H264EncoderImplTest, CanInitializeWithDefaultParameters) { + H264EncoderImpl encoder(cricket::VideoCodec("H264")); + VideoCodec codec_settings; + SetDefaultSettings(&codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(H264PacketizationMode::NonInterleaved, + encoder.PacketizationModeForTesting()); +} + +TEST(H264EncoderImplTest, CanInitializeWithNonInterleavedModeExplicitly) { + cricket::VideoCodec codec("H264"); + codec.SetParam(cricket::kH264FmtpPacketizationMode, "1"); + H264EncoderImpl encoder(codec); + VideoCodec codec_settings; + SetDefaultSettings(&codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(H264PacketizationMode::NonInterleaved, + encoder.PacketizationModeForTesting()); +} + +TEST(H264EncoderImplTest, CanInitializeWithSingleNalUnitModeExplicitly) { + cricket::VideoCodec codec("H264"); + codec.SetParam(cricket::kH264FmtpPacketizationMode, "0"); + H264EncoderImpl encoder(codec); + VideoCodec codec_settings; + SetDefaultSettings(&codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(H264PacketizationMode::SingleNalUnit, + encoder.PacketizationModeForTesting()); +} + +TEST(H264EncoderImplTest, CanInitializeWithRemovedParameter) { + cricket::VideoCodec codec("H264"); + codec.RemoveParam(cricket::kH264FmtpPacketizationMode); + H264EncoderImpl encoder(codec); + VideoCodec codec_settings; + SetDefaultSettings(&codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(H264PacketizationMode::SingleNalUnit, + encoder.PacketizationModeForTesting()); +} + +} // anonymous namespace + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_simulcast_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_simulcast_unittest.cc new file mode 100644 index 0000000000..2acb629a76 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_simulcast_unittest.cc @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <memory> + +#include "api/test/create_simulcast_test_fixture.h" +#include "api/test/simulcast_test_fixture.h" +#include "api/test/video/function_video_decoder_factory.h" +#include "api/test/video/function_video_encoder_factory.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { +std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture() { + std::unique_ptr<VideoEncoderFactory> encoder_factory = + std::make_unique<FunctionVideoEncoderFactory>( + []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); + std::unique_ptr<VideoDecoderFactory> decoder_factory = + std::make_unique<FunctionVideoDecoderFactory>( + []() { return H264Decoder::Create(); }); + return CreateSimulcastTestFixture(std::move(encoder_factory), + std::move(decoder_factory), + SdpVideoFormat("H264")); +} +} // namespace + +TEST(TestH264Simulcast, TestKeyFrameRequestsOnAllStreams) { + GTEST_SKIP() << "Not applicable to H264."; +} + +TEST(TestH264Simulcast, TestKeyFrameRequestsOnSpecificStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestKeyFrameRequestsOnSpecificStreams(); +} + +TEST(TestH264Simulcast, TestPaddingAllStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingAllStreams(); +} + +TEST(TestH264Simulcast, TestPaddingTwoStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingTwoStreams(); +} + +TEST(TestH264Simulcast, TestPaddingTwoStreamsOneMaxedOut) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingTwoStreamsOneMaxedOut(); +} + +TEST(TestH264Simulcast, TestPaddingOneStream) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingOneStream(); +} + +TEST(TestH264Simulcast, TestPaddingOneStreamTwoMaxedOut) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingOneStreamTwoMaxedOut(); +} + +TEST(TestH264Simulcast, TestSendAllStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestSendAllStreams(); +} + +TEST(TestH264Simulcast, TestDisablingStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestDisablingStreams(); +} + +TEST(TestH264Simulcast, TestActiveStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestActiveStreams(); +} + +TEST(TestH264Simulcast, TestSwitchingToOneStream) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestSwitchingToOneStream(); +} + +TEST(TestH264Simulcast, TestSwitchingToOneOddStream) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestSwitchingToOneOddStream(); +} + +TEST(TestH264Simulcast, TestStrideEncodeDecode) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestStrideEncodeDecode(); +} + +TEST(TestH264Simulcast, TestSpatioTemporalLayers333PatternEncoder) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestSpatioTemporalLayers333PatternEncoder(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/include/h264.h b/third_party/libwebrtc/modules/video_coding/codecs/h264/include/h264.h new file mode 100644 index 0000000000..2635b53842 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/include/h264.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_H_ +#define MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "api/video_codecs/h264_profile_level_id.h" +#include "api/video_codecs/scalability_mode.h" +#include "media/base/codec.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +struct SdpVideoFormat; + +// Creates an H264 SdpVideoFormat entry with specified paramters. +RTC_EXPORT SdpVideoFormat +CreateH264Format(H264Profile profile, + H264Level level, + const std::string& packetization_mode, + bool add_scalability_modes = false); + +// Set to disable the H.264 encoder/decoder implementations that are provided if +// `rtc_use_h264` build flag is true (if false, this function does nothing). +// This function should only be called before or during WebRTC initialization +// and is not thread-safe. +RTC_EXPORT void DisableRtcUseH264(); + +// Returns a vector with all supported internal H264 encode profiles that we can +// negotiate in SDP, in order of preference. +std::vector<SdpVideoFormat> SupportedH264Codecs( + bool add_scalability_modes = false); + +// Returns a vector with all supported internal H264 decode profiles that we can +// negotiate in SDP, in order of preference. This will be available for receive +// only connections. +std::vector<SdpVideoFormat> SupportedH264DecoderCodecs(); + +class RTC_EXPORT H264Encoder : public VideoEncoder { + public: + static std::unique_ptr<H264Encoder> Create(const cricket::VideoCodec& codec); + // If H.264 is supported (any implementation). + static bool IsSupported(); + static bool SupportsScalabilityMode(ScalabilityMode scalability_mode); + + ~H264Encoder() override {} +}; + +class RTC_EXPORT H264Decoder : public VideoDecoder { + public: + static std::unique_ptr<H264Decoder> Create(); + static bool IsSupported(); + + ~H264Decoder() override {} +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/include/h264_globals.h b/third_party/libwebrtc/modules/video_coding/codecs/h264/include/h264_globals.h new file mode 100644 index 0000000000..b61dc8c507 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/include/h264_globals.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file contains codec dependent definitions that are needed in +// order to compile the WebRTC codebase, even if this codec is not used. + +#ifndef MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_GLOBALS_H_ +#define MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_GLOBALS_H_ + +#include <string> + +#include "modules/video_coding/codecs/interface/common_constants.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// The packetization types that we support: single, aggregated, and fragmented. +enum H264PacketizationTypes { + kH264SingleNalu, // This packet contains a single NAL unit. + kH264StapA, // This packet contains STAP-A (single time + // aggregation) packets. If this packet has an + // associated NAL unit type, it'll be for the + // first such aggregated packet. + kH264FuA, // This packet contains a FU-A (fragmentation + // unit) packet, meaning it is a part of a frame + // that was too large to fit into a single packet. +}; + +// Packetization modes are defined in RFC 6184 section 6 +// Due to the structure containing this being initialized with zeroes +// in some places, and mode 1 being default, mode 1 needs to have the value +// zero. https://crbug.com/webrtc/6803 +enum class H264PacketizationMode { + NonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed + SingleNalUnit // Mode 0 - only single NALU allowed +}; + +// This function is declared inline because it is not clear which +// .cc file it should belong to. +// TODO(hta): Refactor. https://bugs.webrtc.org/6842 +// TODO(jonasolsson): Use absl::string_view instead when that's available. +inline std::string ToString(H264PacketizationMode mode) { + if (mode == H264PacketizationMode::NonInterleaved) { + return "NonInterleaved"; + } else if (mode == H264PacketizationMode::SingleNalUnit) { + return "SingleNalUnit"; + } + RTC_DCHECK_NOTREACHED(); + return ""; +} + +struct NaluInfo { + uint8_t type; + int sps_id; + int pps_id; +}; + +const size_t kMaxNalusPerPacket = 10; + +struct RTPVideoHeaderH264 { + // The NAL unit type. If this is a header for a + // fragmented packet, it's the NAL unit type of + // the original data. If this is the header for an + // aggregated packet, it's the NAL unit type of + // the first NAL unit in the packet. + uint8_t nalu_type; + // The packetization type of this buffer - single, aggregated or fragmented. + H264PacketizationTypes packetization_type; + NaluInfo nalus[kMaxNalusPerPacket]; + size_t nalus_length; + // The packetization mode of this transport. Packetization mode + // determines which packetization types are allowed when packetizing. + H264PacketizationMode packetization_mode; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_GLOBALS_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc new file mode 100644 index 0000000000..595e627bcc --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <stdint.h> + +#include <memory> + +#include "absl/types/optional.h" +#include "api/video/color_space.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/codecs/h264/include/h264.h" +#include "modules/video_coding/codecs/test/video_codec_unittest.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "test/gtest.h" +#include "test/video_codec_settings.h" + +namespace webrtc { + +class TestH264Impl : public VideoCodecUnitTest { + protected: + std::unique_ptr<VideoEncoder> CreateEncoder() override { + return H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName)); + } + + std::unique_ptr<VideoDecoder> CreateDecoder() override { + return H264Decoder::Create(); + } + + void ModifyCodecSettings(VideoCodec* codec_settings) override { + webrtc::test::CodecSettings(kVideoCodecH264, codec_settings); + } +}; + +#ifdef WEBRTC_USE_H264 +#define MAYBE_EncodeDecode EncodeDecode +#define MAYBE_DecodedQpEqualsEncodedQp DecodedQpEqualsEncodedQp +#else +#define MAYBE_EncodeDecode DISABLED_EncodeDecode +#define MAYBE_DecodedQpEqualsEncodedQp DISABLED_DecodedQpEqualsEncodedQp +#endif + +TEST_F(TestH264Impl, MAYBE_EncodeDecode) { + VideoFrame input_frame = NextInputFrame(); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(input_frame, nullptr)); + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info)); + // First frame should be a key frame. + encoded_frame._frameType = VideoFrameType::kVideoFrameKey; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, 0)); + std::unique_ptr<VideoFrame> decoded_frame; + absl::optional<uint8_t> decoded_qp; + ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp)); + ASSERT_TRUE(decoded_frame); + EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36); + + const ColorSpace color_space = *decoded_frame->color_space(); + EXPECT_EQ(ColorSpace::PrimaryID::kUnspecified, color_space.primaries()); + EXPECT_EQ(ColorSpace::TransferID::kUnspecified, color_space.transfer()); + EXPECT_EQ(ColorSpace::MatrixID::kUnspecified, color_space.matrix()); + EXPECT_EQ(ColorSpace::RangeID::kInvalid, color_space.range()); + EXPECT_EQ(ColorSpace::ChromaSiting::kUnspecified, + color_space.chroma_siting_horizontal()); + EXPECT_EQ(ColorSpace::ChromaSiting::kUnspecified, + color_space.chroma_siting_vertical()); +} + +TEST_F(TestH264Impl, MAYBE_DecodedQpEqualsEncodedQp) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr)); + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info)); + // First frame should be a key frame. + encoded_frame._frameType = VideoFrameType::kVideoFrameKey; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, 0)); + std::unique_ptr<VideoFrame> decoded_frame; + absl::optional<uint8_t> decoded_qp; + ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp)); + ASSERT_TRUE(decoded_frame); + ASSERT_TRUE(decoded_qp); + EXPECT_EQ(encoded_frame.qp_, *decoded_qp); +} + +} // namespace webrtc |