diff options
Diffstat (limited to 'third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc')
| -rw-r--r-- | third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc | 888 |
1 files changed, 327 insertions, 561 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc b/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc index 1c8fe97e84..60c2fcbb6e 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc @@ -8,41 +8,62 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "api/video_codecs/video_codec.h" - -#include <cstddef> #include <memory> #include <string> #include <vector> #include "absl/flags/flag.h" #include "absl/functional/any_invocable.h" -#include "api/test/create_video_codec_tester.h" #include "api/test/metrics/global_metrics_logger_and_exporter.h" -#include "api/test/video_codec_tester.h" -#include "api/test/videocodec_test_stats.h" #include "api/units/data_rate.h" #include "api/units/frequency.h" -#include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" #include "api/video/resolution.h" -#include "api/video/video_frame.h" -#include "api/video_codecs/scalability_mode.h" -#include "api/video_codecs/video_decoder.h" -#include "api/video_codecs/video_encoder.h" -#include "media/engine/internal_decoder_factory.h" -#include "media/engine/internal_encoder_factory.h" -#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "modules/video_coding/include/video_error_codes.h" -#include "modules/video_coding/svc/scalability_mode_util.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" #if defined(WEBRTC_ANDROID) #include "modules/video_coding/codecs/test/android_codec_factory_helper.h" #endif +#include "modules/video_coding/svc/scalability_mode_util.h" #include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" #include "test/gtest.h" #include "test/test_flags.h" #include "test/testsupport/file_utils.h" -#include "test/testsupport/frame_reader.h" +#include "test/video_codec_tester.h" + +ABSL_FLAG(std::string, + video_name, + "FourPeople_1280x720_30", + "Name of input video sequence."); +ABSL_FLAG(std::string, + encoder, + "libaom-av1", + "Encoder: libaom-av1, libvpx-vp9, libvpx-vp8, openh264, hw-vp8, " + "hw-vp9, hw-av1, hw-h264, hw-h265"); +ABSL_FLAG(std::string, + decoder, + "dav1d", + "Decoder: dav1d, libvpx-vp9, libvpx-vp8, ffmpeg-h264, hw-vp8, " + "hw-vp9, hw-av1, hw-h264, hw-h265"); +ABSL_FLAG(std::string, scalability_mode, "L1T1", "Scalability mode."); +ABSL_FLAG(int, width, 1280, "Width."); +ABSL_FLAG(int, height, 720, "Height."); +ABSL_FLAG(std::vector<std::string>, + bitrate_kbps, + {"1024"}, + "Encode target bitrate per layer (l0t0,l0t1,...l1t0,l1t1 and so on) " + "in kbps."); +ABSL_FLAG(double, + framerate_fps, + 30.0, + "Encode target frame rate of the top temporal layer in fps."); +ABSL_FLAG(int, num_frames, 300, "Number of frames to encode and/or decode."); +ABSL_FLAG(std::string, test_name, "", "Test name."); +ABSL_FLAG(bool, dump_decoder_input, false, "Dump decoder input."); +ABSL_FLAG(bool, dump_decoder_output, false, "Dump decoder output."); +ABSL_FLAG(bool, dump_encoder_input, false, "Dump encoder input."); +ABSL_FLAG(bool, dump_encoder_output, false, "Dump encoder output."); +ABSL_FLAG(bool, write_csv, false, "Write metrics to a CSV file."); namespace webrtc { namespace test { @@ -50,6 +71,10 @@ namespace test { namespace { using ::testing::Combine; using ::testing::Values; +using VideoSourceSettings = VideoCodecTester::VideoSourceSettings; +using EncodingSettings = VideoCodecTester::EncodingSettings; +using VideoCodecStats = VideoCodecTester::VideoCodecStats; +using Filter = VideoCodecStats::Filter; using PacingMode = VideoCodecTester::PacingSettings::PacingMode; struct VideoInfo { @@ -58,405 +83,93 @@ struct VideoInfo { Frequency framerate; }; -struct LayerId { - int spatial_idx; - int temporal_idx; - - bool operator==(const LayerId& o) const { - return spatial_idx == o.spatial_idx && temporal_idx == o.temporal_idx; - } - - bool operator<(const LayerId& o) const { - if (spatial_idx < o.spatial_idx) - return true; - if (spatial_idx == o.spatial_idx && temporal_idx < o.temporal_idx) - return true; - return false; - } -}; - -struct EncodingSettings { - ScalabilityMode scalability_mode; - struct LayerSettings { - Resolution resolution; - Frequency framerate; - DataRate bitrate; - }; - std::map<LayerId, LayerSettings> layer_settings; - - bool IsSameSettings(const EncodingSettings& other) const { - if (scalability_mode != other.scalability_mode) { - return false; - } - - for (auto [layer_id, layer] : layer_settings) { - const auto& other_layer = other.layer_settings.at(layer_id); - if (layer.resolution != other_layer.resolution) { - return false; - } - } - - return true; - } - - bool IsSameRate(const EncodingSettings& other) const { - for (auto [layer_id, layer] : layer_settings) { - const auto& other_layer = other.layer_settings.at(layer_id); - if (layer.bitrate != other_layer.bitrate || - layer.framerate != other_layer.framerate) { - return false; - } - } - - return true; - } -}; - -const VideoInfo kFourPeople_1280x720_30 = { - .name = "FourPeople_1280x720_30", - .resolution = {.width = 1280, .height = 720}, - .framerate = Frequency::Hertz(30)}; - -class TestRawVideoSource : public VideoCodecTester::RawVideoSource { - public: - static constexpr Frequency k90kHz = Frequency::Hertz(90000); - - TestRawVideoSource(VideoInfo video_info, - const std::map<int, EncodingSettings>& frame_settings, - int num_frames) - : video_info_(video_info), - frame_settings_(frame_settings), - num_frames_(num_frames), - frame_num_(0), - // Start with non-zero timestamp to force using frame RTP timestamps in - // IvfFrameWriter. - timestamp_rtp_(90000) { - // Ensure settings for the first frame are provided. - RTC_CHECK_GT(frame_settings_.size(), 0u); - RTC_CHECK_EQ(frame_settings_.begin()->first, 0); - - frame_reader_ = CreateYuvFrameReader( - ResourcePath(video_info_.name, "yuv"), video_info_.resolution, - YuvFrameReaderImpl::RepeatMode::kPingPong); - RTC_CHECK(frame_reader_); - } - - // Pulls next frame. Frame RTP timestamp is set accordingly to - // `EncodingSettings::framerate`. - absl::optional<VideoFrame> PullFrame() override { - if (frame_num_ >= num_frames_) { - return absl::nullopt; // End of stream. - } - - const EncodingSettings& encoding_settings = - std::prev(frame_settings_.upper_bound(frame_num_))->second; - - Resolution resolution = - encoding_settings.layer_settings.begin()->second.resolution; - Frequency framerate = - encoding_settings.layer_settings.begin()->second.framerate; - - int pulled_frame; - auto buffer = frame_reader_->PullFrame( - &pulled_frame, resolution, - {.num = static_cast<int>(framerate.millihertz()), - .den = static_cast<int>(video_info_.framerate.millihertz())}); - RTC_CHECK(buffer) << "Cannot pull frame " << frame_num_; - - auto frame = VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp_) - .set_timestamp_us((timestamp_rtp_ / k90kHz).us()) - .build(); - - pulled_frames_[timestamp_rtp_] = pulled_frame; - timestamp_rtp_ += k90kHz / framerate; - ++frame_num_; - - return frame; - } - - // Reads frame specified by `timestamp_rtp`, scales it to `resolution` and - // returns. Frame with the given `timestamp_rtp` is expected to be pulled - // before. - VideoFrame GetFrame(uint32_t timestamp_rtp, Resolution resolution) override { - RTC_CHECK(pulled_frames_.find(timestamp_rtp) != pulled_frames_.end()) - << "Frame with RTP timestamp " << timestamp_rtp - << " was not pulled before"; - auto buffer = - frame_reader_->ReadFrame(pulled_frames_[timestamp_rtp], resolution); - return VideoFrame::Builder() - .set_video_frame_buffer(buffer) - .set_timestamp_rtp(timestamp_rtp) - .build(); - } - - protected: - VideoInfo video_info_; - std::unique_ptr<FrameReader> frame_reader_; - const std::map<int, EncodingSettings>& frame_settings_; - int num_frames_; - int frame_num_; - uint32_t timestamp_rtp_; - std::map<uint32_t, int> pulled_frames_; -}; - -class TestEncoder : public VideoCodecTester::Encoder, - public EncodedImageCallback { - public: - TestEncoder(std::unique_ptr<VideoEncoder> encoder, - const std::string codec_type, - const std::map<int, EncodingSettings>& frame_settings) - : encoder_(std::move(encoder)), - codec_type_(codec_type), - frame_settings_(frame_settings), - frame_num_(0) { - // Ensure settings for the first frame is provided. - RTC_CHECK_GT(frame_settings_.size(), 0u); - RTC_CHECK_EQ(frame_settings_.begin()->first, 0); - - encoder_->RegisterEncodeCompleteCallback(this); - } - - void Initialize() override { - const EncodingSettings& first_frame_settings = frame_settings_.at(0); - Configure(first_frame_settings); - SetRates(first_frame_settings); - } - - void Encode(const VideoFrame& frame, EncodeCallback callback) override { - { - MutexLock lock(&mutex_); - callbacks_[frame.timestamp()] = std::move(callback); - } - - if (auto fs = frame_settings_.find(frame_num_); - fs != frame_settings_.begin() && fs != frame_settings_.end()) { - if (!fs->second.IsSameSettings(std::prev(fs)->second)) { - Configure(fs->second); - } else if (!fs->second.IsSameRate(std::prev(fs)->second)) { - SetRates(fs->second); - } - } - - encoder_->Encode(frame, nullptr); - ++frame_num_; - } - - void Flush() override { - // TODO(webrtc:14852): For codecs which buffer frames we need a to - // flush them to get last frames. Add such functionality to VideoEncoder - // API. On Android it will map directly to `MediaCodec.flush()`. - encoder_->Release(); - } - - VideoEncoder* encoder() { return encoder_.get(); } - - protected: - Result OnEncodedImage(const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_specific_info) override { - MutexLock lock(&mutex_); - auto cb = callbacks_.find(encoded_image.RtpTimestamp()); - RTC_CHECK(cb != callbacks_.end()); - cb->second(encoded_image); - - callbacks_.erase(callbacks_.begin(), cb); - return Result(Result::Error::OK); - } - - void Configure(const EncodingSettings& es) { - VideoCodec vc; - const EncodingSettings::LayerSettings& layer_settings = - es.layer_settings.begin()->second; - vc.width = layer_settings.resolution.width; - vc.height = layer_settings.resolution.height; - const DataRate& bitrate = layer_settings.bitrate; - vc.startBitrate = bitrate.kbps(); - vc.maxBitrate = bitrate.kbps(); - vc.minBitrate = 0; - vc.maxFramerate = static_cast<uint32_t>(layer_settings.framerate.hertz()); - vc.active = true; - vc.qpMax = 63; - vc.numberOfSimulcastStreams = 0; - vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; - vc.SetFrameDropEnabled(true); - vc.SetScalabilityMode(es.scalability_mode); - - vc.codecType = PayloadStringToCodecType(codec_type_); - if (vc.codecType == kVideoCodecVP8) { - *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); - } else if (vc.codecType == kVideoCodecVP9) { - *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); - } else if (vc.codecType == kVideoCodecH264) { - *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); - } - - VideoEncoder::Settings ves( - VideoEncoder::Capabilities(/*loss_notification=*/false), - /*number_of_cores=*/1, - /*max_payload_size=*/1440); - - int result = encoder_->InitEncode(&vc, ves); - ASSERT_EQ(result, WEBRTC_VIDEO_CODEC_OK); - - SetRates(es); - } - - void SetRates(const EncodingSettings& es) { - VideoEncoder::RateControlParameters rc; - int num_spatial_layers = - ScalabilityModeToNumSpatialLayers(es.scalability_mode); - int num_temporal_layers = - ScalabilityModeToNumSpatialLayers(es.scalability_mode); - for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { - for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { - auto layer_settings = - es.layer_settings.find({.spatial_idx = sidx, .temporal_idx = tidx}); - RTC_CHECK(layer_settings != es.layer_settings.end()) - << "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set"; - rc.bitrate.SetBitrate(sidx, tidx, layer_settings->second.bitrate.bps()); - } - } - - rc.framerate_fps = - es.layer_settings.begin()->second.framerate.millihertz() / 1000.0; - encoder_->SetRates(rc); - } - - std::unique_ptr<VideoEncoder> encoder_; - const std::string codec_type_; - const std::map<int, EncodingSettings>& frame_settings_; - int frame_num_; - std::map<uint32_t, EncodeCallback> callbacks_ RTC_GUARDED_BY(mutex_); - Mutex mutex_; -}; - -class TestDecoder : public VideoCodecTester::Decoder, - public DecodedImageCallback { - public: - TestDecoder(std::unique_ptr<VideoDecoder> decoder, - const std::string codec_type) - : decoder_(std::move(decoder)), codec_type_(codec_type) { - decoder_->RegisterDecodeCompleteCallback(this); - } - - void Initialize() override { - VideoDecoder::Settings ds; - ds.set_codec_type(PayloadStringToCodecType(codec_type_)); - ds.set_number_of_cores(1); - ds.set_max_render_resolution({1280, 720}); - - bool result = decoder_->Configure(ds); - ASSERT_TRUE(result); - } - - void Decode(const EncodedImage& frame, DecodeCallback callback) override { - { - MutexLock lock(&mutex_); - callbacks_[frame.RtpTimestamp()] = std::move(callback); - } - - decoder_->Decode(frame, /*render_time_ms=*/0); - } - - void Flush() override { - // TODO(webrtc:14852): For codecs which buffer frames we need a to - // flush them to get last frames. Add such functionality to VideoDecoder - // API. On Android it will map directly to `MediaCodec.flush()`. - decoder_->Release(); - } - - VideoDecoder* decoder() { return decoder_.get(); } - - protected: - int Decoded(VideoFrame& decoded_frame) override { - MutexLock lock(&mutex_); - auto cb = callbacks_.find(decoded_frame.timestamp()); - RTC_CHECK(cb != callbacks_.end()); - cb->second(decoded_frame); +const std::map<std::string, VideoInfo> kRawVideos = { + {"FourPeople_1280x720_30", + {.name = "FourPeople_1280x720_30", + .resolution = {.width = 1280, .height = 720}, + .framerate = Frequency::Hertz(30)}}, + {"vidyo1_1280x720_30", + {.name = "vidyo1_1280x720_30", + .resolution = {.width = 1280, .height = 720}, + .framerate = Frequency::Hertz(30)}}, + {"vidyo4_1280x720_30", + {.name = "vidyo4_1280x720_30", + .resolution = {.width = 1280, .height = 720}, + .framerate = Frequency::Hertz(30)}}, + {"KristenAndSara_1280x720_30", + {.name = "KristenAndSara_1280x720_30", + .resolution = {.width = 1280, .height = 720}, + .framerate = Frequency::Hertz(30)}}, + {"Johnny_1280x720_30", + {.name = "Johnny_1280x720_30", + .resolution = {.width = 1280, .height = 720}, + .framerate = Frequency::Hertz(30)}}}; + +static constexpr Frequency k90kHz = Frequency::Hertz(90000); + +std::string CodecNameToCodecType(std::string name) { + if (name.find("av1") != std::string::npos) { + return "AV1"; + } + if (name.find("vp9") != std::string::npos) { + return "VP9"; + } + if (name.find("vp8") != std::string::npos) { + return "VP8"; + } + if (name.find("h264") != std::string::npos) { + return "H264"; + } + if (name.find("h265") != std::string::npos) { + return "H265"; + } + RTC_CHECK_NOTREACHED(); +} - callbacks_.erase(callbacks_.begin(), cb); - return WEBRTC_VIDEO_CODEC_OK; +// TODO(webrtc:14852): Make Create[Encoder,Decoder]Factory to work with codec +// name directly. +std::string CodecNameToCodecImpl(std::string name) { + if (name.find("hw") != std::string::npos) { + return "mediacodec"; } - - std::unique_ptr<VideoDecoder> decoder_; - const std::string codec_type_; - std::map<uint32_t, DecodeCallback> callbacks_ RTC_GUARDED_BY(mutex_); - Mutex mutex_; -}; - -std::unique_ptr<TestRawVideoSource> CreateVideoSource( - const VideoInfo& video, - const std::map<int, EncodingSettings>& frame_settings, - int num_frames) { - return std::make_unique<TestRawVideoSource>(video, frame_settings, - num_frames); + return "builtin"; } -std::unique_ptr<TestEncoder> CreateEncoder( - std::string type, - std::string impl, - const std::map<int, EncodingSettings>& frame_settings) { - std::unique_ptr<VideoEncoderFactory> factory; +std::unique_ptr<VideoEncoderFactory> CreateEncoderFactory(std::string impl) { if (impl == "builtin") { - factory = std::make_unique<InternalEncoderFactory>(); - } else if (impl == "mediacodec") { + return CreateBuiltinVideoEncoderFactory(); + } #if defined(WEBRTC_ANDROID) - InitializeAndroidObjects(); - factory = CreateAndroidEncoderFactory(); + InitializeAndroidObjects(); + return CreateAndroidEncoderFactory(); +#else + return nullptr; #endif - } - std::unique_ptr<VideoEncoder> encoder = - factory->CreateVideoEncoder(SdpVideoFormat(type)); - if (encoder == nullptr) { - return nullptr; - } - return std::make_unique<TestEncoder>(std::move(encoder), type, - frame_settings); } -std::unique_ptr<TestDecoder> CreateDecoder(std::string type, std::string impl) { - std::unique_ptr<VideoDecoderFactory> factory; +std::unique_ptr<VideoDecoderFactory> CreateDecoderFactory(std::string impl) { if (impl == "builtin") { - factory = std::make_unique<InternalDecoderFactory>(); - } else if (impl == "mediacodec") { + return CreateBuiltinVideoDecoderFactory(); + } #if defined(WEBRTC_ANDROID) - InitializeAndroidObjects(); - factory = CreateAndroidDecoderFactory(); + InitializeAndroidObjects(); + return CreateAndroidDecoderFactory(); +#else + return nullptr; #endif - } - std::unique_ptr<VideoDecoder> decoder = - factory->CreateVideoDecoder(SdpVideoFormat(type)); - if (decoder == nullptr) { - return nullptr; - } - return std::make_unique<TestDecoder>(std::move(decoder), type); } -void SetTargetRates(const std::map<int, EncodingSettings>& frame_settings, - std::vector<VideoCodecStats::Frame>& frames) { - for (VideoCodecStats::Frame& f : frames) { - const EncodingSettings& encoding_settings = - std::prev(frame_settings.upper_bound(f.frame_num))->second; - LayerId layer_id = {.spatial_idx = f.spatial_idx, - .temporal_idx = f.temporal_idx}; - RTC_CHECK(encoding_settings.layer_settings.find(layer_id) != - encoding_settings.layer_settings.end()) - << "Frame frame_num=" << f.frame_num - << " belongs to spatial_idx=" << f.spatial_idx - << " temporal_idx=" << f.temporal_idx - << " but settings for this layer are not provided."; - const EncodingSettings::LayerSettings& layer_settings = - encoding_settings.layer_settings.at(layer_id); - f.target_bitrate = layer_settings.bitrate; - f.target_framerate = layer_settings.framerate; +std::string TestName() { + std::string test_name = absl::GetFlag(FLAGS_test_name); + if (!test_name.empty()) { + return test_name; } + return ::testing::UnitTest::GetInstance()->current_test_info()->name(); } std::string TestOutputPath() { std::string output_path = - OutputPath() + - ::testing::UnitTest::GetInstance()->current_test_info()->name(); + (rtc::StringBuilder() << OutputPath() << TestName()).str(); std::string output_dir = DirName(output_path); bool result = CreateDir(output_dir); RTC_CHECK(result) << "Cannot create " << output_dir; @@ -465,116 +178,120 @@ std::string TestOutputPath() { } // namespace std::unique_ptr<VideoCodecStats> RunEncodeDecodeTest( - std::string codec_type, std::string codec_impl, const VideoInfo& video_info, - const std::map<int, EncodingSettings>& frame_settings, - int num_frames, - bool save_codec_input, - bool save_codec_output) { - std::unique_ptr<TestRawVideoSource> video_source = - CreateVideoSource(video_info, frame_settings, num_frames); - - std::unique_ptr<TestEncoder> encoder = - CreateEncoder(codec_type, codec_impl, frame_settings); - if (encoder == nullptr) { + const std::map<uint32_t, EncodingSettings>& encoding_settings) { + VideoSourceSettings source_settings{ + .file_path = ResourcePath(video_info.name, "yuv"), + .resolution = video_info.resolution, + .framerate = video_info.framerate}; + + const SdpVideoFormat& sdp_video_format = + encoding_settings.begin()->second.sdp_video_format; + + std::unique_ptr<VideoEncoderFactory> encoder_factory = + CreateEncoderFactory(codec_impl); + if (!encoder_factory + ->QueryCodecSupport(sdp_video_format, + /*scalability_mode=*/absl::nullopt) + .is_supported) { + RTC_LOG(LS_WARNING) << "No encoder for video format " + << sdp_video_format.ToString(); return nullptr; } - std::unique_ptr<TestDecoder> decoder = CreateDecoder(codec_type, codec_impl); - if (decoder == nullptr) { - // If platform decoder is not available try built-in one. - if (codec_impl == "builtin") { - return nullptr; - } - - decoder = CreateDecoder(codec_type, "builtin"); - if (decoder == nullptr) { + std::unique_ptr<VideoDecoderFactory> decoder_factory = + CreateDecoderFactory(codec_impl); + if (!decoder_factory + ->QueryCodecSupport(sdp_video_format, + /*reference_scaling=*/false) + .is_supported) { + decoder_factory = CreateDecoderFactory("builtin"); + if (!decoder_factory + ->QueryCodecSupport(sdp_video_format, + /*reference_scaling=*/false) + .is_supported) { + RTC_LOG(LS_WARNING) << "No decoder for video format " + << sdp_video_format.ToString(); return nullptr; } } - RTC_LOG(LS_INFO) << "Encoder implementation: " - << encoder->encoder()->GetEncoderInfo().implementation_name; - RTC_LOG(LS_INFO) << "Decoder implementation: " - << decoder->decoder()->GetDecoderInfo().implementation_name; + std::string output_path = TestOutputPath(); VideoCodecTester::EncoderSettings encoder_settings; - encoder_settings.pacing.mode = - encoder->encoder()->GetEncoderInfo().is_hardware_accelerated - ? PacingMode::kRealTime - : PacingMode::kNoPacing; + encoder_settings.pacing_settings.mode = + codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + if (absl::GetFlag(FLAGS_dump_encoder_input)) { + encoder_settings.encoder_input_base_path = output_path + "_enc_input"; + } + if (absl::GetFlag(FLAGS_dump_encoder_output)) { + encoder_settings.encoder_output_base_path = output_path + "_enc_output"; + } VideoCodecTester::DecoderSettings decoder_settings; - decoder_settings.pacing.mode = - decoder->decoder()->GetDecoderInfo().is_hardware_accelerated - ? PacingMode::kRealTime - : PacingMode::kNoPacing; - - std::string output_path = TestOutputPath(); - if (save_codec_input) { - encoder_settings.encoder_input_base_path = output_path + "_enc_input"; + decoder_settings.pacing_settings.mode = + codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + if (absl::GetFlag(FLAGS_dump_decoder_input)) { decoder_settings.decoder_input_base_path = output_path + "_dec_input"; } - if (save_codec_output) { - encoder_settings.encoder_output_base_path = output_path + "_enc_output"; + if (absl::GetFlag(FLAGS_dump_decoder_output)) { decoder_settings.decoder_output_base_path = output_path + "_dec_output"; } - std::unique_ptr<VideoCodecTester> tester = CreateVideoCodecTester(); - return tester->RunEncodeDecodeTest(video_source.get(), encoder.get(), - decoder.get(), encoder_settings, - decoder_settings); + return VideoCodecTester::RunEncodeDecodeTest( + source_settings, encoder_factory.get(), decoder_factory.get(), + encoder_settings, decoder_settings, encoding_settings); } std::unique_ptr<VideoCodecStats> RunEncodeTest( std::string codec_type, std::string codec_impl, const VideoInfo& video_info, - const std::map<int, EncodingSettings>& frame_settings, - int num_frames, - bool save_codec_input, - bool save_codec_output) { - std::unique_ptr<TestRawVideoSource> video_source = - CreateVideoSource(video_info, frame_settings, num_frames); - - std::unique_ptr<TestEncoder> encoder = - CreateEncoder(codec_type, codec_impl, frame_settings); - if (encoder == nullptr) { + const std::map<uint32_t, EncodingSettings>& encoding_settings) { + VideoSourceSettings source_settings{ + .file_path = ResourcePath(video_info.name, "yuv"), + .resolution = video_info.resolution, + .framerate = video_info.framerate}; + + const SdpVideoFormat& sdp_video_format = + encoding_settings.begin()->second.sdp_video_format; + + std::unique_ptr<VideoEncoderFactory> encoder_factory = + CreateEncoderFactory(codec_impl); + if (!encoder_factory + ->QueryCodecSupport(sdp_video_format, + /*scalability_mode=*/absl::nullopt) + .is_supported) { + RTC_LOG(LS_WARNING) << "No encoder for video format " + << sdp_video_format.ToString(); return nullptr; } - RTC_LOG(LS_INFO) << "Encoder implementation: " - << encoder->encoder()->GetEncoderInfo().implementation_name; - - VideoCodecTester::EncoderSettings encoder_settings; - encoder_settings.pacing.mode = - encoder->encoder()->GetEncoderInfo().is_hardware_accelerated - ? PacingMode::kRealTime - : PacingMode::kNoPacing; - std::string output_path = TestOutputPath(); - if (save_codec_input) { + VideoCodecTester::EncoderSettings encoder_settings; + encoder_settings.pacing_settings.mode = + codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + if (absl::GetFlag(FLAGS_dump_encoder_input)) { encoder_settings.encoder_input_base_path = output_path + "_enc_input"; } - if (save_codec_output) { + if (absl::GetFlag(FLAGS_dump_encoder_output)) { encoder_settings.encoder_output_base_path = output_path + "_enc_output"; } - std::unique_ptr<VideoCodecTester> tester = CreateVideoCodecTester(); - return tester->RunEncodeTest(video_source.get(), encoder.get(), - encoder_settings); + return VideoCodecTester::RunEncodeTest(source_settings, encoder_factory.get(), + encoder_settings, encoding_settings); } -class SpatialQualityTest : public ::testing::TestWithParam< - std::tuple</*codec_type=*/std::string, - /*codec_impl=*/std::string, - VideoInfo, - std::tuple</*width=*/int, - /*height=*/int, - /*framerate_fps=*/double, - /*bitrate_kbps=*/int, - /*min_psnr=*/double>>> { +class SpatialQualityTest : public ::testing::TestWithParam<std::tuple< + /*codec_type=*/std::string, + /*codec_impl=*/std::string, + VideoInfo, + std::tuple</*width=*/int, + /*height=*/int, + /*framerate_fps=*/double, + /*bitrate_kbps=*/int, + /*expected_min_psnr=*/double>>> { public: static std::string TestParamsToString( const ::testing::TestParamInfo<SpatialQualityTest::ParamType>& info) { @@ -590,41 +307,35 @@ class SpatialQualityTest : public ::testing::TestWithParam< TEST_P(SpatialQualityTest, SpatialQuality) { auto [codec_type, codec_impl, video_info, coding_settings] = GetParam(); - auto [width, height, framerate_fps, bitrate_kbps, psnr] = coding_settings; - - std::map<int, EncodingSettings> frame_settings = { - {0, - {.scalability_mode = ScalabilityMode::kL1T1, - .layer_settings = { - {LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = width, .height = height}, - .framerate = Frequency::MilliHertz(1000 * framerate_fps), - .bitrate = DataRate::KilobitsPerSec(bitrate_kbps)}}}}}}; - + auto [width, height, framerate_fps, bitrate_kbps, expected_min_psnr] = + coding_settings; int duration_s = 10; int num_frames = duration_s * framerate_fps; - std::unique_ptr<VideoCodecStats> stats = RunEncodeDecodeTest( - codec_type, codec_impl, video_info, frame_settings, num_frames, - /*save_codec_input=*/false, /*save_codec_output=*/false); + std::map<uint32_t, EncodingSettings> frames_settings = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", width, height, + {bitrate_kbps}, framerate_fps, num_frames); + + std::unique_ptr<VideoCodecStats> stats = + RunEncodeDecodeTest(codec_impl, video_info, frames_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { - std::vector<VideoCodecStats::Frame> frames = stats->Slice(); - SetTargetRates(frame_settings, frames); - stream = stats->Aggregate(frames); + stream = stats->Aggregate(Filter{}); if (absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { - EXPECT_GE(stream.psnr.y.GetAverage(), psnr); + EXPECT_GE(stream.psnr.y.GetAverage(), expected_min_psnr); } } stream.LogMetrics( GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), + /*prefix=*/"", /*metadata=*/ - {{"codec_type", codec_type}, - {"codec_impl", codec_impl}, - {"video_name", video_info.name}}); + {{"video_name", video_info.name}, + {"codec_type", codec_type}, + {"codec_impl", codec_impl}}); } INSTANTIATE_TEST_SUITE_P( @@ -636,7 +347,7 @@ INSTANTIATE_TEST_SUITE_P( #else Values("builtin"), #endif - Values(kFourPeople_1280x720_30), + Values(kRawVideos.at("FourPeople_1280x720_30")), Values(std::make_tuple(320, 180, 30, 32, 28), std::make_tuple(320, 180, 30, 64, 30), std::make_tuple(320, 180, 30, 128, 33), @@ -671,33 +382,32 @@ TEST_P(BitrateAdaptationTest, BitrateAdaptation) { auto [codec_type, codec_impl, video_info, bitrate_kbps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. - int first_frame = duration_s * video_info.framerate.millihertz() / 1000; - int num_frames = 2 * duration_s * video_info.framerate.millihertz() / 1000; - - std::map<int, EncodingSettings> frame_settings = { - {0, - {.layer_settings = {{LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = video_info.framerate, - .bitrate = DataRate::KilobitsPerSec( - bitrate_kbps.first)}}}}}, - {first_frame, - {.layer_settings = { - {LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = video_info.framerate, - .bitrate = DataRate::KilobitsPerSec(bitrate_kbps.second)}}}}}}; - - std::unique_ptr<VideoCodecStats> stats = RunEncodeTest( - codec_type, codec_impl, video_info, frame_settings, num_frames, - /*save_codec_input=*/false, /*save_codec_output=*/false); + int num_frames = + static_cast<int>(duration_s * video_info.framerate.hertz<double>()); + + std::map<uint32_t, EncodingSettings> encoding_settings = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", + /*width=*/640, /*height=*/360, {bitrate_kbps.first}, + /*framerate_fps=*/30, num_frames); + + uint32_t initial_timestamp_rtp = + encoding_settings.rbegin()->first + k90kHz / Frequency::Hertz(30); + + std::map<uint32_t, EncodingSettings> encoding_settings2 = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", + /*width=*/640, /*height=*/360, {bitrate_kbps.second}, + /*framerate_fps=*/30, num_frames, initial_timestamp_rtp); + + encoding_settings.merge(encoding_settings2); + + std::unique_ptr<VideoCodecStats> stats = + RunEncodeTest(codec_type, codec_impl, video_info, encoding_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { - std::vector<VideoCodecStats::Frame> frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - stream = stats->Aggregate(frames); + stream = stats->Aggregate({.min_timestamp_rtp = initial_timestamp_rtp}); if (absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); @@ -707,6 +417,7 @@ TEST_P(BitrateAdaptationTest, BitrateAdaptation) { stream.LogMetrics( GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), + /*prefix=*/"", /*metadata=*/ {{"codec_type", codec_type}, {"codec_impl", codec_impl}, @@ -715,18 +426,18 @@ TEST_P(BitrateAdaptationTest, BitrateAdaptation) { std::to_string(bitrate_kbps.second)}}); } -INSTANTIATE_TEST_SUITE_P(All, - BitrateAdaptationTest, - Combine(Values("AV1", "VP9", "VP8", "H264", "H265"), +INSTANTIATE_TEST_SUITE_P( + All, + BitrateAdaptationTest, + Combine(Values("AV1", "VP9", "VP8", "H264", "H265"), #if defined(WEBRTC_ANDROID) - Values("builtin", "mediacodec"), + Values("builtin", "mediacodec"), #else - Values("builtin"), + Values("builtin"), #endif - Values(kFourPeople_1280x720_30), - Values(std::pair(1024, 512), - std::pair(512, 1024))), - BitrateAdaptationTest::TestParamsToString); + Values(kRawVideos.at("FourPeople_1280x720_30")), + Values(std::pair(1024, 512), std::pair(512, 1024))), + BitrateAdaptationTest::TestParamsToString); class FramerateAdaptationTest : public ::testing::TestWithParam<std::tuple</*codec_type=*/std::string, @@ -749,34 +460,34 @@ TEST_P(FramerateAdaptationTest, FramerateAdaptation) { auto [codec_type, codec_impl, video_info, framerate_fps] = GetParam(); int duration_s = 10; // Duration of fixed rate interval. - int first_frame = static_cast<int>(duration_s * framerate_fps.first); - int num_frames = static_cast<int>( - duration_s * (framerate_fps.first + framerate_fps.second)); - - std::map<int, EncodingSettings> frame_settings = { - {0, - {.layer_settings = {{LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = Frequency::MilliHertz( - 1000 * framerate_fps.first), - .bitrate = DataRate::KilobitsPerSec(512)}}}}}, - {first_frame, - {.layer_settings = { - {LayerId{.spatial_idx = 0, .temporal_idx = 0}, - {.resolution = {.width = 640, .height = 360}, - .framerate = Frequency::MilliHertz(1000 * framerate_fps.second), - .bitrate = DataRate::KilobitsPerSec(512)}}}}}}; - - std::unique_ptr<VideoCodecStats> stats = RunEncodeTest( - codec_type, codec_impl, video_info, frame_settings, num_frames, - /*save_codec_input=*/false, /*save_codec_output=*/false); + + std::map<uint32_t, EncodingSettings> encoding_settings = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", + /*width=*/640, /*height=*/360, + /*layer_bitrates_kbps=*/{512}, framerate_fps.first, + static_cast<int>(duration_s * framerate_fps.first)); + + uint32_t initial_timestamp_rtp = + encoding_settings.rbegin()->first + + k90kHz / Frequency::Hertz(framerate_fps.first); + + std::map<uint32_t, EncodingSettings> encoding_settings2 = + VideoCodecTester::CreateEncodingSettings( + codec_type, /*scalability_mode=*/"L1T1", /*width=*/640, + /*height=*/360, + /*layer_bitrates_kbps=*/{512}, framerate_fps.second, + static_cast<int>(duration_s * framerate_fps.second), + initial_timestamp_rtp); + + encoding_settings.merge(encoding_settings2); + + std::unique_ptr<VideoCodecStats> stats = + RunEncodeTest(codec_type, codec_impl, video_info, encoding_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { - std::vector<VideoCodecStats::Frame> frames = - stats->Slice(VideoCodecStats::Filter{.first_frame = first_frame}); - SetTargetRates(frame_settings, frames); - stream = stats->Aggregate(frames); + stream = stats->Aggregate({.min_timestamp_rtp = initial_timestamp_rtp}); if (absl::GetFlag(FLAGS_webrtc_quick_perf_test)) { EXPECT_NEAR(stream.bitrate_mismatch_pct.GetAverage(), 0, 10); EXPECT_NEAR(stream.framerate_mismatch_pct.GetAverage(), 0, 10); @@ -786,6 +497,7 @@ TEST_P(FramerateAdaptationTest, FramerateAdaptation) { stream.LogMetrics( GetGlobalMetricsLogger(), ::testing::UnitTest::GetInstance()->current_test_info()->name(), + /*prefix=*/"", /*metadata=*/ {{"codec_type", codec_type}, {"codec_impl", codec_impl}, @@ -794,17 +506,71 @@ TEST_P(FramerateAdaptationTest, FramerateAdaptation) { std::to_string(framerate_fps.second)}}); } -INSTANTIATE_TEST_SUITE_P(All, - FramerateAdaptationTest, - Combine(Values("AV1", "VP9", "VP8", "H264", "H265"), +INSTANTIATE_TEST_SUITE_P( + All, + FramerateAdaptationTest, + Combine(Values("AV1", "VP9", "VP8", "H264", "H265"), #if defined(WEBRTC_ANDROID) - Values("builtin", "mediacodec"), + Values("builtin", "mediacodec"), #else - Values("builtin"), + Values("builtin"), #endif - Values(kFourPeople_1280x720_30), - Values(std::pair(30, 15), std::pair(15, 30))), - FramerateAdaptationTest::TestParamsToString); + Values(kRawVideos.at("FourPeople_1280x720_30")), + Values(std::pair(30, 15), std::pair(15, 30))), + FramerateAdaptationTest::TestParamsToString); + +TEST(VideoCodecTest, DISABLED_EncodeDecode) { + std::vector<std::string> bitrate_str = absl::GetFlag(FLAGS_bitrate_kbps); + std::vector<int> bitrate_kbps; + std::transform(bitrate_str.begin(), bitrate_str.end(), + std::back_inserter(bitrate_kbps), + [](const std::string& str) { return std::stoi(str); }); + + std::map<uint32_t, EncodingSettings> frames_settings = + VideoCodecTester::CreateEncodingSettings( + CodecNameToCodecType(absl::GetFlag(FLAGS_encoder)), + absl::GetFlag(FLAGS_scalability_mode), absl::GetFlag(FLAGS_width), + absl::GetFlag(FLAGS_height), {bitrate_kbps}, + absl::GetFlag(FLAGS_framerate_fps), absl::GetFlag(FLAGS_num_frames)); + + // TODO(webrtc:14852): Pass encoder and decoder names directly, and update + // logged test name (implies lossing history in the chromeperf dashboard). + // Sync with changes in Stream::LogMetrics (see TODOs there). + std::unique_ptr<VideoCodecStats> stats = RunEncodeDecodeTest( + CodecNameToCodecImpl(absl::GetFlag(FLAGS_encoder)), + kRawVideos.at(absl::GetFlag(FLAGS_video_name)), frames_settings); + ASSERT_NE(nullptr, stats); + + // Log unsliced metrics. + VideoCodecStats::Stream stream = stats->Aggregate(Filter{}); + stream.LogMetrics(GetGlobalMetricsLogger(), TestName(), /*prefix=*/"", + /*metadata=*/{}); + + // Log metrics sliced on spatial and temporal layer. + ScalabilityMode scalability_mode = + *ScalabilityModeFromString(absl::GetFlag(FLAGS_scalability_mode)); + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + std::string metric_name_prefix = + (rtc::StringBuilder() << "s" << sidx << "t" << tidx << "_").str(); + stream = stats->Aggregate( + {.layer_id = {{.spatial_idx = sidx, .temporal_idx = tidx}}}); + stream.LogMetrics(GetGlobalMetricsLogger(), TestName(), + metric_name_prefix, + /*metadata=*/{}); + } + } + + if (absl::GetFlag(FLAGS_write_csv)) { + stats->LogMetrics( + (rtc::StringBuilder() << TestOutputPath() << ".csv").str(), + stats->Slice(Filter{}, /*merge=*/false), /*metadata=*/ + {{"test_name", TestName()}}); + } +} } // namespace test |