diff options
Diffstat (limited to '')
10 files changed, 1089 insertions, 0 deletions
diff --git a/third_party/libwebrtc/api/test/video/BUILD.gn b/third_party/libwebrtc/api/test/video/BUILD.gn new file mode 100644 index 0000000000..d24ffa5fa6 --- /dev/null +++ b/third_party/libwebrtc/api/test/video/BUILD.gn @@ -0,0 +1,31 @@ +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_library("function_video_factory") { + visibility = [ "*" ] + testonly = true + public = [ + "function_video_decoder_factory.h", + "function_video_encoder_factory.h", + ] + + deps = [ + "../../../rtc_base:checks", + "../../video_codecs:video_codecs_api", + ] +} + +rtc_library("video_frame_writer") { + visibility = [ "*" ] + testonly = true + public = [ "video_frame_writer.h" ] + + deps = [ "../../video:video_frame" ] +} diff --git a/third_party/libwebrtc/api/test/video/function_video_decoder_factory.h b/third_party/libwebrtc/api/test/video/function_video_decoder_factory.h new file mode 100644 index 0000000000..2145c71bff --- /dev/null +++ b/third_party/libwebrtc/api/test/video/function_video_decoder_factory.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_FUNCTION_VIDEO_DECODER_FACTORY_H_ +#define API_TEST_VIDEO_FUNCTION_VIDEO_DECODER_FACTORY_H_ + +#include <functional> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +// A decoder factory producing decoders by calling a supplied create function. +class FunctionVideoDecoderFactory final : public VideoDecoderFactory { + public: + explicit FunctionVideoDecoderFactory( + std::function<std::unique_ptr<VideoDecoder>()> create) + : create_([create = std::move(create)](const SdpVideoFormat&) { + return create(); + }) {} + explicit FunctionVideoDecoderFactory( + std::function<std::unique_ptr<VideoDecoder>(const SdpVideoFormat&)> + create) + : create_(std::move(create)) {} + FunctionVideoDecoderFactory( + std::function<std::unique_ptr<VideoDecoder>()> create, + std::vector<SdpVideoFormat> sdp_video_formats) + : create_([create = std::move(create)](const SdpVideoFormat&) { + return create(); + }), + sdp_video_formats_(std::move(sdp_video_formats)) {} + + std::vector<SdpVideoFormat> GetSupportedFormats() const override { + return sdp_video_formats_; + } + + std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) override { + return create_(format); + } + + private: + const std::function<std::unique_ptr<VideoDecoder>(const SdpVideoFormat&)> + create_; + const std::vector<SdpVideoFormat> sdp_video_formats_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_FUNCTION_VIDEO_DECODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/video/function_video_encoder_factory.h b/third_party/libwebrtc/api/test/video/function_video_encoder_factory.h new file mode 100644 index 0000000000..98ece2bc94 --- /dev/null +++ b/third_party/libwebrtc/api/test/video/function_video_encoder_factory.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_FUNCTION_VIDEO_ENCODER_FACTORY_H_ +#define API_TEST_VIDEO_FUNCTION_VIDEO_ENCODER_FACTORY_H_ + +#include <functional> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +// An encoder factory producing encoders by calling a supplied create +// function. +class FunctionVideoEncoderFactory final : public VideoEncoderFactory { + public: + explicit FunctionVideoEncoderFactory( + std::function<std::unique_ptr<VideoEncoder>()> create) + : create_([create = std::move(create)](const SdpVideoFormat&) { + return create(); + }) {} + explicit FunctionVideoEncoderFactory( + std::function<std::unique_ptr<VideoEncoder>(const SdpVideoFormat&)> + create) + : create_(std::move(create)) {} + + // Unused by tests. + std::vector<SdpVideoFormat> GetSupportedFormats() const override { + RTC_DCHECK_NOTREACHED(); + return {}; + } + + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override { + return create_(format); + } + + private: + const std::function<std::unique_ptr<VideoEncoder>(const SdpVideoFormat&)> + create_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_FUNCTION_VIDEO_ENCODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/video/video_frame_writer.h b/third_party/libwebrtc/api/test/video/video_frame_writer.h new file mode 100644 index 0000000000..ac72534890 --- /dev/null +++ b/third_party/libwebrtc/api/test/video/video_frame_writer.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_VIDEO_FRAME_WRITER_H_ +#define API_TEST_VIDEO_VIDEO_FRAME_WRITER_H_ + +#include "api/video/video_frame.h" + +namespace webrtc { +namespace test { + +class VideoFrameWriter { + public: + virtual ~VideoFrameWriter() = default; + + // Writes `VideoFrame` and returns true if operation was successful, false + // otherwise. + // + // Calling `WriteFrame` after `Close` is not allowed. + virtual bool WriteFrame(const VideoFrame& frame) = 0; + + // Closes writer and cleans up all resources. No invocations to `WriteFrame` + // are allowed after `Close` was invoked. + virtual void Close() = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_VIDEO_FRAME_WRITER_H_ diff --git a/third_party/libwebrtc/api/test/video_codec_tester.h b/third_party/libwebrtc/api/test/video_codec_tester.h new file mode 100644 index 0000000000..0eaaa1b895 --- /dev/null +++ b/third_party/libwebrtc/api/test/video_codec_tester.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_CODEC_TESTER_H_ +#define API_TEST_VIDEO_CODEC_TESTER_H_ + +#include <memory> + +#include "absl/functional/any_invocable.h" +#include "api/test/videocodec_test_stats.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video/video_frame.h" + +namespace webrtc { +namespace test { + +// Interface for a video codec tester. The interface provides minimalistic set +// of data structures that enables implementation of decode-only, encode-only +// and encode-decode tests. +class VideoCodecTester { + public: + // Pacing settings for codec input. + struct PacingSettings { + enum PacingMode { + // Pacing is not used. Frames are sent to codec back-to-back. + kNoPacing, + // Pace with the rate equal to the target video frame rate. Pacing time is + // derived from RTP timestamp. + kRealTime, + // Pace with the explicitly provided rate. + kConstantRate, + }; + PacingMode mode = PacingMode::kNoPacing; + // Pacing rate for `kConstantRate` mode. + Frequency constant_rate = Frequency::Zero(); + }; + + struct DecoderSettings { + PacingSettings pacing; + }; + + struct EncoderSettings { + PacingSettings pacing; + }; + + virtual ~VideoCodecTester() = default; + + // Interface for a raw video frames source. + class RawVideoSource { + public: + virtual ~RawVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional<VideoFrame> PullFrame() = 0; + + // Returns early pulled frame with RTP timestamp equal to `timestamp_rtp`. + virtual VideoFrame GetFrame(uint32_t timestamp_rtp, + Resolution resolution) = 0; + }; + + // Interface for a coded video frames source. + class CodedVideoSource { + public: + virtual ~CodedVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional<EncodedImage> PullFrame() = 0; + }; + + // Interface for a video encoder. + class Encoder { + public: + using EncodeCallback = + absl::AnyInvocable<void(const EncodedImage& encoded_frame)>; + + virtual ~Encoder() = default; + + virtual void Encode(const VideoFrame& frame, EncodeCallback callback) = 0; + }; + + // Interface for a video decoder. + class Decoder { + public: + using DecodeCallback = + absl::AnyInvocable<void(const VideoFrame& decoded_frame)>; + + virtual ~Decoder() = default; + + virtual void Decode(const EncodedImage& frame, DecodeCallback callback) = 0; + }; + + // Pulls coded video frames from `video_source` and passes them to `decoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr<VideoCodecTestStats> RunDecodeTest( + std::unique_ptr<CodedVideoSource> video_source, + std::unique_ptr<Decoder> decoder, + const DecoderSettings& decoder_settings) = 0; + + // Pulls raw video frames from `video_source` and passes them to `encoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr<VideoCodecTestStats> RunEncodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + const EncoderSettings& encoder_settings) = 0; + + // Pulls raw video frames from `video_source`, passes them to `encoder` and + // then passes encoded frames to `decoder`. Returns `VideoCodecTestStats` + // object that contains collected per-frame metrics. + virtual std::unique_ptr<VideoCodecTestStats> RunEncodeDecodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + std::unique_ptr<Decoder> decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_CODEC_TESTER_H_ diff --git a/third_party/libwebrtc/api/test/video_quality_analyzer_interface.h b/third_party/libwebrtc/api/test/video_quality_analyzer_interface.h new file mode 100644 index 0000000000..d35be8ca1a --- /dev/null +++ b/third_party/libwebrtc/api/test/video_quality_analyzer_interface.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_QUALITY_ANALYZER_INTERFACE_H_ +#define API_TEST_VIDEO_QUALITY_ANALYZER_INTERFACE_H_ + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/test/stats_observer_interface.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_encoder.h" + +namespace webrtc { + +// API is in development and can be changed without notice. + +// Base interface for video quality analyzer for peer connection level end-2-end +// tests. Interface has only one abstract method, which have to return frame id. +// Other methods have empty implementation by default, so user can override only +// required parts. +// +// VideoQualityAnalyzerInterface will be injected into WebRTC pipeline on both +// sides of the call. Here is video data flow in WebRTC pipeline +// +// Alice: +// ___________ ________ _________ +// | | | | | | +// | Frame |-(A)→| WebRTC |-(B)→| Video |-(C)┐ +// | Generator | | Stack | | Decoder | | +// ¯¯¯¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯¯ | +// __↓________ +// | Transport | +// | & | +// | Network | +// ¯¯|¯¯¯¯¯¯¯¯ +// Bob: | +// _______ ________ _________ | +// | | | | | | | +// | Video |←(F)-| WebRTC |←(E)-| Video |←(D)----┘ +// | Sink | | Stack | | Decoder | +// ¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯¯ +// The analyzer will be injected in all points from A to F. +class VideoQualityAnalyzerInterface + : public webrtc_pc_e2e::StatsObserverInterface { + public: + // Contains extra statistic provided by video encoder. + struct EncoderStats { + std::string encoder_name = "unknown"; + // TODO(hbos) https://crbug.com/webrtc/9547, + // https://crbug.com/webrtc/11443: improve stats API to make available + // there. + uint32_t target_encode_bitrate = 0; + // Encoder quantizer value. + int qp = -1; + }; + // Contains extra statistic provided by video decoder. + struct DecoderStats { + std::string decoder_name = "unknown"; + // Decode time provided by decoder itself. If decoder doesn’t produce such + // information can be omitted. + absl::optional<int32_t> decode_time_ms = absl::nullopt; + }; + + ~VideoQualityAnalyzerInterface() override = default; + + // Will be called by framework before test. + // `test_case_name` is name of test case, that should be used to report all + // video metrics. + // `threads_count` is number of threads that analyzer can use for heavy + // calculations. Analyzer can perform simple calculations on the calling + // thread in each method, but should remember, that it is the same thread, + // that is used in video pipeline. + virtual void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) {} + + // Will be called when frame was generated from the input stream. + // `peer_name` is name of the peer on which side frame was captured. + // Returns frame id, that will be set by framework to the frame. + virtual uint16_t OnFrameCaptured(absl::string_view peer_name, + const std::string& stream_label, + const VideoFrame& frame) = 0; + // Will be called before calling the encoder. + // `peer_name` is name of the peer on which side frame came to encoder. + virtual void OnFramePreEncode(absl::string_view peer_name, + const VideoFrame& frame) {} + // Will be called for each EncodedImage received from encoder. Single + // VideoFrame can produce multiple EncodedImages. Each encoded image will + // have id from VideoFrame. + // `peer_name` is name of the peer on which side frame was encoded. + virtual void OnFrameEncoded(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) {} + // Will be called for each frame dropped by encoder. + // `peer_name` is name of the peer on which side frame drop was detected. + virtual void OnFrameDropped(absl::string_view peer_name, + EncodedImageCallback::DropReason reason) {} + // Will be called before calling the decoder. + // `peer_name` is name of the peer on which side frame was received. + virtual void OnFramePreDecode(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image) {} + // Will be called after decoding the frame. + // `peer_name` is name of the peer on which side frame was decoded. + virtual void OnFrameDecoded(absl::string_view peer_name, + const VideoFrame& frame, + const DecoderStats& stats) {} + // Will be called when frame will be obtained from PeerConnection stack. + // `peer_name` is name of the peer on which side frame was rendered. + virtual void OnFrameRendered(absl::string_view peer_name, + const VideoFrame& frame) {} + // Will be called if encoder return not WEBRTC_VIDEO_CODEC_OK. + // All available codes are listed in + // modules/video_coding/include/video_error_codes.h + // `peer_name` is name of the peer on which side error acquired. + virtual void OnEncoderError(absl::string_view peer_name, + const VideoFrame& frame, + int32_t error_code) {} + // Will be called if decoder return not WEBRTC_VIDEO_CODEC_OK. + // All available codes are listed in + // modules/video_coding/include/video_error_codes.h + // `peer_name` is name of the peer on which side error acquired. + virtual void OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) {} + // Will be called every time new stats reports are available for the + // Peer Connection identified by `pc_label`. + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override {} + + // Will be called before test adds new participant in the middle of a call. + virtual void RegisterParticipantInCall(absl::string_view peer_name) {} + // Will be called after test removed existing participant in the middle of the + // call. + virtual void UnregisterParticipantInCall(absl::string_view peer_name) {} + + // Tells analyzer that analysis complete and it should calculate final + // statistics. + virtual void Stop() {} + + // Returns the last stream where this frame was captured. It means that if + // frame ids space wraps around, then stream label for frame id may change. + // It will crash, if the specified `frame_id` wasn't captured. + virtual std::string GetStreamLabel(uint16_t frame_id) = 0; +}; + +} // namespace webrtc + +#endif // API_TEST_VIDEO_QUALITY_ANALYZER_INTERFACE_H_ diff --git a/third_party/libwebrtc/api/test/video_quality_test_fixture.h b/third_party/libwebrtc/api/test/video_quality_test_fixture.h new file mode 100644 index 0000000000..b45faef286 --- /dev/null +++ b/third_party/libwebrtc/api/test/video_quality_test_fixture.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_ +#define API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "api/fec_controller.h" +#include "api/media_types.h" +#include "api/network_state_predictor.h" +#include "api/test/simulated_network.h" +#include "api/transport/bitrate_settings.h" +#include "api/transport/network_control.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "video/config/video_encoder_config.h" + +namespace webrtc { + +class VideoQualityTestFixtureInterface { + public: + // Parameters are grouped into smaller structs to make it easier to set + // the desired elements and skip unused. + struct Params { + struct CallConfig { + bool send_side_bwe = false; + bool generic_descriptor = false; + bool dependency_descriptor = false; + BitrateConstraints call_bitrate_config; + int num_thumbnails = 0; + // Indicates if secondary_(video|ss|screenshare) structures are used. + bool dual_video = false; + } call; + struct Video { + bool enabled = false; + size_t width = 640; + size_t height = 480; + int32_t fps = 30; + int min_bitrate_bps = 50; + int target_bitrate_bps = 800; + int max_bitrate_bps = 800; + bool suspend_below_min_bitrate = false; + std::string codec = "VP8"; + int num_temporal_layers = 1; + int selected_tl = -1; + int min_transmit_bps = 0; + bool ulpfec = false; + bool flexfec = false; + bool automatic_scaling = false; + std::string clip_path; // "Generator" to generate frames instead. + size_t capture_device_index = 0; + SdpVideoFormat::Parameters sdp_params; + double encoder_overshoot_factor = 0.0; + } video[2]; + struct Audio { + bool enabled = false; + bool sync_video = false; + bool dtx = false; + bool use_real_adm = false; + absl::optional<std::string> ana_config; + } audio; + struct Screenshare { + bool enabled = false; + bool generate_slides = false; + int32_t slide_change_interval = 10; + int32_t scroll_duration = 0; + std::vector<std::string> slides; + } screenshare[2]; + struct Analyzer { + std::string test_label; + double avg_psnr_threshold = 0.0; // (*) + double avg_ssim_threshold = 0.0; // (*) + int test_durations_secs = 0; + std::string graph_data_output_filename; + std::string graph_title; + } analyzer; + // Config for default simulation implementation. Must be nullopt if + // `sender_network` and `receiver_network` in InjectionComponents are + // non-null. May be nullopt even if `sender_network` and `receiver_network` + // are null; in that case, a default config will be used. + absl::optional<BuiltInNetworkBehaviorConfig> config; + struct SS { // Spatial scalability. + std::vector<VideoStream> streams; // If empty, one stream is assumed. + size_t selected_stream = 0; + int num_spatial_layers = 0; + int selected_sl = -1; + InterLayerPredMode inter_layer_pred = InterLayerPredMode::kOn; + // If empty, bitrates are generated in VP9Impl automatically. + std::vector<SpatialLayer> spatial_layers; + // If set, default parameters will be used instead of `streams`. + bool infer_streams = false; + } ss[2]; + struct Logging { + std::string rtc_event_log_name; + std::string rtp_dump_name; + std::string encoded_frame_base_path; + } logging; + }; + + // Contains objects, that will be injected on different layers of test + // framework to override the behavior of system parts. + struct InjectionComponents { + InjectionComponents(); + ~InjectionComponents(); + + // Simulations of sender and receiver networks. They must either both be + // null (in which case `config` from Params is used), or both be non-null + // (in which case `config` from Params must be nullopt). + std::unique_ptr<NetworkBehaviorInterface> sender_network; + std::unique_ptr<NetworkBehaviorInterface> receiver_network; + + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory; + std::unique_ptr<VideoEncoderFactory> video_encoder_factory; + std::unique_ptr<VideoDecoderFactory> video_decoder_factory; + std::unique_ptr<NetworkStatePredictorFactoryInterface> + network_state_predictor_factory; + std::unique_ptr<NetworkControllerFactoryInterface> + network_controller_factory; + }; + + virtual ~VideoQualityTestFixtureInterface() = default; + + virtual void RunWithAnalyzer(const Params& params) = 0; + virtual void RunWithRenderers(const Params& params) = 0; + + virtual const std::map<uint8_t, webrtc::MediaType>& payload_type_map() = 0; +}; + +} // namespace webrtc + +#endif // API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/videocodec_test_fixture.h b/third_party/libwebrtc/api/test/videocodec_test_fixture.h new file mode 100644 index 0000000000..dbf20993e2 --- /dev/null +++ b/third_party/libwebrtc/api/test/videocodec_test_fixture.h @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ +#define API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ + +#include <string> +#include <vector> + +#include "api/test/videocodec_test_stats.h" +#include "api/video_codecs/h264_profile_level_id.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/include/video_codec_interface.h" + +namespace webrtc { +namespace test { + +// Rates for the encoder and the frame number when to apply profile. +struct RateProfile { + size_t target_kbps; + double input_fps; + size_t frame_num; +}; + +struct RateControlThresholds { + double max_avg_bitrate_mismatch_percent; + double max_time_to_reach_target_bitrate_sec; + // TODO(ssilkin): Use absolute threshold for framerate. + double max_avg_framerate_mismatch_percent; + double max_avg_buffer_level_sec; + double max_max_key_frame_delay_sec; + double max_max_delta_frame_delay_sec; + size_t max_num_spatial_resizes; + size_t max_num_key_frames; +}; + +struct QualityThresholds { + double min_avg_psnr; + double min_min_psnr; + double min_avg_ssim; + double min_min_ssim; +}; + +struct BitstreamThresholds { + size_t max_max_nalu_size_bytes; +}; + +// NOTE: This class is still under development and may change without notice. +class VideoCodecTestFixture { + public: + class EncodedFrameChecker { + public: + virtual ~EncodedFrameChecker() = default; + virtual void CheckEncodedFrame(VideoCodecType codec, + const EncodedImage& encoded_frame) const = 0; + }; + + struct Config { + Config(); + void SetCodecSettings(std::string codec_name, + size_t num_simulcast_streams, + size_t num_spatial_layers, + size_t num_temporal_layers, + bool denoising_on, + bool frame_dropper_on, + bool spatial_resize_on, + size_t width, + size_t height); + + size_t NumberOfCores() const; + size_t NumberOfTemporalLayers() const; + size_t NumberOfSpatialLayers() const; + size_t NumberOfSimulcastStreams() const; + + std::string ToString() const; + std::string CodecName() const; + + // Name of this config, to be used for accounting by the test runner. + std::string test_name; + + // Plain name of YUV file to process without file extension. + std::string filename; + // Dimensions of test clip. Falls back to (codec_settings.width/height) if + // not set. + absl::optional<int> clip_width; + absl::optional<int> clip_height; + // Framerate of input clip. Defaults to 30fps if not set. + absl::optional<int> clip_fps; + + // The resolution at which psnr/ssim comparisons should be made. Frames + // will be scaled to this size if different. + absl::optional<int> reference_width; + absl::optional<int> reference_height; + + // File to process. This must be a video file in the YUV format. + std::string filepath; + + // Number of frames to process. + size_t num_frames = 0; + + // Bitstream constraints. + size_t max_payload_size_bytes = 1440; + + // Should we decode the encoded frames? + bool decode = true; + + // Force the encoder and decoder to use a single core for processing. + bool use_single_core = false; + + // Should cpu usage be measured? + // If set to true, the encoding will run in real-time. + bool measure_cpu = false; + + // Simulate frames arriving in real-time by adding delays between frames. + bool encode_in_real_time = false; + + // Codec settings to use. + VideoCodec codec_settings; + + // Name of the codec being tested. + std::string codec_name; + + // Encoder and decoder format and parameters. If provided, format is used to + // instantiate the codec. If not provided, the test creates and uses the + // default `SdpVideoFormat` based on `codec_name`. + // Encoder and decoder name (`SdpVideoFormat::name`) should be the same as + // `codec_name`. + absl::optional<SdpVideoFormat> encoder_format; + absl::optional<SdpVideoFormat> decoder_format; + + // H.264 specific settings. + struct H264CodecSettings { + H264Profile profile = H264Profile::kProfileConstrainedBaseline; + H264PacketizationMode packetization_mode = + H264PacketizationMode::NonInterleaved; + } h264_codec_settings; + + // Custom checker that will be called for each frame. + const EncodedFrameChecker* encoded_frame_checker = nullptr; + + // Print out frame level stats. + bool print_frame_level_stats = false; + + // Path to a directory where encoded or/and decoded video should be saved. + std::string output_path; + + // Should video be saved persistently to disk for post-run visualization? + struct VisualizationParams { + bool save_encoded_ivf = false; + bool save_decoded_y4m = false; + } visualization_params; + + // Enables quality analysis for dropped frames. + bool analyze_quality_of_dropped_frames = false; + }; + + virtual ~VideoCodecTestFixture() = default; + + virtual void RunTest(const std::vector<RateProfile>& rate_profiles, + const std::vector<RateControlThresholds>* rc_thresholds, + const std::vector<QualityThresholds>* quality_thresholds, + const BitstreamThresholds* bs_thresholds) = 0; + virtual VideoCodecTestStats& GetStats() = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/videocodec_test_stats.cc b/third_party/libwebrtc/api/test/videocodec_test_stats.cc new file mode 100644 index 0000000000..f082b1e935 --- /dev/null +++ b/third_party/libwebrtc/api/test/videocodec_test_stats.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/videocodec_test_stats.h" + +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace test { + +VideoCodecTestStats::FrameStatistics::FrameStatistics(size_t frame_number, + size_t rtp_timestamp, + size_t spatial_idx) + : frame_number(frame_number), + rtp_timestamp(rtp_timestamp), + spatial_idx(spatial_idx) {} + +std::string VideoCodecTestStats::FrameStatistics::ToString() const { + rtc::StringBuilder ss; + for (const auto& entry : ToMap()) { + if (ss.size() > 0) { + ss << " "; + } + ss << entry.first << " " << entry.second; + } + return ss.Release(); +} + +std::map<std::string, std::string> VideoCodecTestStats::FrameStatistics::ToMap() + const { + std::map<std::string, std::string> map; + map["frame_number"] = std::to_string(frame_number); + map["decoded_width"] = std::to_string(decoded_width); + map["decoded_height"] = std::to_string(decoded_height); + map["spatial_idx"] = std::to_string(spatial_idx); + map["temporal_idx"] = std::to_string(temporal_idx); + map["inter_layer_predicted"] = std::to_string(inter_layer_predicted); + map["non_ref_for_inter_layer_pred"] = + std::to_string(non_ref_for_inter_layer_pred); + map["frame_type"] = std::to_string(static_cast<int>(frame_type)); + map["length_bytes"] = std::to_string(length_bytes); + map["qp"] = std::to_string(qp); + map["psnr"] = std::to_string(psnr); + map["psnr_y"] = std::to_string(psnr_y); + map["psnr_u"] = std::to_string(psnr_u); + map["psnr_v"] = std::to_string(psnr_v); + map["ssim"] = std::to_string(ssim); + map["encode_time_us"] = std::to_string(encode_time_us); + map["decode_time_us"] = std::to_string(decode_time_us); + map["rtp_timestamp"] = std::to_string(rtp_timestamp); + map["target_bitrate_kbps"] = std::to_string(target_bitrate_kbps); + map["target_framerate_fps"] = std::to_string(target_framerate_fps); + return map; +} + +std::string VideoCodecTestStats::VideoStatistics::ToString( + std::string prefix) const { + rtc::StringBuilder ss; + for (const auto& entry : ToMap()) { + if (ss.size() > 0) { + ss << "\n"; + } + ss << prefix << entry.first << ": " << entry.second; + } + return ss.Release(); +} + +std::map<std::string, std::string> VideoCodecTestStats::VideoStatistics::ToMap() + const { + std::map<std::string, std::string> map; + map["target_bitrate_kbps"] = std::to_string(target_bitrate_kbps); + map["input_framerate_fps"] = std::to_string(input_framerate_fps); + map["spatial_idx"] = std::to_string(spatial_idx); + map["temporal_idx"] = std::to_string(temporal_idx); + map["width"] = std::to_string(width); + map["height"] = std::to_string(height); + map["length_bytes"] = std::to_string(length_bytes); + map["bitrate_kbps"] = std::to_string(bitrate_kbps); + map["framerate_fps"] = std::to_string(framerate_fps); + map["enc_speed_fps"] = std::to_string(enc_speed_fps); + map["dec_speed_fps"] = std::to_string(dec_speed_fps); + map["avg_encode_latency_sec"] = std::to_string(avg_encode_latency_sec); + map["max_encode_latency_sec"] = std::to_string(max_encode_latency_sec); + map["avg_decode_latency_sec"] = std::to_string(avg_decode_latency_sec); + map["max_decode_latency_sec"] = std::to_string(max_decode_latency_sec); + map["avg_delay_sec"] = std::to_string(avg_delay_sec); + map["max_key_frame_delay_sec"] = std::to_string(max_key_frame_delay_sec); + map["max_delta_frame_delay_sec"] = std::to_string(max_delta_frame_delay_sec); + map["time_to_reach_target_bitrate_sec"] = + std::to_string(time_to_reach_target_bitrate_sec); + map["avg_bitrate_mismatch_pct"] = std::to_string(avg_bitrate_mismatch_pct); + map["avg_framerate_mismatch_pct"] = + std::to_string(avg_framerate_mismatch_pct); + map["avg_key_frame_size_bytes"] = std::to_string(avg_key_frame_size_bytes); + map["avg_delta_frame_size_bytes"] = + std::to_string(avg_delta_frame_size_bytes); + map["avg_qp"] = std::to_string(avg_qp); + map["avg_psnr"] = std::to_string(avg_psnr); + map["min_psnr"] = std::to_string(min_psnr); + map["avg_ssim"] = std::to_string(avg_ssim); + map["min_ssim"] = std::to_string(min_ssim); + map["num_input_frames"] = std::to_string(num_input_frames); + map["num_encoded_frames"] = std::to_string(num_encoded_frames); + map["num_decoded_frames"] = std::to_string(num_decoded_frames); + map["num_dropped_frames"] = + std::to_string(num_input_frames - num_encoded_frames); + map["num_key_frames"] = std::to_string(num_key_frames); + map["num_spatial_resizes"] = std::to_string(num_spatial_resizes); + map["max_nalu_size_bytes"] = std::to_string(max_nalu_size_bytes); + return map; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/videocodec_test_stats.h b/third_party/libwebrtc/api/test/videocodec_test_stats.h new file mode 100644 index 0000000000..12c60638db --- /dev/null +++ b/third_party/libwebrtc/api/test/videocodec_test_stats.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEOCODEC_TEST_STATS_H_ +#define API_TEST_VIDEOCODEC_TEST_STATS_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <map> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/units/data_rate.h" +#include "api/units/frequency.h" +#include "api/video/video_frame_type.h" + +namespace webrtc { +namespace test { + +// Statistics for a sequence of processed frames. This class is not thread safe. +class VideoCodecTestStats { + public: + // Statistics for one processed frame. + struct FrameStatistics { + FrameStatistics(size_t frame_number, + size_t rtp_timestamp, + size_t spatial_idx); + + std::string ToString() const; + + // Returns name -> value text map of frame statistics. + std::map<std::string, std::string> ToMap() const; + + size_t frame_number = 0; + size_t rtp_timestamp = 0; + + // Encoding. + int64_t encode_start_ns = 0; + int encode_return_code = 0; + bool encoding_successful = false; + size_t encode_time_us = 0; + size_t target_bitrate_kbps = 0; + double target_framerate_fps = 0.0; + size_t length_bytes = 0; + VideoFrameType frame_type = VideoFrameType::kVideoFrameDelta; + + // Layering. + size_t spatial_idx = 0; + size_t temporal_idx = 0; + bool inter_layer_predicted = false; + bool non_ref_for_inter_layer_pred = true; + + // H264 specific. + size_t max_nalu_size_bytes = 0; + + // Decoding. + int64_t decode_start_ns = 0; + int decode_return_code = 0; + bool decoding_successful = false; + size_t decode_time_us = 0; + size_t decoded_width = 0; + size_t decoded_height = 0; + + // Quantization. + int qp = -1; + + // Quality. + bool quality_analysis_successful = false; + float psnr_y = 0.0f; + float psnr_u = 0.0f; + float psnr_v = 0.0f; + float psnr = 0.0f; // 10 * log10(255^2 / (mse_y + mse_u + mse_v)). + float ssim = 0.0f; // 0.8 * ssim_y + 0.1 * (ssim_u + ssim_v). + }; + + struct VideoStatistics { + std::string ToString(std::string prefix) const; + + // Returns name -> value text map of video statistics. + std::map<std::string, std::string> ToMap() const; + + size_t target_bitrate_kbps = 0; + float input_framerate_fps = 0.0f; + + size_t spatial_idx = 0; + size_t temporal_idx = 0; + + size_t width = 0; + size_t height = 0; + + size_t length_bytes = 0; + size_t bitrate_kbps = 0; + float framerate_fps = 0; + + float enc_speed_fps = 0.0f; + float dec_speed_fps = 0.0f; + + float avg_encode_latency_sec = 0.0f; + float max_encode_latency_sec = 0.0f; + float avg_decode_latency_sec = 0.0f; + float max_decode_latency_sec = 0.0f; + + float avg_delay_sec = 0.0f; + float max_key_frame_delay_sec = 0.0f; + float max_delta_frame_delay_sec = 0.0f; + float time_to_reach_target_bitrate_sec = 0.0f; + float avg_bitrate_mismatch_pct = 0.0f; + float avg_framerate_mismatch_pct = 0.0f; + + float avg_key_frame_size_bytes = 0.0f; + float avg_delta_frame_size_bytes = 0.0f; + float avg_qp = 0.0f; + + float avg_psnr_y = 0.0f; + float avg_psnr_u = 0.0f; + float avg_psnr_v = 0.0f; + float avg_psnr = 0.0f; + float min_psnr = 0.0f; + float avg_ssim = 0.0f; + float min_ssim = 0.0f; + + size_t num_input_frames = 0; + size_t num_encoded_frames = 0; + size_t num_decoded_frames = 0; + size_t num_key_frames = 0; + size_t num_spatial_resizes = 0; + size_t max_nalu_size_bytes = 0; + }; + + virtual ~VideoCodecTestStats() = default; + + virtual std::vector<FrameStatistics> GetFrameStatistics() const = 0; + + virtual std::vector<VideoStatistics> SliceAndCalcLayerVideoStatistic( + size_t first_frame_num, + size_t last_frame_num) = 0; + + virtual VideoStatistics CalcVideoStatistic(size_t first_frame, + size_t last_frame, + DataRate target_bitrate, + Frequency target_framerate) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEOCODEC_TEST_STATS_H_ |