diff options
Diffstat (limited to 'third_party/libwebrtc/api/test')
124 files changed, 11204 insertions, 0 deletions
diff --git a/third_party/libwebrtc/api/test/DEPS b/third_party/libwebrtc/api/test/DEPS new file mode 100644 index 0000000000..270b274c5f --- /dev/null +++ b/third_party/libwebrtc/api/test/DEPS @@ -0,0 +1,39 @@ +specific_include_rules = { + ".*": [ + "+modules/video_coding", + ], + ".*": [ + "+video" + ], + "dummy_peer_connection\.h": [ + "+rtc_base/ref_counted_object.h", + ], + "neteq_factory_with_codecs\.h": [ + "+system_wrappers/include/clock.h", + ], + "network_emulation_manager\.h": [ + "+rtc_base/thread.h", + "+rtc_base/network.h", + "+rtc_base/network_constants.h", + ], + "peer_network_dependencies\.h": [ + "+rtc_base/network.h", + "+rtc_base/thread.h", + ], + "peerconnection_quality_test_fixture\.h": [ + "+logging/rtc_event_log/rtc_event_log_factory_interface.h", + "+rtc_base/network.h", + "+rtc_base/rtc_certificate_generator.h", + "+rtc_base/ssl_certificate.h", + "+rtc_base/thread.h", + "+media/base/media_constants.h", + "+modules/audio_processing/include/audio_processing.h", + ], + "time_controller\.h": [ + "+rtc_base/synchronization/yield_policy.h", + "+system_wrappers/include/clock.h", + ], + "create_frame_generator\.h": [ + "+system_wrappers/include/clock.h", + ], +} diff --git a/third_party/libwebrtc/api/test/OWNERS b/third_party/libwebrtc/api/test/OWNERS new file mode 100644 index 0000000000..a7392abe31 --- /dev/null +++ b/third_party/libwebrtc/api/test/OWNERS @@ -0,0 +1,5 @@ +mbonadei@webrtc.org +sprang@webrtc.org +srte@webrtc.org +titovartem@webrtc.org + diff --git a/third_party/libwebrtc/api/test/audio_quality_analyzer_interface.h b/third_party/libwebrtc/api/test/audio_quality_analyzer_interface.h new file mode 100644 index 0000000000..2eb7817445 --- /dev/null +++ b/third_party/libwebrtc/api/test/audio_quality_analyzer_interface.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_AUDIO_QUALITY_ANALYZER_INTERFACE_H_ +#define API_TEST_AUDIO_QUALITY_ANALYZER_INTERFACE_H_ + +#include <string> + +#include "api/test/stats_observer_interface.h" +#include "api/test/track_id_stream_info_map.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// API is in development. Can be changed/removed without notice. +class AudioQualityAnalyzerInterface : public StatsObserverInterface { + public: + ~AudioQualityAnalyzerInterface() override = default; + + // Will be called by the framework before the test. + // `test_case_name` is name of test case, that should be used to report all + // audio metrics. + // `analyzer_helper` is a pointer to a class that will allow track_id to + // stream_id matching. The caller is responsible for ensuring the + // AnalyzerHelper outlives the instance of the AudioQualityAnalyzerInterface. + virtual void Start(std::string test_case_name, + TrackIdStreamInfoMap* analyzer_helper) = 0; + + // Will be called by the framework at the end of the test. The analyzer + // has to finalize all its stats and it should report them. + virtual void Stop() = 0; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_AUDIO_QUALITY_ANALYZER_INTERFACE_H_ diff --git a/third_party/libwebrtc/api/test/audioproc_float.cc b/third_party/libwebrtc/api/test/audioproc_float.cc new file mode 100644 index 0000000000..c8d7ff7193 --- /dev/null +++ b/third_party/libwebrtc/api/test/audioproc_float.cc @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/audioproc_float.h" + +#include <utility> + +#include "modules/audio_processing/test/audioproc_float_impl.h" + +namespace webrtc { +namespace test { + +int AudioprocFloat(rtc::scoped_refptr<AudioProcessing> audio_processing, + int argc, + char* argv[]) { + return AudioprocFloatImpl(std::move(audio_processing), argc, argv); +} + +int AudioprocFloat(std::unique_ptr<AudioProcessingBuilder> ap_builder, + int argc, + char* argv[]) { + return AudioprocFloatImpl(std::move(ap_builder), argc, argv, + /*input_aecdump=*/"", + /*processed_capture_samples=*/nullptr); +} + +int AudioprocFloat(std::unique_ptr<AudioProcessingBuilder> ap_builder, + int argc, + char* argv[], + absl::string_view input_aecdump, + std::vector<float>* processed_capture_samples) { + return AudioprocFloatImpl(std::move(ap_builder), argc, argv, input_aecdump, + processed_capture_samples); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/audioproc_float.h b/third_party/libwebrtc/api/test/audioproc_float.h new file mode 100644 index 0000000000..1ef1c9828f --- /dev/null +++ b/third_party/libwebrtc/api/test/audioproc_float.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_AUDIOPROC_FLOAT_H_ +#define API_TEST_AUDIOPROC_FLOAT_H_ + +#include <memory> +#include <vector> + +#include "modules/audio_processing/include/audio_processing.h" + +namespace webrtc { +namespace test { + +// This is an interface for the audio processing simulation utility. This +// utility can be used to simulate the audioprocessing module using a recording +// (either an AEC dump or wav files), and generate the output as a wav file. +// Any audio_processing object specified in the input is used for the +// simulation. The optional `audio_processing` object provides the +// AudioProcessing instance that is used during the simulation. Note that when +// the audio_processing object is specified all functionality that relies on +// using the AudioProcessingBuilder is deactivated, since the AudioProcessing +// object is already created and the builder is not used in the simulation. It +// is needed to pass the command line flags as `argc` and `argv`, so these can +// be interpreted properly by the utility. To see a list of all supported +// command line flags, run the executable with the '--help' flag. +int AudioprocFloat(rtc::scoped_refptr<AudioProcessing> audio_processing, + int argc, + char* argv[]); + +// This is an interface for the audio processing simulation utility. This +// utility can be used to simulate the audioprocessing module using a recording +// (either an AEC dump or wav files), and generate the output as a wav file. +// The `ap_builder` object will be used to create the AudioProcessing instance +// that is used during the simulation. The `ap_builder` supports setting of +// injectable components, which will be passed on to the created AudioProcessing +// instance. It is needed to pass the command line flags as `argc` and `argv`, +// so these can be interpreted properly by the utility. +// To get a fully-working audioproc_f utility, all that is needed is to write a +// main function, create an AudioProcessingBuilder, optionally set custom +// processing components on it, and pass the builder together with the command +// line arguments into this function. +// To see a list of all supported command line flags, run the executable with +// the '--help' flag. +int AudioprocFloat(std::unique_ptr<AudioProcessingBuilder> ap_builder, + int argc, + char* argv[]); + +// Interface for the audio processing simulation utility, which is similar to +// the one above, but which adds the option of receiving the input as a string +// and returning the output as an array. The first three arguments fulfill the +// same purpose as above. Pass the `input_aecdump` to provide the content of an +// AEC dump file as a string. After the simulation is completed, +// `processed_capture_samples` will contain the the samples processed on the +// capture side. +int AudioprocFloat(std::unique_ptr<AudioProcessingBuilder> ap_builder, + int argc, + char* argv[], + absl::string_view input_aecdump, + std::vector<float>* processed_capture_samples); +} // namespace test +} // namespace webrtc + +#endif // API_TEST_AUDIOPROC_FLOAT_H_ diff --git a/third_party/libwebrtc/api/test/compile_all_headers.cc b/third_party/libwebrtc/api/test/compile_all_headers.cc new file mode 100644 index 0000000000..1fcf63e97b --- /dev/null +++ b/third_party/libwebrtc/api/test/compile_all_headers.cc @@ -0,0 +1,53 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file verifies that all include files in this directory can be +// compiled without errors or other required includes. + +// Note: The following header files are not not tested here, as their +// associated targets are not included in all configurations. +// "api/test/audioproc_float.h" +// "api/test/create_video_quality_test_fixture.h" +// "api/test/neteq_simulator_factory.h" +// "api/test/video_quality_test_fixture.h" +// The following header files are also not tested: +// "api/test/create_simulcast_test_fixture.h" +// "api/test/create_videocodec_test_fixture.h" +// "api/test/neteq_simulator.h" +// "api/test/simulated_network.h" +// "api/test/simulcast_test_fixture.h" +// "api/test/test_dependency_factory.h" +// "api/test/videocodec_test_fixture.h" +// "api/test/videocodec_test_stats.h" + +#include "api/test/fake_frame_decryptor.h" +#include "api/test/fake_frame_encryptor.h" +#include "api/test/mock_async_dns_resolver.h" +#include "api/test/mock_audio_mixer.h" +#include "api/test/mock_audio_sink.h" +#include "api/test/mock_data_channel.h" +#include "api/test/mock_dtmf_sender.h" +#include "api/test/mock_frame_decryptor.h" +#include "api/test/mock_frame_encryptor.h" +#include "api/test/mock_media_stream_interface.h" +#include "api/test/mock_peer_connection_factory_interface.h" +#include "api/test/mock_peerconnectioninterface.h" +#include "api/test/mock_rtp_transceiver.h" +#include "api/test/mock_rtpreceiver.h" +#include "api/test/mock_rtpsender.h" +#include "api/test/mock_session_description_interface.h" +#include "api/test/mock_transformable_video_frame.h" +#include "api/test/mock_video_bitrate_allocator.h" +#include "api/test/mock_video_bitrate_allocator_factory.h" +#include "api/test/mock_video_decoder.h" +#include "api/test/mock_video_decoder_factory.h" +#include "api/test/mock_video_encoder.h" +#include "api/test/mock_video_encoder_factory.h" +#include "api/test/mock_video_track.h" diff --git a/third_party/libwebrtc/api/test/create_frame_generator.cc b/third_party/libwebrtc/api/test/create_frame_generator.cc new file mode 100644 index 0000000000..5e6fb3228b --- /dev/null +++ b/third_party/libwebrtc/api/test/create_frame_generator.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_frame_generator.h" + +#include <cstdio> +#include <utility> + +#include "rtc_base/checks.h" +#include "test/frame_generator.h" +#include "test/testsupport/ivf_video_frame_generator.h" + +namespace webrtc { +namespace test { + +std::unique_ptr<FrameGeneratorInterface> CreateSquareFrameGenerator( + int width, + int height, + absl::optional<FrameGeneratorInterface::OutputType> type, + absl::optional<int> num_squares) { + return std::make_unique<SquareGenerator>( + width, height, type.value_or(FrameGeneratorInterface::OutputType::kI420), + num_squares.value_or(10)); +} + +std::unique_ptr<FrameGeneratorInterface> CreateFromYuvFileFrameGenerator( + std::vector<std::string> filenames, + size_t width, + size_t height, + int frame_repeat_count) { + RTC_DCHECK(!filenames.empty()); + std::vector<FILE*> files; + for (const std::string& filename : filenames) { + FILE* file = fopen(filename.c_str(), "rb"); + RTC_DCHECK(file != nullptr) << "Failed to open: '" << filename << "'\n"; + files.push_back(file); + } + + return std::make_unique<YuvFileGenerator>(files, width, height, + frame_repeat_count); +} + +std::unique_ptr<FrameGeneratorInterface> CreateFromNV12FileFrameGenerator( + std::vector<std::string> filenames, + size_t width, + size_t height, + int frame_repeat_count) { + RTC_DCHECK(!filenames.empty()); + std::vector<FILE*> files; + for (const std::string& filename : filenames) { + FILE* file = fopen(filename.c_str(), "rb"); + RTC_DCHECK(file != nullptr) << "Failed to open: '" << filename << "'\n"; + files.push_back(file); + } + + return std::make_unique<NV12FileGenerator>(files, width, height, + frame_repeat_count); +} + +std::unique_ptr<FrameGeneratorInterface> CreateFromIvfFileFrameGenerator( + std::string filename) { + return std::make_unique<IvfVideoFrameGenerator>(std::move(filename)); +} + +std::unique_ptr<FrameGeneratorInterface> +CreateScrollingInputFromYuvFilesFrameGenerator( + Clock* clock, + std::vector<std::string> filenames, + size_t source_width, + size_t source_height, + size_t target_width, + size_t target_height, + int64_t scroll_time_ms, + int64_t pause_time_ms) { + RTC_DCHECK(!filenames.empty()); + std::vector<FILE*> files; + for (const std::string& filename : filenames) { + FILE* file = fopen(filename.c_str(), "rb"); + RTC_DCHECK(file != nullptr); + files.push_back(file); + } + + return std::make_unique<ScrollingImageFrameGenerator>( + clock, files, source_width, source_height, target_width, target_height, + scroll_time_ms, pause_time_ms); +} + +std::unique_ptr<FrameGeneratorInterface> +CreateSlideFrameGenerator(int width, int height, int frame_repeat_count) { + return std::make_unique<SlideGenerator>(width, height, frame_repeat_count); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_frame_generator.h b/third_party/libwebrtc/api/test/create_frame_generator.h new file mode 100644 index 0000000000..70be0c4e8e --- /dev/null +++ b/third_party/libwebrtc/api/test/create_frame_generator.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_FRAME_GENERATOR_H_ +#define API_TEST_CREATE_FRAME_GENERATOR_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/test/frame_generator_interface.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace test { + +// Creates a frame generator that produces frames with small squares that +// move randomly towards the lower right corner. +// `type` has the default value FrameGeneratorInterface::OutputType::I420. +// `num_squares` has the default value 10. +std::unique_ptr<FrameGeneratorInterface> CreateSquareFrameGenerator( + int width, + int height, + absl::optional<FrameGeneratorInterface::OutputType> type, + absl::optional<int> num_squares); + +// Creates a frame generator that repeatedly plays a set of yuv files. +// The frame_repeat_count determines how many times each frame is shown, +// with 1 = show each frame once, etc. +std::unique_ptr<FrameGeneratorInterface> CreateFromYuvFileFrameGenerator( + std::vector<std::string> filenames, + size_t width, + size_t height, + int frame_repeat_count); + +// Creates a frame generator that repeatedly plays a set of nv12 files. +// The frame_repeat_count determines how many times each frame is shown, +// with 1 = show each frame once, etc. +std::unique_ptr<FrameGeneratorInterface> CreateFromNV12FileFrameGenerator( + std::vector<std::string> filenames, + size_t width, + size_t height, + int frame_repeat_count = 1); + +// Creates a frame generator that repeatedly plays an ivf file. +std::unique_ptr<FrameGeneratorInterface> CreateFromIvfFileFrameGenerator( + std::string filename); + +// Creates a frame generator which takes a set of yuv files (wrapping a +// frame generator created by CreateFromYuvFile() above), but outputs frames +// that have been cropped to specified resolution: source_width/source_height +// is the size of the source images, target_width/target_height is the size of +// the cropped output. For each source image read, the cropped viewport will +// be scrolled top to bottom/left to right for scroll_tim_ms milliseconds. +// After that the image will stay in place for pause_time_ms milliseconds, +// and then this will be repeated with the next file from the input set. +std::unique_ptr<FrameGeneratorInterface> +CreateScrollingInputFromYuvFilesFrameGenerator( + Clock* clock, + std::vector<std::string> filenames, + size_t source_width, + size_t source_height, + size_t target_width, + size_t target_height, + int64_t scroll_time_ms, + int64_t pause_time_ms); + +// Creates a frame generator that produces randomly generated slides. It fills +// the frames with randomly sized and colored squares. +// `frame_repeat_count` determines how many times each slide is shown. +std::unique_ptr<FrameGeneratorInterface> +CreateSlideFrameGenerator(int width, int height, int frame_repeat_count); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_CREATE_FRAME_GENERATOR_H_ diff --git a/third_party/libwebrtc/api/test/create_network_emulation_manager.cc b/third_party/libwebrtc/api/test/create_network_emulation_manager.cc new file mode 100644 index 0000000000..f5d5a1bc88 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_network_emulation_manager.cc @@ -0,0 +1,27 @@ + +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_network_emulation_manager.h" + +#include <memory> + +#include "test/network/network_emulation_manager.h" + +namespace webrtc { + +std::unique_ptr<NetworkEmulationManager> CreateNetworkEmulationManager( + TimeMode time_mode, + EmulatedNetworkStatsGatheringMode stats_gathering_mode) { + return std::make_unique<test::NetworkEmulationManagerImpl>( + time_mode, stats_gathering_mode); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_network_emulation_manager.h b/third_party/libwebrtc/api/test/create_network_emulation_manager.h new file mode 100644 index 0000000000..941b2b1c52 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_network_emulation_manager.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_NETWORK_EMULATION_MANAGER_H_ +#define API_TEST_CREATE_NETWORK_EMULATION_MANAGER_H_ + +#include <memory> + +#include "api/test/network_emulation_manager.h" + +namespace webrtc { + +// Returns a non-null NetworkEmulationManager instance. +std::unique_ptr<NetworkEmulationManager> CreateNetworkEmulationManager( + TimeMode time_mode = TimeMode::kRealTime, + EmulatedNetworkStatsGatheringMode stats_gathering_mode = + EmulatedNetworkStatsGatheringMode::kDefault); + +} // namespace webrtc + +#endif // API_TEST_CREATE_NETWORK_EMULATION_MANAGER_H_ diff --git a/third_party/libwebrtc/api/test/create_peer_connection_quality_test_frame_generator.cc b/third_party/libwebrtc/api/test/create_peer_connection_quality_test_frame_generator.cc new file mode 100644 index 0000000000..a1c53635f9 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_peer_connection_quality_test_frame_generator.cc @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_peer_connection_quality_test_frame_generator.h" + +#include <utility> +#include <vector> + +#include "api/test/create_frame_generator.h" +#include "api/test/pclf/media_configuration.h" +#include "rtc_base/checks.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void ValidateScreenShareConfig(const VideoConfig& video_config, + const ScreenShareConfig& screen_share_config) { + if (screen_share_config.slides_yuv_file_names.empty()) { + if (screen_share_config.scrolling_params) { + // If we have scrolling params, then its `source_width` and `source_heigh` + // will be used as width and height of video input, so we have to validate + // it against width and height of default input. + RTC_CHECK_EQ(screen_share_config.scrolling_params->source_width, + kDefaultSlidesWidth); + RTC_CHECK_EQ(screen_share_config.scrolling_params->source_height, + kDefaultSlidesHeight); + } else { + RTC_CHECK_EQ(video_config.width, kDefaultSlidesWidth); + RTC_CHECK_EQ(video_config.height, kDefaultSlidesHeight); + } + } + if (screen_share_config.scrolling_params) { + RTC_CHECK_LE(screen_share_config.scrolling_params->duration, + screen_share_config.slide_change_interval); + RTC_CHECK_GE(screen_share_config.scrolling_params->source_width, + video_config.width); + RTC_CHECK_GE(screen_share_config.scrolling_params->source_height, + video_config.height); + } +} + +std::unique_ptr<test::FrameGeneratorInterface> CreateSquareFrameGenerator( + const VideoConfig& video_config, + absl::optional<test::FrameGeneratorInterface::OutputType> type) { + return test::CreateSquareFrameGenerator( + video_config.width, video_config.height, std::move(type), absl::nullopt); +} + +std::unique_ptr<test::FrameGeneratorInterface> CreateFromYuvFileFrameGenerator( + const VideoConfig& video_config, + std::string filename) { + return test::CreateFromYuvFileFrameGenerator( + {std::move(filename)}, video_config.width, video_config.height, + /*frame_repeat_count=*/1); +} + +std::unique_ptr<test::FrameGeneratorInterface> CreateScreenShareFrameGenerator( + const VideoConfig& video_config, + const ScreenShareConfig& screen_share_config) { + ValidateScreenShareConfig(video_config, screen_share_config); + if (screen_share_config.generate_slides) { + return test::CreateSlideFrameGenerator( + video_config.width, video_config.height, + screen_share_config.slide_change_interval.seconds() * video_config.fps); + } + std::vector<std::string> slides = screen_share_config.slides_yuv_file_names; + if (slides.empty()) { + // If slides is empty we need to add default slides as source. In such case + // video width and height is validated to be equal to kDefaultSlidesWidth + // and kDefaultSlidesHeight. + slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv")); + slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv")); + slides.push_back(test::ResourcePath("photo_1850_1110", "yuv")); + slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv")); + } + if (!screen_share_config.scrolling_params) { + // Cycle image every slide_change_interval seconds. + return test::CreateFromYuvFileFrameGenerator( + slides, video_config.width, video_config.height, + screen_share_config.slide_change_interval.seconds() * video_config.fps); + } + + TimeDelta pause_duration = screen_share_config.slide_change_interval - + screen_share_config.scrolling_params->duration; + RTC_DCHECK(pause_duration >= TimeDelta::Zero()); + return test::CreateScrollingInputFromYuvFilesFrameGenerator( + Clock::GetRealTimeClock(), slides, + screen_share_config.scrolling_params->source_width, + screen_share_config.scrolling_params->source_height, video_config.width, + video_config.height, screen_share_config.scrolling_params->duration.ms(), + pause_duration.ms()); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_peer_connection_quality_test_frame_generator.h b/third_party/libwebrtc/api/test/create_peer_connection_quality_test_frame_generator.h new file mode 100644 index 0000000000..62043d140a --- /dev/null +++ b/third_party/libwebrtc/api/test/create_peer_connection_quality_test_frame_generator.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_CREATE_PEER_CONNECTION_QUALITY_TEST_FRAME_GENERATOR_H_ +#define API_TEST_CREATE_PEER_CONNECTION_QUALITY_TEST_FRAME_GENERATOR_H_ + +#include <memory> +#include <string> + +#include "absl/types/optional.h" +#include "api/test/frame_generator_interface.h" +#include "api/test/pclf/media_configuration.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Creates a frame generator that produces frames with small squares that move +// randomly towards the lower right corner. `type` has the default value +// FrameGeneratorInterface::OutputType::I420. video_config specifies frame +// weight and height. +std::unique_ptr<test::FrameGeneratorInterface> CreateSquareFrameGenerator( + const VideoConfig& video_config, + absl::optional<test::FrameGeneratorInterface::OutputType> type); + +// Creates a frame generator that plays frames from the yuv file. +std::unique_ptr<test::FrameGeneratorInterface> CreateFromYuvFileFrameGenerator( + const VideoConfig& video_config, + std::string filename); + +// Creates a proper frame generator for testing screen sharing. +std::unique_ptr<test::FrameGeneratorInterface> CreateScreenShareFrameGenerator( + const VideoConfig& video_config, + const ScreenShareConfig& screen_share_config); + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_CREATE_PEER_CONNECTION_QUALITY_TEST_FRAME_GENERATOR_H_ diff --git a/third_party/libwebrtc/api/test/create_peerconnection_quality_test_fixture.cc b/third_party/libwebrtc/api/test/create_peerconnection_quality_test_fixture.cc new file mode 100644 index 0000000000..e156991ed4 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_peerconnection_quality_test_fixture.cc @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_peerconnection_quality_test_fixture.h" + +#include <memory> +#include <utility> + +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/time_controller.h" +#include "test/pc/e2e/peer_connection_quality_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +std::unique_ptr<PeerConnectionE2EQualityTestFixture> +CreatePeerConnectionE2EQualityTestFixture( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer, + std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer) { + return std::make_unique<PeerConnectionE2EQualityTest>( + std::move(test_case_name), time_controller, + std::move(audio_quality_analyzer), std::move(video_quality_analyzer), + test::GetGlobalMetricsLogger()); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_peerconnection_quality_test_fixture.h b/third_party/libwebrtc/api/test/create_peerconnection_quality_test_fixture.h new file mode 100644 index 0000000000..a0b0d08dd4 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_peerconnection_quality_test_fixture.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_CREATE_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ +#define API_TEST_CREATE_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ + +#include <memory> +#include <string> + +#include "api/test/audio_quality_analyzer_interface.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/time_controller.h" +#include "api/test/video_quality_analyzer_interface.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// API is in development. Can be changed/removed without notice. + +// Create test fixture to establish test call between Alice and Bob. +// During the test Alice will be caller and Bob will answer the call. +// `test_case_name` is a name of test case, that will be used for all metrics +// reporting. +// `time_controller` is used to manage all rtc::Thread's and TaskQueue +// instances. Instance of `time_controller` have to outlive created fixture. +// Returns a non-null PeerConnectionE2EQualityTestFixture instance. +std::unique_ptr<PeerConnectionE2EQualityTestFixture> +CreatePeerConnectionE2EQualityTestFixture( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer, + std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer); + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_CREATE_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/create_simulcast_test_fixture.cc b/third_party/libwebrtc/api/test/create_simulcast_test_fixture.cc new file mode 100644 index 0000000000..024145dff0 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_simulcast_test_fixture.cc @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_simulcast_test_fixture.h" + +#include <memory> +#include <utility> + +#include "api/test/simulcast_test_fixture.h" +#include "modules/video_coding/utility/simulcast_test_fixture_impl.h" + +namespace webrtc { +namespace test { + +std::unique_ptr<SimulcastTestFixture> CreateSimulcastTestFixture( + std::unique_ptr<VideoEncoderFactory> encoder_factory, + std::unique_ptr<VideoDecoderFactory> decoder_factory, + SdpVideoFormat video_format) { + return std::make_unique<SimulcastTestFixtureImpl>( + std::move(encoder_factory), std::move(decoder_factory), video_format); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_simulcast_test_fixture.h b/third_party/libwebrtc/api/test/create_simulcast_test_fixture.h new file mode 100644 index 0000000000..87f229c009 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_simulcast_test_fixture.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_SIMULCAST_TEST_FIXTURE_H_ +#define API_TEST_CREATE_SIMULCAST_TEST_FIXTURE_H_ + +#include <memory> + +#include "api/test/simulcast_test_fixture.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" + +namespace webrtc { +namespace test { + +std::unique_ptr<SimulcastTestFixture> CreateSimulcastTestFixture( + std::unique_ptr<VideoEncoderFactory> encoder_factory, + std::unique_ptr<VideoDecoderFactory> decoder_factory, + SdpVideoFormat video_format); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_CREATE_SIMULCAST_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/create_time_controller.cc b/third_party/libwebrtc/api/test/create_time_controller.cc new file mode 100644 index 0000000000..d198f2b0fe --- /dev/null +++ b/third_party/libwebrtc/api/test/create_time_controller.cc @@ -0,0 +1,53 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_time_controller.h" + +#include <memory> + +#include "call/call.h" +#include "call/rtp_transport_config.h" +#include "call/rtp_transport_controller_send_factory_interface.h" +#include "test/time_controller/external_time_controller.h" +#include "test/time_controller/simulated_time_controller.h" + +namespace webrtc { + +std::unique_ptr<TimeController> CreateTimeController( + ControlledAlarmClock* alarm) { + return std::make_unique<ExternalTimeController>(alarm); +} + +std::unique_ptr<TimeController> CreateSimulatedTimeController() { + return std::make_unique<GlobalSimulatedTimeController>( + Timestamp::Seconds(10000)); +} + +std::unique_ptr<CallFactoryInterface> CreateTimeControllerBasedCallFactory( + TimeController* time_controller) { + class TimeControllerBasedCallFactory : public CallFactoryInterface { + public: + explicit TimeControllerBasedCallFactory(TimeController* time_controller) + : time_controller_(time_controller) {} + Call* CreateCall(const Call::Config& config) override { + RtpTransportConfig transportConfig = config.ExtractTransportConfig(); + + return Call::Create(config, time_controller_->GetClock(), + config.rtp_transport_controller_send_factory->Create( + transportConfig, time_controller_->GetClock())); + } + + private: + TimeController* time_controller_; + }; + return std::make_unique<TimeControllerBasedCallFactory>(time_controller); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_time_controller.h b/third_party/libwebrtc/api/test/create_time_controller.h new file mode 100644 index 0000000000..e7bc9cb465 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_time_controller.h @@ -0,0 +1,34 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_CREATE_TIME_CONTROLLER_H_ +#define API_TEST_CREATE_TIME_CONTROLLER_H_ + +#include <memory> + +#include "api/call/call_factory_interface.h" +#include "api/test/time_controller.h" + +namespace webrtc { + +// Creates a time coltroller that wraps `alarm`. +std::unique_ptr<TimeController> CreateTimeController( + ControlledAlarmClock* alarm); + +// Creates a time controller that runs in simulated time. +std::unique_ptr<TimeController> CreateSimulatedTimeController(); + +// This is creates a call factory that creates Call instances that are backed by +// a time controller. +std::unique_ptr<CallFactoryInterface> CreateTimeControllerBasedCallFactory( + TimeController* time_controller); + +} // namespace webrtc + +#endif // API_TEST_CREATE_TIME_CONTROLLER_H_ diff --git a/third_party/libwebrtc/api/test/create_time_controller_unittest.cc b/third_party/libwebrtc/api/test/create_time_controller_unittest.cc new file mode 100644 index 0000000000..0ea868c5cc --- /dev/null +++ b/third_party/libwebrtc/api/test/create_time_controller_unittest.cc @@ -0,0 +1,76 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_time_controller.h" + +#include "api/test/time_controller.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +class FakeAlarm : public ControlledAlarmClock { + public: + explicit FakeAlarm(Timestamp start_time); + + Clock* GetClock() override; + bool ScheduleAlarmAt(Timestamp deadline) override; + void SetCallback(std::function<void()> callback) override; + void Sleep(TimeDelta duration) override; + + private: + SimulatedClock clock_; + Timestamp deadline_; + std::function<void()> callback_; +}; + +FakeAlarm::FakeAlarm(Timestamp start_time) + : clock_(start_time), + deadline_(Timestamp::PlusInfinity()), + callback_([] {}) {} + +Clock* FakeAlarm::GetClock() { + return &clock_; +} + +bool FakeAlarm::ScheduleAlarmAt(Timestamp deadline) { + if (deadline < deadline_) { + deadline_ = deadline; + return true; + } + return false; +} + +void FakeAlarm::SetCallback(std::function<void()> callback) { + callback_ = callback; +} + +void FakeAlarm::Sleep(TimeDelta duration) { + Timestamp end_time = clock_.CurrentTime() + duration; + + while (deadline_ <= end_time) { + clock_.AdvanceTime(deadline_ - clock_.CurrentTime()); + deadline_ = Timestamp::PlusInfinity(); + callback_(); + } + + clock_.AdvanceTime(end_time - clock_.CurrentTime()); +} + +TEST(CreateTimeControllerTest, CreatesNonNullController) { + FakeAlarm alarm(Timestamp::Millis(100)); + EXPECT_NE(CreateTimeController(&alarm), nullptr); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_video_codec_tester.cc b/third_party/libwebrtc/api/test/create_video_codec_tester.cc new file mode 100644 index 0000000000..a1efefdb48 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_video_codec_tester.cc @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_video_codec_tester.h" + +#include <memory> +#include <utility> + +#include "api/test/video_codec_tester.h" +#include "modules/video_coding/codecs/test/video_codec_tester_impl.h" + +namespace webrtc { +namespace test { + +std::unique_ptr<VideoCodecTester> CreateVideoCodecTester() { + return std::make_unique<VideoCodecTesterImpl>(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_video_codec_tester.h b/third_party/libwebrtc/api/test/create_video_codec_tester.h new file mode 100644 index 0000000000..c68864ce85 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_video_codec_tester.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ +#define API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ + +#include <memory> + +#include "api/test/video_codec_tester.h" + +namespace webrtc { +namespace test { + +std::unique_ptr<VideoCodecTester> CreateVideoCodecTester(); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_CREATE_VIDEO_CODEC_TESTER_H_ diff --git a/third_party/libwebrtc/api/test/create_video_quality_test_fixture.cc b/third_party/libwebrtc/api/test/create_video_quality_test_fixture.cc new file mode 100644 index 0000000000..1fa7d243cc --- /dev/null +++ b/third_party/libwebrtc/api/test/create_video_quality_test_fixture.cc @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_video_quality_test_fixture.h" + +#include <memory> +#include <utility> + +#include "video/video_quality_test.h" + +namespace webrtc { + +std::unique_ptr<VideoQualityTestFixtureInterface> +CreateVideoQualityTestFixture() { + // By default, we don't override the FEC module, so pass an empty factory. + return std::make_unique<VideoQualityTest>(nullptr); +} + +std::unique_ptr<VideoQualityTestFixtureInterface> CreateVideoQualityTestFixture( + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory) { + auto components = + std::make_unique<VideoQualityTestFixtureInterface::InjectionComponents>(); + components->fec_controller_factory = std::move(fec_controller_factory); + return std::make_unique<VideoQualityTest>(std::move(components)); +} + +std::unique_ptr<VideoQualityTestFixtureInterface> CreateVideoQualityTestFixture( + std::unique_ptr<VideoQualityTestFixtureInterface::InjectionComponents> + components) { + return std::make_unique<VideoQualityTest>(std::move(components)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_video_quality_test_fixture.h b/third_party/libwebrtc/api/test/create_video_quality_test_fixture.h new file mode 100644 index 0000000000..ed618fefc8 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_video_quality_test_fixture.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_CREATE_VIDEO_QUALITY_TEST_FIXTURE_H_ +#define API_TEST_CREATE_VIDEO_QUALITY_TEST_FIXTURE_H_ + +#include <memory> + +#include "api/fec_controller.h" +#include "api/test/video_quality_test_fixture.h" + +namespace webrtc { + +std::unique_ptr<VideoQualityTestFixtureInterface> +CreateVideoQualityTestFixture(); + +std::unique_ptr<VideoQualityTestFixtureInterface> CreateVideoQualityTestFixture( + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory); + +std::unique_ptr<VideoQualityTestFixtureInterface> CreateVideoQualityTestFixture( + std::unique_ptr<VideoQualityTestFixtureInterface::InjectionComponents> + components); +} // namespace webrtc + +#endif // API_TEST_CREATE_VIDEO_QUALITY_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/create_videocodec_test_fixture.cc b/third_party/libwebrtc/api/test/create_videocodec_test_fixture.cc new file mode 100644 index 0000000000..1f618e5db8 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_videocodec_test_fixture.cc @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/create_videocodec_test_fixture.h" + +#include <memory> +#include <utility> + +#include "api/test/videocodec_test_fixture.h" +#include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h" + +namespace webrtc { +namespace test { + +using Config = VideoCodecTestFixture::Config; + +std::unique_ptr<VideoCodecTestFixture> CreateVideoCodecTestFixture( + const Config& config) { + return std::make_unique<VideoCodecTestFixtureImpl>(config); +} + +std::unique_ptr<VideoCodecTestFixture> CreateVideoCodecTestFixture( + const Config& config, + std::unique_ptr<VideoDecoderFactory> decoder_factory, + std::unique_ptr<VideoEncoderFactory> encoder_factory) { + return std::make_unique<VideoCodecTestFixtureImpl>( + config, std::move(decoder_factory), std::move(encoder_factory)); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/create_videocodec_test_fixture.h b/third_party/libwebrtc/api/test/create_videocodec_test_fixture.h new file mode 100644 index 0000000000..7a44f6b058 --- /dev/null +++ b/third_party/libwebrtc/api/test/create_videocodec_test_fixture.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_CREATE_VIDEOCODEC_TEST_FIXTURE_H_ +#define API_TEST_CREATE_VIDEOCODEC_TEST_FIXTURE_H_ + +#include <memory> + +#include "api/test/videocodec_test_fixture.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" + +namespace webrtc { +namespace test { + +std::unique_ptr<VideoCodecTestFixture> CreateVideoCodecTestFixture( + const VideoCodecTestFixture::Config& config); + +std::unique_ptr<VideoCodecTestFixture> CreateVideoCodecTestFixture( + const VideoCodecTestFixture::Config& config, + std::unique_ptr<VideoDecoderFactory> decoder_factory, + std::unique_ptr<VideoEncoderFactory> encoder_factory); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_CREATE_VIDEOCODEC_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/fake_frame_decryptor.cc b/third_party/libwebrtc/api/test/fake_frame_decryptor.cc new file mode 100644 index 0000000000..16cb1bd3b6 --- /dev/null +++ b/third_party/libwebrtc/api/test/fake_frame_decryptor.cc @@ -0,0 +1,71 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/fake_frame_decryptor.h" + +#include <vector> + +#include "rtc_base/checks.h" + +namespace webrtc { + +FakeFrameDecryptor::FakeFrameDecryptor(uint8_t fake_key, + uint8_t expected_postfix_byte) + : fake_key_(fake_key), expected_postfix_byte_(expected_postfix_byte) {} + +FakeFrameDecryptor::Result FakeFrameDecryptor::Decrypt( + cricket::MediaType media_type, + const std::vector<uint32_t>& csrcs, + rtc::ArrayView<const uint8_t> additional_data, + rtc::ArrayView<const uint8_t> encrypted_frame, + rtc::ArrayView<uint8_t> frame) { + if (fail_decryption_) { + return Result(Status::kFailedToDecrypt, 0); + } + + RTC_CHECK_EQ(frame.size() + 1, encrypted_frame.size()); + for (size_t i = 0; i < frame.size(); i++) { + frame[i] = encrypted_frame[i] ^ fake_key_; + } + + if (encrypted_frame[frame.size()] != expected_postfix_byte_) { + return Result(Status::kFailedToDecrypt, 0); + } + + return Result(Status::kOk, frame.size()); +} + +size_t FakeFrameDecryptor::GetMaxPlaintextByteSize( + cricket::MediaType media_type, + size_t encrypted_frame_size) { + return encrypted_frame_size - 1; +} + +void FakeFrameDecryptor::SetFakeKey(uint8_t fake_key) { + fake_key_ = fake_key; +} + +uint8_t FakeFrameDecryptor::GetFakeKey() const { + return fake_key_; +} + +void FakeFrameDecryptor::SetExpectedPostfixByte(uint8_t expected_postfix_byte) { + expected_postfix_byte_ = expected_postfix_byte; +} + +uint8_t FakeFrameDecryptor::GetExpectedPostfixByte() const { + return expected_postfix_byte_; +} + +void FakeFrameDecryptor::SetFailDecryption(bool fail_decryption) { + fail_decryption_ = fail_decryption; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/fake_frame_decryptor.h b/third_party/libwebrtc/api/test/fake_frame_decryptor.h new file mode 100644 index 0000000000..783bc805c4 --- /dev/null +++ b/third_party/libwebrtc/api/test/fake_frame_decryptor.h @@ -0,0 +1,70 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_FAKE_FRAME_DECRYPTOR_H_ +#define API_TEST_FAKE_FRAME_DECRYPTOR_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <vector> + +#include "api/array_view.h" +#include "api/crypto/frame_decryptor_interface.h" +#include "api/media_types.h" + +namespace webrtc { + +// The FakeFrameDecryptor is a TEST ONLY fake implementation of the +// FrameDecryptorInterface. It is constructed with a simple single digit key and +// a fixed postfix byte. This is just to validate that the core code works +// as expected. +class FakeFrameDecryptor : public FrameDecryptorInterface { + public: + // Provide a key (0,255) and some postfix byte (0,255) this should match the + // byte you expect from the FakeFrameEncryptor. + explicit FakeFrameDecryptor(uint8_t fake_key = 0xAA, + uint8_t expected_postfix_byte = 255); + // Fake decryption that just xors the payload with the 1 byte key and checks + // the postfix byte. This will always fail if fail_decryption_ is set to true. + Result Decrypt(cricket::MediaType media_type, + const std::vector<uint32_t>& csrcs, + rtc::ArrayView<const uint8_t> additional_data, + rtc::ArrayView<const uint8_t> encrypted_frame, + rtc::ArrayView<uint8_t> frame) override; + // Always returns 1 less than the size of the encrypted frame. + size_t GetMaxPlaintextByteSize(cricket::MediaType media_type, + size_t encrypted_frame_size) override; + // Sets the fake key to use for encryption. + void SetFakeKey(uint8_t fake_key); + // Returns the fake key used for encryption. + uint8_t GetFakeKey() const; + // Set the Postfix byte that is expected in the encrypted payload. + void SetExpectedPostfixByte(uint8_t expected_postfix_byte); + // Returns the postfix byte that will be checked for in the encrypted payload. + uint8_t GetExpectedPostfixByte() const; + // If set to true will force all encryption to fail. + void SetFailDecryption(bool fail_decryption); + // Simple error codes for tests to validate against. + enum class FakeDecryptStatus : int { + OK = 0, + FORCED_FAILURE = 1, + INVALID_POSTFIX = 2 + }; + + private: + uint8_t fake_key_ = 0; + uint8_t expected_postfix_byte_ = 0; + bool fail_decryption_ = false; +}; + +} // namespace webrtc + +#endif // API_TEST_FAKE_FRAME_DECRYPTOR_H_ diff --git a/third_party/libwebrtc/api/test/fake_frame_encryptor.cc b/third_party/libwebrtc/api/test/fake_frame_encryptor.cc new file mode 100644 index 0000000000..89d14aab88 --- /dev/null +++ b/third_party/libwebrtc/api/test/fake_frame_encryptor.cc @@ -0,0 +1,66 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/fake_frame_encryptor.h" + +#include "rtc_base/checks.h" + +namespace webrtc { +FakeFrameEncryptor::FakeFrameEncryptor(uint8_t fake_key, uint8_t postfix_byte) + : fake_key_(fake_key), postfix_byte_(postfix_byte) {} + +// FrameEncryptorInterface implementation +int FakeFrameEncryptor::Encrypt(cricket::MediaType media_type, + uint32_t ssrc, + rtc::ArrayView<const uint8_t> additional_data, + rtc::ArrayView<const uint8_t> frame, + rtc::ArrayView<uint8_t> encrypted_frame, + size_t* bytes_written) { + if (fail_encryption_) { + return static_cast<int>(FakeEncryptionStatus::FORCED_FAILURE); + } + + RTC_CHECK_EQ(frame.size() + 1, encrypted_frame.size()); + for (size_t i = 0; i < frame.size(); i++) { + encrypted_frame[i] = frame[i] ^ fake_key_; + } + + encrypted_frame[frame.size()] = postfix_byte_; + *bytes_written = encrypted_frame.size(); + return static_cast<int>(FakeEncryptionStatus::OK); +} + +size_t FakeFrameEncryptor::GetMaxCiphertextByteSize( + cricket::MediaType media_type, + size_t frame_size) { + return frame_size + 1; +} + +void FakeFrameEncryptor::SetFakeKey(uint8_t fake_key) { + fake_key_ = fake_key; +} + +uint8_t FakeFrameEncryptor::GetFakeKey() const { + return fake_key_; +} + +void FakeFrameEncryptor::SetPostfixByte(uint8_t postfix_byte) { + postfix_byte_ = postfix_byte; +} + +uint8_t FakeFrameEncryptor::GetPostfixByte() const { + return postfix_byte_; +} + +void FakeFrameEncryptor::SetFailEncryption(bool fail_encryption) { + fail_encryption_ = fail_encryption; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/fake_frame_encryptor.h b/third_party/libwebrtc/api/test/fake_frame_encryptor.h new file mode 100644 index 0000000000..074981b183 --- /dev/null +++ b/third_party/libwebrtc/api/test/fake_frame_encryptor.h @@ -0,0 +1,69 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_FAKE_FRAME_ENCRYPTOR_H_ +#define API_TEST_FAKE_FRAME_ENCRYPTOR_H_ + +#include <stddef.h> +#include <stdint.h> + +#include "api/array_view.h" +#include "api/crypto/frame_encryptor_interface.h" +#include "api/media_types.h" +#include "rtc_base/ref_counted_object.h" + +namespace webrtc { + +// The FakeFrameEncryptor is a TEST ONLY fake implementation of the +// FrameEncryptorInterface. It is constructed with a simple single digit key and +// a fixed postfix byte. This is just to validate that the core code works +// as expected. +class FakeFrameEncryptor + : public rtc::RefCountedObject<FrameEncryptorInterface> { + public: + // Provide a key (0,255) and some postfix byte (0,255). + explicit FakeFrameEncryptor(uint8_t fake_key = 0xAA, + uint8_t postfix_byte = 255); + // Simply xors each payload with the provided fake key and adds the postfix + // bit to the end. This will always fail if fail_encryption_ is set to true. + int Encrypt(cricket::MediaType media_type, + uint32_t ssrc, + rtc::ArrayView<const uint8_t> additional_data, + rtc::ArrayView<const uint8_t> frame, + rtc::ArrayView<uint8_t> encrypted_frame, + size_t* bytes_written) override; + // Always returns 1 more than the size of the frame. + size_t GetMaxCiphertextByteSize(cricket::MediaType media_type, + size_t frame_size) override; + // Sets the fake key to use during encryption. + void SetFakeKey(uint8_t fake_key); + // Returns the fake key used during encryption. + uint8_t GetFakeKey() const; + // Set the postfix byte to use. + void SetPostfixByte(uint8_t expected_postfix_byte); + // Return a postfix byte added to each outgoing payload. + uint8_t GetPostfixByte() const; + // Force all encryptions to fail. + void SetFailEncryption(bool fail_encryption); + + enum class FakeEncryptionStatus : int { + OK = 0, + FORCED_FAILURE = 1, + }; + + private: + uint8_t fake_key_ = 0; + uint8_t postfix_byte_ = 0; + bool fail_encryption_ = false; +}; + +} // namespace webrtc + +#endif // API_TEST_FAKE_FRAME_ENCRYPTOR_H_ diff --git a/third_party/libwebrtc/api/test/frame_generator_interface.cc b/third_party/libwebrtc/api/test/frame_generator_interface.cc new file mode 100644 index 0000000000..fe7b1e883d --- /dev/null +++ b/third_party/libwebrtc/api/test/frame_generator_interface.cc @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/frame_generator_interface.h" + +namespace webrtc { +namespace test { + +// static +const char* FrameGeneratorInterface::OutputTypeToString( + FrameGeneratorInterface::OutputType type) { + switch (type) { + case OutputType::kI420: + return "I420"; + case OutputType::kI420A: + return "I420A"; + case OutputType::kI010: + return "I010"; + case OutputType::kNV12: + return "NV12"; + default: + RTC_DCHECK_NOTREACHED(); + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/frame_generator_interface.h b/third_party/libwebrtc/api/test/frame_generator_interface.h new file mode 100644 index 0000000000..90e60debac --- /dev/null +++ b/third_party/libwebrtc/api/test/frame_generator_interface.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_FRAME_GENERATOR_INTERFACE_H_ +#define API_TEST_FRAME_GENERATOR_INTERFACE_H_ + +#include <utility> + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" + +namespace webrtc { +namespace test { + +class FrameGeneratorInterface { + public: + struct VideoFrameData { + VideoFrameData(rtc::scoped_refptr<VideoFrameBuffer> buffer, + absl::optional<VideoFrame::UpdateRect> update_rect) + : buffer(std::move(buffer)), update_rect(update_rect) {} + + rtc::scoped_refptr<VideoFrameBuffer> buffer; + absl::optional<VideoFrame::UpdateRect> update_rect; + }; + + enum class OutputType { kI420, kI420A, kI010, kNV12 }; + static const char* OutputTypeToString(OutputType type); + + virtual ~FrameGeneratorInterface() = default; + + // Returns VideoFrameBuffer and area where most of update was done to set them + // on the VideoFrame object. + virtual VideoFrameData NextFrame() = 0; + + // Change the capture resolution. + virtual void ChangeResolution(size_t width, size_t height) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_FRAME_GENERATOR_INTERFACE_H_ diff --git a/third_party/libwebrtc/api/test/metrics/BUILD.gn b/third_party/libwebrtc/api/test/metrics/BUILD.gn new file mode 100644 index 0000000000..309b699329 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/BUILD.gn @@ -0,0 +1,281 @@ +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") +if (rtc_enable_protobuf) { + import("//third_party/protobuf/proto_library.gni") +} + +group("metrics") { + deps = [ + ":global_metrics_logger_and_exporter", + ":metric", + ":metrics_accumulator", + ":metrics_exporter", + ":metrics_logger", + ":stdout_metrics_exporter", + ] +} + +if (rtc_include_tests) { + group("metrics_unittests") { + testonly = true + + deps = [ + ":global_metrics_logger_and_exporter_test", + ":metrics_accumulator_test", + ":metrics_logger_test", + ":print_result_proxy_metrics_exporter_test", + ":stdout_metrics_exporter_test", + ] + + if (rtc_enable_protobuf) { + deps += [ + ":chrome_perf_dashboard_metrics_exporter_test", + ":metrics_set_proto_file_exporter_test", + ] + } + } +} + +rtc_library("metric") { + visibility = [ "*" ] + sources = [ + "metric.cc", + "metric.h", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + deps = [ "../../../api/units:timestamp" ] +} + +rtc_library("metrics_logger") { + visibility = [ "*" ] + sources = [ + "metrics_logger.cc", + "metrics_logger.h", + ] + deps = [ + ":metric", + ":metrics_accumulator", + "../..:array_view", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers", + "../../numerics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("metrics_accumulator") { + visibility = [ "*" ] + sources = [ + "metrics_accumulator.cc", + "metrics_accumulator.h", + ] + deps = [ + ":metric", + "../../../rtc_base:macromagic", + "../../../rtc_base/synchronization:mutex", + "../../numerics", + "../../units:timestamp", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("metrics_exporter") { + visibility = [ "*" ] + sources = [ "metrics_exporter.h" ] + deps = [ + ":metric", + "../..:array_view", + ] +} + +rtc_library("stdout_metrics_exporter") { + visibility = [ "*" ] + sources = [ + "stdout_metrics_exporter.cc", + "stdout_metrics_exporter.h", + ] + deps = [ + ":metric", + ":metrics_exporter", + "../..:array_view", + "../../../rtc_base:stringutils", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("chrome_perf_dashboard_metrics_exporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "chrome_perf_dashboard_metrics_exporter.cc", + "chrome_perf_dashboard_metrics_exporter.h", + ] + deps = [ + ":metric", + ":metrics_exporter", + "../../../api:array_view", + "../../../test:fileutils", + "../../../test:perf_test", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + ] +} + +if (rtc_enable_protobuf) { + proto_library("metric_proto") { + visibility = [ "*" ] + sources = [ "proto/metric.proto" ] + proto_out_dir = "api/test/metrics/proto" + cc_generator_options = "lite" + } +} + +rtc_library("metrics_set_proto_file_exporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "metrics_set_proto_file_exporter.cc", + "metrics_set_proto_file_exporter.h", + ] + deps = [ + ":metric", + ":metrics_exporter", + "../..:array_view", + "../../../rtc_base:logging", + "../../../test:fileutils", + ] + + if (rtc_enable_protobuf) { + deps += [ ":metric_proto" ] + } +} + +rtc_library("print_result_proxy_metrics_exporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "print_result_proxy_metrics_exporter.cc", + "print_result_proxy_metrics_exporter.h", + ] + deps = [ + ":metric", + ":metrics_exporter", + "../..:array_view", + "../../../test:perf_test", + ] +} + +rtc_library("global_metrics_logger_and_exporter") { + visibility = [ "*" ] + sources = [ + "global_metrics_logger_and_exporter.cc", + "global_metrics_logger_and_exporter.h", + ] + deps = [ + ":metrics_exporter", + ":metrics_logger", + "../../../rtc_base:checks", + "../../../system_wrappers", + ] +} + +if (rtc_include_tests) { + rtc_library("metrics_logger_test") { + testonly = true + sources = [ "metrics_logger_test.cc" ] + deps = [ + ":metric", + ":metrics_logger", + "../../../system_wrappers", + "../../../test:test_support", + "../../numerics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("metrics_accumulator_test") { + testonly = true + sources = [ "metrics_accumulator_test.cc" ] + deps = [ + ":metric", + ":metrics_accumulator", + "../../../test:test_support", + "../../units:timestamp", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("stdout_metrics_exporter_test") { + testonly = true + sources = [ "stdout_metrics_exporter_test.cc" ] + deps = [ + ":metric", + ":stdout_metrics_exporter", + "../../../test:test_support", + "../../units:timestamp", + ] + } + + rtc_library("print_result_proxy_metrics_exporter_test") { + testonly = true + sources = [ "print_result_proxy_metrics_exporter_test.cc" ] + deps = [ + ":metric", + ":print_result_proxy_metrics_exporter", + "../../../test:test_support", + "../../units:timestamp", + ] + } + + rtc_library("global_metrics_logger_and_exporter_test") { + testonly = true + sources = [ "global_metrics_logger_and_exporter_test.cc" ] + deps = [ + ":global_metrics_logger_and_exporter", + ":metric", + ":metrics_exporter", + ":metrics_logger", + "../../../system_wrappers", + "../../../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + if (rtc_enable_protobuf) { + rtc_library("metrics_set_proto_file_exporter_test") { + testonly = true + sources = [ "metrics_set_proto_file_exporter_test.cc" ] + deps = [ + ":metric", + ":metric_proto", + ":metrics_set_proto_file_exporter", + "../../../rtc_base:protobuf_utils", + "../../../test:fileutils", + "../../../test:test_support", + "../../units:timestamp", + ] + } + + rtc_library("chrome_perf_dashboard_metrics_exporter_test") { + testonly = true + sources = [ "chrome_perf_dashboard_metrics_exporter_test.cc" ] + deps = [ + ":chrome_perf_dashboard_metrics_exporter", + ":metric", + "../../../api/units:timestamp", + "../../../test:fileutils", + "../../../test:test_support", + "//third_party/catapult/tracing/tracing:histogram", + ] + } + } +} diff --git a/third_party/libwebrtc/api/test/metrics/DEPS b/third_party/libwebrtc/api/test/metrics/DEPS new file mode 100644 index 0000000000..74889c61c7 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/DEPS @@ -0,0 +1,14 @@ +specific_include_rules = { + "metrics_logger_and_exporter\.h": [ + "+rtc_base/synchronization/mutex.h", + "+system_wrappers/include/clock.h", + ], + "metrics_logger\.h": [ + "+rtc_base/synchronization/mutex.h", + "+system_wrappers/include/clock.h", + ], + "metrics_accumulator\.h": [ + "+rtc_base/synchronization/mutex.h", + "+rtc_base/thread_annotations.h", + ], +} diff --git a/third_party/libwebrtc/api/test/metrics/chrome_perf_dashboard_metrics_exporter.cc b/third_party/libwebrtc/api/test/metrics/chrome_perf_dashboard_metrics_exporter.cc new file mode 100644 index 0000000000..018d110b12 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/chrome_perf_dashboard_metrics_exporter.cc @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/chrome_perf_dashboard_metrics_exporter.h" + +#include <stdio.h> + +#include <memory> +#include <string> +#include <vector> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/perf_test_histogram_writer.h" +#include "test/testsupport/perf_test_result_writer.h" + +namespace webrtc { +namespace test { +namespace { + +std::string ToChromePerfDashboardUnit(Unit unit) { + switch (unit) { + case Unit::kMilliseconds: + return "msBestFitFormat"; + case Unit::kPercent: + return "n%"; + case Unit::kBytes: + return "sizeInBytes"; + case Unit::kKilobitsPerSecond: + // Chrome Perf Dashboard doesn't have kpbs units, so we change the unit + // and value accordingly. + return "bytesPerSecond"; + case Unit::kHertz: + return "Hz"; + case Unit::kUnitless: + return "unitless"; + case Unit::kCount: + return "count"; + } +} + +double ToChromePerfDashboardValue(double value, Unit unit) { + switch (unit) { + case Unit::kKilobitsPerSecond: + // Chrome Perf Dashboard doesn't have kpbs units, so we change the unit + // and value accordingly. + return value * 1000 / 8; + default: + return value; + } +} + +ImproveDirection ToChromePerfDashboardImproveDirection( + ImprovementDirection direction) { + switch (direction) { + case ImprovementDirection::kBiggerIsBetter: + return ImproveDirection::kBiggerIsBetter; + case ImprovementDirection::kNeitherIsBetter: + return ImproveDirection::kNone; + case ImprovementDirection::kSmallerIsBetter: + return ImproveDirection::kSmallerIsBetter; + } +} + +bool WriteMetricsToFile(const std::string& path, const std::string& data) { + CreateDir(DirName(path)); + FILE* output = fopen(path.c_str(), "wb"); + if (output == NULL) { + printf("Failed to write to %s.\n", path.c_str()); + return false; + } + size_t written = fwrite(data.c_str(), sizeof(char), data.size(), output); + fclose(output); + + if (written != data.size()) { + size_t expected = data.size(); + printf("Wrote %zu, tried to write %zu\n", written, expected); + return false; + } + return true; +} + +bool IsEmpty(const Metric::Stats& stats) { + return !stats.mean.has_value() && !stats.stddev.has_value() && + !stats.min.has_value() && !stats.max.has_value(); +} + +} // namespace + +ChromePerfDashboardMetricsExporter::ChromePerfDashboardMetricsExporter( + absl::string_view export_file_path) + : export_file_path_(export_file_path) {} + +bool ChromePerfDashboardMetricsExporter::Export( + rtc::ArrayView<const Metric> metrics) { + std::unique_ptr<PerfTestResultWriter> writer = + absl::WrapUnique<PerfTestResultWriter>(CreateHistogramWriter()); + for (const Metric& metric : metrics) { + if (metric.time_series.samples.empty() && IsEmpty(metric.stats)) { + // If there were no data collected for the metric it is expected that 0 + // will be exported, so add 0 to the samples. + writer->LogResult( + metric.name, metric.test_case, + ToChromePerfDashboardValue(0, metric.unit), + ToChromePerfDashboardUnit(metric.unit), + /*important=*/false, + ToChromePerfDashboardImproveDirection(metric.improvement_direction)); + continue; + } + + if (metric.time_series.samples.empty()) { + writer->LogResultMeanAndError( + metric.name, metric.test_case, + ToChromePerfDashboardValue(*metric.stats.mean, metric.unit), + ToChromePerfDashboardValue(*metric.stats.stddev, metric.unit), + ToChromePerfDashboardUnit(metric.unit), + /*important=*/false, + ToChromePerfDashboardImproveDirection(metric.improvement_direction)); + continue; + } + + std::vector<double> samples(metric.time_series.samples.size()); + for (size_t i = 0; i < metric.time_series.samples.size(); ++i) { + samples[i] = ToChromePerfDashboardValue( + metric.time_series.samples[i].value, metric.unit); + } + writer->LogResultList( + metric.name, metric.test_case, samples, + ToChromePerfDashboardUnit(metric.unit), + /*important=*/false, + ToChromePerfDashboardImproveDirection(metric.improvement_direction)); + } + return WriteMetricsToFile(export_file_path_, writer->Serialize()); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/chrome_perf_dashboard_metrics_exporter.h b/third_party/libwebrtc/api/test/metrics/chrome_perf_dashboard_metrics_exporter.h new file mode 100644 index 0000000000..dda17a08c6 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/chrome_perf_dashboard_metrics_exporter.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_CHROME_PERF_DASHBOARD_METRICS_EXPORTER_H_ +#define API_TEST_METRICS_CHROME_PERF_DASHBOARD_METRICS_EXPORTER_H_ + +#include <string> + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_exporter.h" + +namespace webrtc { +namespace test { + +// Exports all collected metrics in the Chrome Perf Dashboard proto format. +class ChromePerfDashboardMetricsExporter : public MetricsExporter { + public: + // `export_file_path` - path where the proto file will be written. + explicit ChromePerfDashboardMetricsExporter( + absl::string_view export_file_path); + ~ChromePerfDashboardMetricsExporter() override = default; + + bool Export(rtc::ArrayView<const Metric> metrics) override; + + private: + const std::string export_file_path_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_CHROME_PERF_DASHBOARD_METRICS_EXPORTER_H_ diff --git a/third_party/libwebrtc/api/test/metrics/chrome_perf_dashboard_metrics_exporter_test.cc b/third_party/libwebrtc/api/test/metrics/chrome_perf_dashboard_metrics_exporter_test.cc new file mode 100644 index 0000000000..5d3136f49a --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/chrome_perf_dashboard_metrics_exporter_test.cc @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/chrome_perf_dashboard_metrics_exporter.h" + +#include <fstream> +#include <map> +#include <vector> + +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "third_party/catapult/tracing/tracing/value/histogram.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::DoubleNear; +using ::testing::Eq; +using ::testing::Test; + +namespace proto = ::catapult::tracing::tracing::proto; + +std::map<std::string, std::string> DefaultMetadata() { + return std::map<std::string, std::string>{{"key", "value"}}; +} + +Metric::TimeSeries::Sample Sample(double value) { + return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1), + .value = value, + .sample_metadata = DefaultMetadata()}; +} + +std::string ReadFileAsString(const std::string& filename) { + std::ifstream infile(filename, std::ios_base::binary); + auto buffer = std::vector<char>(std::istreambuf_iterator<char>(infile), + std::istreambuf_iterator<char>()); + return std::string(buffer.begin(), buffer.end()); +} + +class ChromePerfDashboardMetricsExporterTest : public Test { + protected: + ~ChromePerfDashboardMetricsExporterTest() override = default; + + void SetUp() override { + temp_filename_ = webrtc::test::TempFilename( + webrtc::test::OutputPath(), + "chrome_perf_dashboard_metrics_exporter_test"); + } + + void TearDown() override { + ASSERT_TRUE(webrtc::test::RemoveFile(temp_filename_)); + } + + std::string temp_filename_; +}; + +TEST_F(ChromePerfDashboardMetricsExporterTest, ExportMetricFormatCorrect) { + Metric metric1{ + .name = "test_metric1", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name1", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}}, + .stats = + Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}}; + Metric metric2{ + .name = "test_metric2", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .test_case = "test_case_name2", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}}, + .stats = Metric::Stats{ + .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}}; + + ChromePerfDashboardMetricsExporter exporter(temp_filename_); + + ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2})); + proto::HistogramSet actual_histogram_set; + actual_histogram_set.ParseFromString(ReadFileAsString(temp_filename_)); + EXPECT_THAT(actual_histogram_set.histograms().size(), Eq(2)); + + // Validate output for `metric1` + EXPECT_THAT(actual_histogram_set.histograms(0).name(), Eq("test_metric1")); + EXPECT_THAT(actual_histogram_set.histograms(0).unit().unit(), + Eq(proto::Unit::MS_BEST_FIT_FORMAT)); + EXPECT_THAT(actual_histogram_set.histograms(0).unit().improvement_direction(), + Eq(proto::ImprovementDirection::BIGGER_IS_BETTER)); + EXPECT_THAT( + actual_histogram_set.histograms(0).diagnostics().diagnostic_map().size(), + Eq(1lu)); + EXPECT_THAT(actual_histogram_set.histograms(0) + .diagnostics() + .diagnostic_map() + .at("stories") + .generic_set() + .values(0), + Eq("\"test_case_name1\"")); + EXPECT_THAT(actual_histogram_set.histograms(0).sample_values().size(), Eq(2)); + EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(0), Eq(10.0)); + EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(1), Eq(20.0)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().count(), Eq(2)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().max(), Eq(20)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().meanlogs(), + DoubleNear(2.64916, 0.1)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().mean(), Eq(15)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().min(), Eq(10)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().sum(), Eq(30)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().variance(), Eq(50)); + + // Validate output for `metric2` + EXPECT_THAT(actual_histogram_set.histograms(1).name(), Eq("test_metric2")); + EXPECT_THAT(actual_histogram_set.histograms(1).unit().unit(), + Eq(proto::Unit::BYTES_PER_SECOND)); + EXPECT_THAT(actual_histogram_set.histograms(1).unit().improvement_direction(), + Eq(proto::ImprovementDirection::SMALLER_IS_BETTER)); + EXPECT_THAT( + actual_histogram_set.histograms(1).diagnostics().diagnostic_map().size(), + Eq(1lu)); + EXPECT_THAT(actual_histogram_set.histograms(1) + .diagnostics() + .diagnostic_map() + .at("stories") + .generic_set() + .values(0), + Eq("\"test_case_name2\"")); + EXPECT_THAT(actual_histogram_set.histograms(1).sample_values().size(), Eq(2)); + EXPECT_THAT(actual_histogram_set.histograms(1).sample_values(0), Eq(2500.0)); + EXPECT_THAT(actual_histogram_set.histograms(1).sample_values(1), Eq(5000.0)); + EXPECT_THAT(actual_histogram_set.histograms(1).running().count(), Eq(2)); + EXPECT_THAT(actual_histogram_set.histograms(1).running().max(), Eq(5000)); + EXPECT_THAT(actual_histogram_set.histograms(1).running().meanlogs(), + DoubleNear(8.17062, 0.1)); + EXPECT_THAT(actual_histogram_set.histograms(1).running().mean(), Eq(3750)); + EXPECT_THAT(actual_histogram_set.histograms(1).running().min(), Eq(2500)); + EXPECT_THAT(actual_histogram_set.histograms(1).running().sum(), Eq(7500)); + EXPECT_THAT(actual_histogram_set.histograms(1).running().variance(), + Eq(3125000)); +} + +TEST_F(ChromePerfDashboardMetricsExporterTest, + ExportEmptyMetricExportsZeroValue) { + Metric metric{.name = "test_metric", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name", + .metric_metadata = DefaultMetadata(), + .time_series = Metric::TimeSeries{.samples = {}}, + .stats = Metric::Stats{}}; + + ChromePerfDashboardMetricsExporter exporter(temp_filename_); + + ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric})); + proto::HistogramSet actual_histogram_set; + actual_histogram_set.ParseFromString(ReadFileAsString(temp_filename_)); + EXPECT_THAT(actual_histogram_set.histograms().size(), Eq(1)); + + // Validate values for `metric` + EXPECT_THAT(actual_histogram_set.histograms(0).sample_values().size(), Eq(1)); + EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(0), Eq(0.0)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().count(), Eq(1)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().max(), + DoubleNear(0, 1e-6)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().meanlogs(), Eq(0)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().mean(), Eq(0)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().min(), Eq(0)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().sum(), Eq(0)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().variance(), Eq(0)); +} + +TEST_F(ChromePerfDashboardMetricsExporterTest, + ExportMetricWithOnlyStatsExportsMeanValues) { + Metric metric{.name = "test_metric", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name", + .metric_metadata = DefaultMetadata(), + .time_series = Metric::TimeSeries{.samples = {}}, + .stats = Metric::Stats{ + .mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}}; + + ChromePerfDashboardMetricsExporter exporter(temp_filename_); + + ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric})); + proto::HistogramSet actual_histogram_set; + actual_histogram_set.ParseFromString(ReadFileAsString(temp_filename_)); + EXPECT_THAT(actual_histogram_set.histograms().size(), Eq(1)); + + // Validate values for `metric` + EXPECT_THAT(actual_histogram_set.histograms(0).sample_values().size(), Eq(1)); + EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(0), Eq(15.0)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().count(), Eq(1)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().max(), Eq(15)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().meanlogs(), + DoubleNear(2.70805, 0.1)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().mean(), Eq(15)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().min(), Eq(15)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().sum(), Eq(15)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().variance(), Eq(0)); +} + +TEST_F(ChromePerfDashboardMetricsExporterTest, + ExportMetricWithOnlyStatsConvertsMeanValuesWhenRequired) { + Metric metric{.name = "test_metric", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name", + .metric_metadata = DefaultMetadata(), + .time_series = Metric::TimeSeries{.samples = {}}, + .stats = Metric::Stats{ + .mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}}; + + ChromePerfDashboardMetricsExporter exporter(temp_filename_); + + ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric})); + proto::HistogramSet actual_histogram_set; + actual_histogram_set.ParseFromString(ReadFileAsString(temp_filename_)); + EXPECT_THAT(actual_histogram_set.histograms().size(), Eq(1)); + + // Validate values for `metric` + EXPECT_THAT(actual_histogram_set.histograms(0).sample_values().size(), Eq(1)); + EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(0), Eq(1875.0)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().count(), Eq(1)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().max(), Eq(1875)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().meanlogs(), + DoubleNear(7.53636, 0.1)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().mean(), Eq(1875)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().min(), Eq(1875)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().sum(), Eq(1875)); + EXPECT_THAT(actual_histogram_set.histograms(0).running().variance(), Eq(0)); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/global_metrics_logger_and_exporter.cc b/third_party/libwebrtc/api/test/metrics/global_metrics_logger_and_exporter.cc new file mode 100644 index 0000000000..2d42a976aa --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/global_metrics_logger_and_exporter.cc @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/global_metrics_logger_and_exporter.h" + +#include <memory> +#include <utility> +#include <vector> + +#include "api/test/metrics/metrics_exporter.h" +#include "api/test/metrics/metrics_logger.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace test { + +DefaultMetricsLogger* GetGlobalMetricsLogger() { + static DefaultMetricsLogger* logger_ = + new DefaultMetricsLogger(Clock::GetRealTimeClock()); + return logger_; +} + +bool ExportPerfMetric(MetricsLogger& logger, + std::vector<std::unique_ptr<MetricsExporter>> exporters) { + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + bool success = true; + for (auto& exporter : exporters) { + bool export_result = exporter->Export(metrics); + success = success && export_result; + } + return success; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/global_metrics_logger_and_exporter.h b/third_party/libwebrtc/api/test/metrics/global_metrics_logger_and_exporter.h new file mode 100644 index 0000000000..f77ff1c737 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/global_metrics_logger_and_exporter.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_GLOBAL_METRICS_LOGGER_AND_EXPORTER_H_ +#define API_TEST_METRICS_GLOBAL_METRICS_LOGGER_AND_EXPORTER_H_ + +#include <memory> +#include <vector> + +#include "api/test/metrics/metrics_exporter.h" +#include "api/test/metrics/metrics_logger.h" + +namespace webrtc { +namespace test { + +// Returns non-null global `MetricsLogger` to log metrics. +DefaultMetricsLogger* GetGlobalMetricsLogger(); + +bool ExportPerfMetric(MetricsLogger& logger, + std::vector<std::unique_ptr<MetricsExporter>> exporters); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_GLOBAL_METRICS_LOGGER_AND_EXPORTER_H_ diff --git a/third_party/libwebrtc/api/test/metrics/global_metrics_logger_and_exporter_test.cc b/third_party/libwebrtc/api/test/metrics/global_metrics_logger_and_exporter_test.cc new file mode 100644 index 0000000000..567b3da9e3 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/global_metrics_logger_and_exporter_test.cc @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/global_metrics_logger_and_exporter.h" + +#include <map> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_exporter.h" +#include "api/test/metrics/metrics_logger.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::Eq; +using ::testing::IsEmpty; + +std::map<std::string, std::string> DefaultMetadata() { + return std::map<std::string, std::string>{{"key", "value"}}; +} + +struct TestMetricsExporterFactory { + public: + std::unique_ptr<MetricsExporter> CreateExporter() { + return std::make_unique<TestMetricsExporter>(this, /*export_result=*/true); + } + + std::unique_ptr<MetricsExporter> CreateFailureExporter() { + return std::make_unique<TestMetricsExporter>(this, /*export_result=*/false); + } + + std::vector<Metric> exported_metrics; + + private: + class TestMetricsExporter : public MetricsExporter { + public: + TestMetricsExporter(TestMetricsExporterFactory* factory, bool export_result) + : factory_(factory), export_result_(export_result) {} + ~TestMetricsExporter() override = default; + + bool Export(rtc::ArrayView<const Metric> metrics) override { + factory_->exported_metrics = + std::vector<Metric>(metrics.begin(), metrics.end()); + return export_result_; + } + + TestMetricsExporterFactory* factory_; + bool export_result_; + }; +}; + +TEST(ExportPerfMetricTest, CollectedMetricsAreExporter) { + TestMetricsExporterFactory exporter_factory; + + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + logger.LogSingleValueMetric( + "metric_name", "test_case_name", + /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + std::map<std::string, std::string>{{"key", "value"}}); + + std::vector<std::unique_ptr<MetricsExporter>> exporters; + exporters.push_back(exporter_factory.CreateExporter()); + ASSERT_TRUE(ExportPerfMetric(logger, std::move(exporters))); + + std::vector<Metric> metrics = exporter_factory.exported_metrics; + ASSERT_THAT(metrics.size(), Eq(1lu)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map<std::string, std::string>{{"key", "value"}})); + ASSERT_THAT(metric.time_series.samples.size(), Eq(1lu)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{})); + ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::nullopt); + ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0)); +} + +TEST(ExportPerfMetricTest, OneFailedExporterDoesNotPreventExportToOthers) { + TestMetricsExporterFactory exporter_factory1; + TestMetricsExporterFactory exporter_factory2; + TestMetricsExporterFactory exporter_factory3; + + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + logger.LogSingleValueMetric("metric_name", "test_case_name", + /*value=*/10, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + + std::vector<std::unique_ptr<MetricsExporter>> exporters; + exporters.push_back(exporter_factory1.CreateExporter()); + exporters.push_back(exporter_factory2.CreateFailureExporter()); + exporters.push_back(exporter_factory3.CreateExporter()); + ASSERT_FALSE(ExportPerfMetric(logger, std::move(exporters))); + + std::vector<Metric> metrics1 = exporter_factory1.exported_metrics; + std::vector<Metric> metrics2 = exporter_factory2.exported_metrics; + std::vector<Metric> metrics3 = exporter_factory3.exported_metrics; + ASSERT_THAT(metrics1.size(), Eq(1lu)) + << metrics1[0].name << "; " << metrics1[1].name; + EXPECT_THAT(metrics1[0].name, Eq("metric_name")); + ASSERT_THAT(metrics2.size(), Eq(1lu)); + EXPECT_THAT(metrics2[0].name, Eq("metric_name")); + ASSERT_THAT(metrics3.size(), Eq(1lu)); + EXPECT_THAT(metrics3[0].name, Eq("metric_name")); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/metric.cc b/third_party/libwebrtc/api/test/metrics/metric.cc new file mode 100644 index 0000000000..3c30f36f49 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metric.cc @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metric.h" + +#include <string> + +namespace webrtc { +namespace test { + +absl::string_view ToString(Unit unit) { + switch (unit) { + case Unit::kMilliseconds: + return "Milliseconds"; + case Unit::kPercent: + return "Percent"; + case Unit::kBytes: + return "Bytes"; + case Unit::kKilobitsPerSecond: + return "KilobitsPerSecond"; + case Unit::kHertz: + return "Hertz"; + case Unit::kUnitless: + return "Unitless"; + case Unit::kCount: + return "Count"; + } +} + +absl::string_view ToString(ImprovementDirection direction) { + switch (direction) { + case ImprovementDirection::kBiggerIsBetter: + return "BiggerIsBetter"; + case ImprovementDirection::kNeitherIsBetter: + return "NeitherIsBetter"; + case ImprovementDirection::kSmallerIsBetter: + return "SmallerIsBetter"; + } +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/metric.h b/third_party/libwebrtc/api/test/metrics/metric.h new file mode 100644 index 0000000000..17c1755f95 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metric.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_METRIC_H_ +#define API_TEST_METRICS_METRIC_H_ + +#include <map> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/units/timestamp.h" + +namespace webrtc { +namespace test { + +enum class Unit { + kMilliseconds, + kPercent, + kBytes, + kKilobitsPerSecond, + kHertz, + // General unitless value. Can be used either for dimensionless quantities + // (ex ratio) or for units not presented in this enum and too specific to add + // to this enum. + kUnitless, + kCount +}; + +absl::string_view ToString(Unit unit); + +enum class ImprovementDirection { + kBiggerIsBetter, + kNeitherIsBetter, + kSmallerIsBetter +}; + +absl::string_view ToString(ImprovementDirection direction); + +struct Metric { + struct TimeSeries { + struct Sample { + // Timestamp in microseconds associated with a sample. For example, + // the timestamp when the sample was collected. + webrtc::Timestamp timestamp; + double value; + // Metadata associated with this particular sample. + std::map<std::string, std::string> sample_metadata; + }; + + // All samples collected for this metric. It can be empty if the Metric + // object only contains `stats`. + std::vector<Sample> samples; + }; + + // Contains metric's precomputed statistics based on the `time_series` or if + // `time_series` is omitted (has 0 samples) contains precomputed statistics + // provided by the metric's calculator. + struct Stats { + // Sample mean of the metric + // (https://en.wikipedia.org/wiki/Sample_mean_and_covariance). + absl::optional<double> mean; + // Standard deviation (https://en.wikipedia.org/wiki/Standard_deviation). + // Is undefined if `time_series` contains only a single value. + absl::optional<double> stddev; + absl::optional<double> min; + absl::optional<double> max; + }; + + // Metric name, for example PSNR, SSIM, decode_time, etc. + std::string name; + Unit unit; + ImprovementDirection improvement_direction; + // If the metric is generated by a test, this field can be used to specify + // this information. + std::string test_case; + // Metadata associated with the whole metric. + std::map<std::string, std::string> metric_metadata; + // Contains all samples of the metric collected during test execution. + // It can be empty if the user only stores precomputed statistics into + // `stats`. + TimeSeries time_series; + Stats stats; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_METRIC_H_ diff --git a/third_party/libwebrtc/api/test/metrics/metrics_accumulator.cc b/third_party/libwebrtc/api/test/metrics/metrics_accumulator.cc new file mode 100644 index 0000000000..c34396be97 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_accumulator.cc @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_accumulator.h" + +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace test { +namespace { + +Metric::Stats ToStats(const SamplesStatsCounter& values) { + if (values.IsEmpty()) { + return Metric::Stats(); + } + return Metric::Stats{.mean = values.GetAverage(), + .stddev = values.GetStandardDeviation(), + .min = values.GetMin(), + .max = values.GetMax()}; +} + +Metric SetTimeseries(const Metric& prototype, + const SamplesStatsCounter& counter) { + Metric output(prototype); + Metric::TimeSeries time_series; + for (const SamplesStatsCounter::StatsSample& sample : + counter.GetTimedSamples()) { + time_series.samples.push_back( + Metric::TimeSeries::Sample{.timestamp = sample.time, + .value = sample.value, + .sample_metadata = sample.metadata}); + } + output.time_series = std::move(time_series); + output.stats = ToStats(counter); + return output; +} + +} // namespace + +bool operator<(const MetricsAccumulator::MetricKey& a, + const MetricsAccumulator::MetricKey& b) { + if (a.test_case_name < b.test_case_name) { + return true; + } else if (a.test_case_name > b.test_case_name) { + return false; + } else { + return a.metric_name < b.metric_name; + } +} + +bool MetricsAccumulator::AddSample( + absl::string_view metric_name, + absl::string_view test_case_name, + double value, + Timestamp timestamp, + std::map<std::string, std::string> point_metadata) { + MutexLock lock(&mutex_); + bool created; + MetricValue* metric_value = + GetOrCreateMetric(metric_name, test_case_name, &created); + metric_value->counter.AddSample( + SamplesStatsCounter::StatsSample{.value = value, + .time = timestamp, + .metadata = std::move(point_metadata)}); + return created; +} + +bool MetricsAccumulator::AddMetricMetadata( + absl::string_view metric_name, + absl::string_view test_case_name, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metric_metadata) { + MutexLock lock(&mutex_); + bool created; + MetricValue* metric_value = + GetOrCreateMetric(metric_name, test_case_name, &created); + metric_value->metric.unit = unit; + metric_value->metric.improvement_direction = improvement_direction; + metric_value->metric.metric_metadata = std::move(metric_metadata); + return created; +} + +std::vector<Metric> MetricsAccumulator::GetCollectedMetrics() const { + MutexLock lock(&mutex_); + std::vector<Metric> out; + out.reserve(metrics_.size()); + for (const auto& [unused_key, metric_value] : metrics_) { + out.push_back(SetTimeseries(metric_value.metric, metric_value.counter)); + } + return out; +} + +MetricsAccumulator::MetricValue* MetricsAccumulator::GetOrCreateMetric( + absl::string_view metric_name, + absl::string_view test_case_name, + bool* created) { + MetricKey key(metric_name, test_case_name); + auto it = metrics_.find(key); + if (it != metrics_.end()) { + *created = false; + return &it->second; + } + *created = true; + + Metric metric{ + .name = key.metric_name, + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .test_case = key.test_case_name, + }; + return &metrics_.emplace(key, MetricValue{.metric = std::move(metric)}) + .first->second; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/metrics_accumulator.h b/third_party/libwebrtc/api/test/metrics/metrics_accumulator.h new file mode 100644 index 0000000000..c75bd9429c --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_accumulator.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_METRICS_ACCUMULATOR_H_ +#define API_TEST_METRICS_METRICS_ACCUMULATOR_H_ + +#include <map> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { +namespace test { + +// Accumulates metrics' samples internally and provides API to get collected +// ones. +// +// This object is thread safe. +class MetricsAccumulator { + public: + MetricsAccumulator() = default; + + // Adds sample for the specified `metric_name` within specified + // `test_case_name`. If it is the first time when this combination of + // `metric_name` and `test_case_name` is used, creates a new Metric to collect + // samples, otherwise adds a sample to the previously created Metric. + // + // By default metric will use `Unit::kUnitless` and + // `ImprovementDirection::kNeitherIsBetter`. + // + // `point_metadata` - the metadata to be added to the single data point that + // this method adds to the Metric (it is not a metric global metadata). + // + // Returns true if a new metric was created and false otherwise. + bool AddSample(absl::string_view metric_name, + absl::string_view test_case_name, + double value, + Timestamp timestamp, + std::map<std::string, std::string> point_metadata = {}); + + // Adds metadata to the metric specified by `metric_name` within specified + // `test_case_name`. If such a metric doesn't exist, creates a new one, + // otherwise overrides previously recorded values. + // + // Returns true if a new metric was created and false otherwise. + bool AddMetricMetadata( + absl::string_view metric_name, + absl::string_view test_case_name, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metric_metadata = {}); + + // Returns all metrics collected by this accumulator. No order guarantees + // provided. + std::vector<Metric> GetCollectedMetrics() const; + + private: + struct MetricKey { + MetricKey(absl::string_view metric_name, absl::string_view test_case_name) + : metric_name(metric_name), test_case_name(test_case_name) {} + + std::string metric_name; + std::string test_case_name; + }; + friend bool operator<(const MetricKey& a, const MetricKey& b); + + struct MetricValue { + SamplesStatsCounter counter; + Metric metric; + }; + + // Gets existing metrics or creates a new one. If metric was created `created` + // will be set to true. + MetricValue* GetOrCreateMetric(absl::string_view metric_name, + absl::string_view test_case_name, + bool* created) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + mutable Mutex mutex_; + std::map<MetricKey, MetricValue> metrics_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_METRICS_ACCUMULATOR_H_ diff --git a/third_party/libwebrtc/api/test/metrics/metrics_accumulator_test.cc b/third_party/libwebrtc/api/test/metrics/metrics_accumulator_test.cc new file mode 100644 index 0000000000..677f523339 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_accumulator_test.cc @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_accumulator.h" + +#include <map> +#include <vector> + +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::SizeIs; + +TEST(MetricsAccumulatorTest, AddSampleToTheNewMetricWillCreateOne) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/std::map<std::string, std::string>{{"key", "value"}})); + + std::vector<Metric> metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metric.metric_metadata, IsEmpty()); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{{"key", "value"}})); + ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0)); + ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0)); +} + +TEST(MetricsAccumulatorTest, AddSamplesToExistingMetricWontCreateNewOne) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map<std::string, std::string>{{"key1", "value1"}})); + ASSERT_FALSE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/20, Timestamp::Seconds(2), + /*point_metadata=*/ + std::map<std::string, std::string>{{"key2", "value2"}})); + + std::vector<Metric> metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metric.metric_metadata, IsEmpty()); + ASSERT_THAT(metric.time_series.samples, SizeIs(2)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{{"key1", "value1"}})); + EXPECT_THAT(metric.time_series.samples[1].value, Eq(20.0)); + EXPECT_THAT(metric.time_series.samples[1].timestamp, + Eq(Timestamp::Seconds(2))); + EXPECT_THAT(metric.time_series.samples[1].sample_metadata, + Eq(std::map<std::string, std::string>{{"key2", "value2"}})); + ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0)); + ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0)); +} + +TEST(MetricsAccumulatorTest, AddSampleToDifferentMetricsWillCreateBoth) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddSample( + "metric_name1", "test_case_name1", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map<std::string, std::string>{{"key1", "value1"}})); + ASSERT_TRUE(accumulator.AddSample( + "metric_name2", "test_case_name2", + /*value=*/20, Timestamp::Seconds(2), + /*point_metadata=*/ + std::map<std::string, std::string>{{"key2", "value2"}})); + + std::vector<Metric> metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(2)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[0].unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metrics[0].improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metrics[0].metric_metadata, IsEmpty()); + ASSERT_THAT(metrics[0].time_series.samples, SizeIs(1)); + EXPECT_THAT(metrics[0].time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metrics[0].time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metrics[0].time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{{"key1", "value1"}})); + ASSERT_THAT(metrics[0].stats.mean, absl::optional<double>(10.0)); + ASSERT_THAT(metrics[0].stats.stddev, absl::optional<double>(0.0)); + ASSERT_THAT(metrics[0].stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metrics[0].stats.max, absl::optional<double>(10.0)); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); + EXPECT_THAT(metrics[1].unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metrics[1].improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metrics[1].metric_metadata, IsEmpty()); + ASSERT_THAT(metrics[1].time_series.samples, SizeIs(1)); + EXPECT_THAT(metrics[1].time_series.samples[0].value, Eq(20.0)); + EXPECT_THAT(metrics[1].time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(2))); + EXPECT_THAT(metrics[1].time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{{"key2", "value2"}})); + ASSERT_THAT(metrics[1].stats.mean, absl::optional<double>(20.0)); + ASSERT_THAT(metrics[1].stats.stddev, absl::optional<double>(0.0)); + ASSERT_THAT(metrics[1].stats.min, absl::optional<double>(20.0)); + ASSERT_THAT(metrics[1].stats.max, absl::optional<double>(20.0)); +} + +TEST(MetricsAccumulatorTest, AddMetadataToTheNewMetricWillCreateOne) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map<std::string, std::string>{{"key", "value"}})); + + std::vector<Metric> metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map<std::string, std::string>{{"key", "value"}})); + ASSERT_THAT(metric.time_series.samples, IsEmpty()); + ASSERT_THAT(metric.stats.mean, absl::nullopt); + ASSERT_THAT(metric.stats.stddev, absl::nullopt); + ASSERT_THAT(metric.stats.min, absl::nullopt); + ASSERT_THAT(metric.stats.max, absl::nullopt); +} + +TEST(MetricsAccumulatorTest, + AddMetadataToTheExistingMetricWillOverwriteValues) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map<std::string, std::string>{{"key1", "value1"}})); + + ASSERT_FALSE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kBytes, + ImprovementDirection::kSmallerIsBetter, + /*metric_metadata=*/ + std::map<std::string, std::string>{{"key2", "value2"}})); + + std::vector<Metric> metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kBytes)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kSmallerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map<std::string, std::string>{{"key2", "value2"}})); + ASSERT_THAT(metric.time_series.samples, IsEmpty()); + ASSERT_THAT(metric.stats.mean, absl::nullopt); + ASSERT_THAT(metric.stats.stddev, absl::nullopt); + ASSERT_THAT(metric.stats.min, absl::nullopt); + ASSERT_THAT(metric.stats.max, absl::nullopt); +} + +TEST(MetricsAccumulatorTest, AddMetadataToDifferentMetricsWillCreateBoth) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name1", "test_case_name1", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map<std::string, std::string>{{"key1", "value1"}})); + + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name2", "test_case_name2", Unit::kBytes, + ImprovementDirection::kSmallerIsBetter, + /*metric_metadata=*/ + std::map<std::string, std::string>{{"key2", "value2"}})); + + std::vector<Metric> metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(2)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[0].unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metrics[0].improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metrics[0].metric_metadata, + Eq(std::map<std::string, std::string>{{"key1", "value1"}})); + ASSERT_THAT(metrics[0].time_series.samples, IsEmpty()); + ASSERT_THAT(metrics[0].stats.mean, absl::nullopt); + ASSERT_THAT(metrics[0].stats.stddev, absl::nullopt); + ASSERT_THAT(metrics[0].stats.min, absl::nullopt); + ASSERT_THAT(metrics[0].stats.max, absl::nullopt); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); + EXPECT_THAT(metrics[1].unit, Eq(Unit::kBytes)); + EXPECT_THAT(metrics[1].improvement_direction, + Eq(ImprovementDirection::kSmallerIsBetter)); + EXPECT_THAT(metrics[1].metric_metadata, + Eq(std::map<std::string, std::string>{{"key2", "value2"}})); + ASSERT_THAT(metrics[1].time_series.samples, IsEmpty()); + ASSERT_THAT(metrics[1].stats.mean, absl::nullopt); + ASSERT_THAT(metrics[1].stats.stddev, absl::nullopt); + ASSERT_THAT(metrics[1].stats.min, absl::nullopt); + ASSERT_THAT(metrics[1].stats.max, absl::nullopt); +} + +TEST(MetricsAccumulatorTest, AddMetadataAfterAddingSampleWontCreateNewMetric) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map<std::string, std::string>{{"key_s", "value_s"}})); + ASSERT_FALSE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map<std::string, std::string>{{"key_m", "value_m"}})); + + std::vector<Metric> metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map<std::string, std::string>{{"key_m", "value_m"}})); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{{"key_s", "value_s"}})); + ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0)); + ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0)); +} + +TEST(MetricsAccumulatorTest, AddSampleAfterAddingMetadataWontCreateNewMetric) { + MetricsAccumulator accumulator; + ASSERT_TRUE(accumulator.AddMetricMetadata( + "metric_name", "test_case_name", Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + /*metric_metadata=*/ + std::map<std::string, std::string>{{"key_m", "value_m"}})); + ASSERT_FALSE(accumulator.AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map<std::string, std::string>{{"key_s", "value_s"}})); + + std::vector<Metric> metrics = accumulator.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map<std::string, std::string>{{"key_m", "value_m"}})); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{{"key_s", "value_s"}})); + ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0)); + ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0)); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/metrics_exporter.h b/third_party/libwebrtc/api/test/metrics/metrics_exporter.h new file mode 100644 index 0000000000..23954b6b1f --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_exporter.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_METRICS_EXPORTER_H_ +#define API_TEST_METRICS_METRICS_EXPORTER_H_ + +#include "api/array_view.h" +#include "api/test/metrics/metric.h" + +namespace webrtc { +namespace test { + +// Exports metrics in the requested format. +class MetricsExporter { + public: + virtual ~MetricsExporter() = default; + + // Exports specified metrics in a format that depends on the implementation. + // Returns true if export succeeded, false otherwise. + virtual bool Export(rtc::ArrayView<const Metric> metrics) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_METRICS_EXPORTER_H_ diff --git a/third_party/libwebrtc/api/test/metrics/metrics_logger.cc b/third_party/libwebrtc/api/test/metrics/metrics_logger.cc new file mode 100644 index 0000000000..1e24400367 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_logger.cc @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_logger.h" + +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace test { +namespace { + +Metric::Stats ToStats(const SamplesStatsCounter& values) { + if (values.IsEmpty()) { + return Metric::Stats(); + } + return Metric::Stats{.mean = values.GetAverage(), + .stddev = values.GetStandardDeviation(), + .min = values.GetMin(), + .max = values.GetMax()}; +} + +} // namespace + +void DefaultMetricsLogger::LogSingleValueMetric( + absl::string_view name, + absl::string_view test_case_name, + double value, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metadata) { + MutexLock lock(&mutex_); + metrics_.push_back(Metric{ + .name = std::string(name), + .unit = unit, + .improvement_direction = improvement_direction, + .test_case = std::string(test_case_name), + .metric_metadata = std::move(metadata), + .time_series = + Metric::TimeSeries{.samples = std::vector{Metric::TimeSeries::Sample{ + .timestamp = Now(), .value = value}}}, + .stats = Metric::Stats{ + .mean = value, .stddev = absl::nullopt, .min = value, .max = value}}); +} + +void DefaultMetricsLogger::LogMetric( + absl::string_view name, + absl::string_view test_case_name, + const SamplesStatsCounter& values, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metadata) { + MutexLock lock(&mutex_); + Metric::TimeSeries time_series; + for (const SamplesStatsCounter::StatsSample& sample : + values.GetTimedSamples()) { + time_series.samples.push_back( + Metric::TimeSeries::Sample{.timestamp = sample.time, + .value = sample.value, + .sample_metadata = sample.metadata}); + } + + metrics_.push_back(Metric{.name = std::string(name), + .unit = unit, + .improvement_direction = improvement_direction, + .test_case = std::string(test_case_name), + .metric_metadata = std::move(metadata), + .time_series = std::move(time_series), + .stats = ToStats(values)}); +} + +void DefaultMetricsLogger::LogMetric( + absl::string_view name, + absl::string_view test_case_name, + const Metric::Stats& metric_stats, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metadata) { + MutexLock lock(&mutex_); + metrics_.push_back(Metric{.name = std::string(name), + .unit = unit, + .improvement_direction = improvement_direction, + .test_case = std::string(test_case_name), + .metric_metadata = std::move(metadata), + .time_series = Metric::TimeSeries{.samples = {}}, + .stats = std::move(metric_stats)}); +} + +std::vector<Metric> DefaultMetricsLogger::GetCollectedMetrics() const { + std::vector<Metric> out = metrics_accumulator_.GetCollectedMetrics(); + MutexLock lock(&mutex_); + out.insert(out.end(), metrics_.begin(), metrics_.end()); + return out; +} + +Timestamp DefaultMetricsLogger::Now() { + return clock_->CurrentTime(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/metrics_logger.h b/third_party/libwebrtc/api/test/metrics/metrics_logger.h new file mode 100644 index 0000000000..66f9e55b95 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_logger.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_METRICS_LOGGER_H_ +#define API_TEST_METRICS_METRICS_LOGGER_H_ + +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_accumulator.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace test { + +// Provides API to log and collect performance metrics. +class MetricsLogger { + public: + virtual ~MetricsLogger() = default; + + // Adds a metric with a single value. + // `metadata` - metric's level metadata to add. + virtual void LogSingleValueMetric( + absl::string_view name, + absl::string_view test_case_name, + double value, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metadata = {}) = 0; + + // Adds metrics with a time series created based on the provided `values`. + // `metadata` - metric's level metadata to add. + virtual void LogMetric(absl::string_view name, + absl::string_view test_case_name, + const SamplesStatsCounter& values, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metadata = {}) = 0; + + // Adds metric with a time series with only stats object and without actual + // collected values. + // `metadata` - metric's level metadata to add. + virtual void LogMetric(absl::string_view name, + absl::string_view test_case_name, + const Metric::Stats& metric_stats, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metadata = {}) = 0; + + // Returns all metrics collected by this logger. + virtual std::vector<Metric> GetCollectedMetrics() const = 0; +}; + +class DefaultMetricsLogger : public MetricsLogger { + public: + explicit DefaultMetricsLogger(webrtc::Clock* clock) : clock_(clock) {} + ~DefaultMetricsLogger() override = default; + + void LogSingleValueMetric( + absl::string_view name, + absl::string_view test_case_name, + double value, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metadata = {}) override; + + void LogMetric(absl::string_view name, + absl::string_view test_case_name, + const SamplesStatsCounter& values, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metadata = {}) override; + + void LogMetric(absl::string_view name, + absl::string_view test_case_name, + const Metric::Stats& metric_stats, + Unit unit, + ImprovementDirection improvement_direction, + std::map<std::string, std::string> metadata = {}) override; + + // Returns all metrics collected by this logger and its `MetricsAccumulator`. + std::vector<Metric> GetCollectedMetrics() const override; + + MetricsAccumulator* GetMetricsAccumulator() { return &metrics_accumulator_; } + + private: + webrtc::Timestamp Now(); + + webrtc::Clock* const clock_; + MetricsAccumulator metrics_accumulator_; + + mutable Mutex mutex_; + std::vector<Metric> metrics_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_METRICS_LOGGER_H_ diff --git a/third_party/libwebrtc/api/test/metrics/metrics_logger_test.cc b/third_party/libwebrtc/api/test/metrics/metrics_logger_test.cc new file mode 100644 index 0000000000..de4501ca36 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_logger_test.cc @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_logger.h" + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::SizeIs; + +std::map<std::string, std::string> DefaultMetadata() { + return std::map<std::string, std::string>{{"key", "value"}}; +} + +TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMetric) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + logger.LogSingleValueMetric( + "metric_name", "test_case_name", + /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + std::map<std::string, std::string>{{"key", "value"}}); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map<std::string, std::string>{{"key", "value"}})); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{})); + ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::nullopt); + ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0)); +} + +TEST(DefaultMetricsLoggerTest, LogMetricWithSamplesStatsCounterRecordsMetric) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + + SamplesStatsCounter values; + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 10, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = + std::map<std::string, std::string>{{"point_key1", "value1"}}}); + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 20, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = + std::map<std::string, std::string>{{"point_key2", "value2"}}}); + logger.LogMetric("metric_name", "test_case_name", values, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + std::map<std::string, std::string>{{"key", "value"}}); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map<std::string, std::string>{{"key", "value"}})); + ASSERT_THAT(metric.time_series.samples, SizeIs(2)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{{"point_key1", "value1"}})); + EXPECT_THAT(metric.time_series.samples[1].value, Eq(20.0)); + EXPECT_THAT(metric.time_series.samples[1].sample_metadata, + Eq(std::map<std::string, std::string>{{"point_key2", "value2"}})); + ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0)); + ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0)); +} + +TEST(DefaultMetricsLoggerTest, + LogMetricWithEmptySamplesStatsCounterRecordsEmptyMetric) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + SamplesStatsCounter values; + logger.LogMetric("metric_name", "test_case_name", values, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, DefaultMetadata()); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + EXPECT_THAT(metrics[0].name, Eq("metric_name")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name")); + EXPECT_THAT(metrics[0].time_series.samples, IsEmpty()); + ASSERT_THAT(metrics[0].stats.mean, Eq(absl::nullopt)); + ASSERT_THAT(metrics[0].stats.stddev, Eq(absl::nullopt)); + ASSERT_THAT(metrics[0].stats.min, Eq(absl::nullopt)); + ASSERT_THAT(metrics[0].stats.max, Eq(absl::nullopt)); +} + +TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMetric) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20}; + logger.LogMetric("metric_name", "test_case_name", metric_stats, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + std::map<std::string, std::string>{{"key", "value"}}); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metric.metric_metadata, + Eq(std::map<std::string, std::string>{{"key", "value"}})); + ASSERT_THAT(metric.time_series.samples, IsEmpty()); + ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0)); + ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0)); +} + +TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMultipleMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + + logger.LogSingleValueMetric("metric_name1", "test_case_name1", + /*value=*/10, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogSingleValueMetric("metric_name2", "test_case_name2", + /*value=*/10, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(2)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); +} + +TEST(DefaultMetricsLoggerTest, + LogMetricWithSamplesStatsCounterRecordsMultipleMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + SamplesStatsCounter values; + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 10, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = DefaultMetadata()}); + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 20, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = DefaultMetadata()}); + + logger.LogMetric("metric_name1", "test_case_name1", values, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogMetric("metric_name2", "test_case_name2", values, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(2)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); +} + +TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMultipleMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20}; + + logger.LogMetric("metric_name1", "test_case_name1", metric_stats, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogMetric("metric_name2", "test_case_name2", metric_stats, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(2)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); +} + +TEST(DefaultMetricsLoggerTest, + LogMetricThroughtAllMethodsAccumulateAllMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + SamplesStatsCounter values; + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 10, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = DefaultMetadata()}); + values.AddSample(SamplesStatsCounter::StatsSample{ + .value = 20, + .time = Clock::GetRealTimeClock()->CurrentTime(), + .metadata = DefaultMetadata()}); + Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20}; + + logger.LogSingleValueMetric("metric_name1", "test_case_name1", + /*value=*/10, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogMetric("metric_name2", "test_case_name2", values, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + logger.LogMetric("metric_name3", "test_case_name3", metric_stats, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + DefaultMetadata()); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics.size(), Eq(3lu)); + EXPECT_THAT(metrics[0].name, Eq("metric_name1")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].name, Eq("metric_name2")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); + EXPECT_THAT(metrics[2].name, Eq("metric_name3")); + EXPECT_THAT(metrics[2].test_case, Eq("test_case_name3")); +} + +TEST(DefaultMetricsLoggerTest, AccumulatedMetricsReturnedInCollectedMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + logger.GetMetricsAccumulator()->AddSample( + "metric_name", "test_case_name", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/std::map<std::string, std::string>{{"key", "value"}}); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(1)); + const Metric& metric = metrics[0]; + EXPECT_THAT(metric.name, Eq("metric_name")); + EXPECT_THAT(metric.test_case, Eq("test_case_name")); + EXPECT_THAT(metric.unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metric.improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metric.metric_metadata, IsEmpty()); + ASSERT_THAT(metric.time_series.samples, SizeIs(1)); + EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metric.time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metric.time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{{"key", "value"}})); + ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0)); + ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0)); +} + +TEST(DefaultMetricsLoggerTest, + AccumulatedMetricsReturnedTogetherWithLoggedMetrics) { + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + logger.LogSingleValueMetric( + "metric_name1", "test_case_name1", + /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, + std::map<std::string, std::string>{{"key_m", "value_m"}}); + logger.GetMetricsAccumulator()->AddSample( + "metric_name2", "test_case_name2", + /*value=*/10, Timestamp::Seconds(1), + /*point_metadata=*/ + std::map<std::string, std::string>{{"key_s", "value_s"}}); + + std::vector<Metric> metrics = logger.GetCollectedMetrics(); + ASSERT_THAT(metrics, SizeIs(2)); + EXPECT_THAT(metrics[0].name, Eq("metric_name2")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name2")); + EXPECT_THAT(metrics[0].unit, Eq(Unit::kUnitless)); + EXPECT_THAT(metrics[0].improvement_direction, + Eq(ImprovementDirection::kNeitherIsBetter)); + EXPECT_THAT(metrics[0].metric_metadata, IsEmpty()); + ASSERT_THAT(metrics[0].time_series.samples, SizeIs(1)); + EXPECT_THAT(metrics[0].time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metrics[0].time_series.samples[0].timestamp, + Eq(Timestamp::Seconds(1))); + EXPECT_THAT(metrics[0].time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{{"key_s", "value_s"}})); + ASSERT_THAT(metrics[0].stats.mean, absl::optional<double>(10.0)); + ASSERT_THAT(metrics[0].stats.stddev, absl::optional<double>(0.0)); + ASSERT_THAT(metrics[0].stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metrics[0].stats.max, absl::optional<double>(10.0)); + EXPECT_THAT(metrics[1].name, Eq("metric_name1")); + EXPECT_THAT(metrics[1].test_case, Eq("test_case_name1")); + EXPECT_THAT(metrics[1].unit, Eq(Unit::kMilliseconds)); + EXPECT_THAT(metrics[1].improvement_direction, + Eq(ImprovementDirection::kBiggerIsBetter)); + EXPECT_THAT(metrics[1].metric_metadata, + Eq(std::map<std::string, std::string>{{"key_m", "value_m"}})); + ASSERT_THAT(metrics[1].time_series.samples, SizeIs(1)); + EXPECT_THAT(metrics[1].time_series.samples[0].value, Eq(10.0)); + EXPECT_THAT(metrics[1].time_series.samples[0].sample_metadata, + Eq(std::map<std::string, std::string>{})); + ASSERT_THAT(metrics[1].stats.mean, absl::optional<double>(10.0)); + ASSERT_THAT(metrics[1].stats.stddev, absl::nullopt); + ASSERT_THAT(metrics[1].stats.min, absl::optional<double>(10.0)); + ASSERT_THAT(metrics[1].stats.max, absl::optional<double>(10.0)); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/metrics_set_proto_file_exporter.cc b/third_party/libwebrtc/api/test/metrics/metrics_set_proto_file_exporter.cc new file mode 100644 index 0000000000..86e6f2e136 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_set_proto_file_exporter.cc @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_set_proto_file_exporter.h" + +#include <stdio.h> + +#include <string> + +#include "api/test/metrics/metric.h" +#include "rtc_base/logging.h" +#include "test/testsupport/file_utils.h" + +#if WEBRTC_ENABLE_PROTOBUF +#include "api/test/metrics/proto/metric.pb.h" +#endif + +namespace webrtc { +namespace test { +namespace { + +#if WEBRTC_ENABLE_PROTOBUF +webrtc::test_metrics::Unit ToProtoUnit(Unit unit) { + switch (unit) { + case Unit::kMilliseconds: + return webrtc::test_metrics::Unit::MILLISECONDS; + case Unit::kPercent: + return webrtc::test_metrics::Unit::PERCENT; + case Unit::kBytes: + return webrtc::test_metrics::Unit::BYTES; + case Unit::kKilobitsPerSecond: + return webrtc::test_metrics::Unit::KILOBITS_PER_SECOND; + case Unit::kHertz: + return webrtc::test_metrics::Unit::HERTZ; + case Unit::kUnitless: + return webrtc::test_metrics::Unit::UNITLESS; + case Unit::kCount: + return webrtc::test_metrics::Unit::COUNT; + } +} + +webrtc::test_metrics::ImprovementDirection ToProtoImprovementDirection( + ImprovementDirection direction) { + switch (direction) { + case ImprovementDirection::kBiggerIsBetter: + return webrtc::test_metrics::ImprovementDirection::BIGGER_IS_BETTER; + case ImprovementDirection::kNeitherIsBetter: + return webrtc::test_metrics::ImprovementDirection::NEITHER_IS_BETTER; + case ImprovementDirection::kSmallerIsBetter: + return webrtc::test_metrics::ImprovementDirection::SMALLER_IS_BETTER; + } +} + +void SetTimeSeries( + const Metric::TimeSeries& time_series, + webrtc::test_metrics::Metric::TimeSeries* proto_time_series) { + for (const Metric::TimeSeries::Sample& sample : time_series.samples) { + webrtc::test_metrics::Metric::TimeSeries::Sample* proto_sample = + proto_time_series->add_samples(); + proto_sample->set_value(sample.value); + proto_sample->set_timestamp_us(sample.timestamp.us()); + for (const auto& [key, value] : sample.sample_metadata) { + proto_sample->mutable_sample_metadata()->insert({key, value}); + } + } +} + +void SetStats(const Metric::Stats& stats, + webrtc::test_metrics::Metric::Stats* proto_stats) { + if (stats.mean.has_value()) { + proto_stats->set_mean(*stats.mean); + } + if (stats.stddev.has_value()) { + proto_stats->set_stddev(*stats.stddev); + } + if (stats.min.has_value()) { + proto_stats->set_min(*stats.min); + } + if (stats.max.has_value()) { + proto_stats->set_max(*stats.max); + } +} + +bool WriteMetricsToFile(const std::string& path, + const webrtc::test_metrics::MetricsSet& metrics_set) { + std::string data; + bool ok = metrics_set.SerializeToString(&data); + if (!ok) { + RTC_LOG(LS_ERROR) << "Failed to serialize histogram set to string"; + return false; + } + + CreateDir(DirName(path)); + FILE* output = fopen(path.c_str(), "wb"); + if (output == NULL) { + RTC_LOG(LS_ERROR) << "Failed to write to " << path; + return false; + } + size_t written = fwrite(data.c_str(), sizeof(char), data.size(), output); + fclose(output); + + if (written != data.size()) { + size_t expected = data.size(); + RTC_LOG(LS_ERROR) << "Wrote " << written << ", tried to write " << expected; + return false; + } + return true; +} +#endif // WEBRTC_ENABLE_PROTOBUF + +} // namespace + +MetricsSetProtoFileExporter::Options::Options( + absl::string_view export_file_path) + : export_file_path(export_file_path) {} +MetricsSetProtoFileExporter::Options::Options( + absl::string_view export_file_path, + bool export_whole_time_series) + : export_file_path(export_file_path), + export_whole_time_series(export_whole_time_series) {} + +bool MetricsSetProtoFileExporter::Export(rtc::ArrayView<const Metric> metrics) { +#if WEBRTC_ENABLE_PROTOBUF + webrtc::test_metrics::MetricsSet metrics_set; + for (const Metric& metric : metrics) { + webrtc::test_metrics::Metric* metric_proto = metrics_set.add_metrics(); + metric_proto->set_name(metric.name); + metric_proto->set_unit(ToProtoUnit(metric.unit)); + metric_proto->set_improvement_direction( + ToProtoImprovementDirection(metric.improvement_direction)); + metric_proto->set_test_case(metric.test_case); + for (const auto& [key, value] : metric.metric_metadata) { + metric_proto->mutable_metric_metadata()->insert({key, value}); + } + + if (options_.export_whole_time_series) { + SetTimeSeries(metric.time_series, metric_proto->mutable_time_series()); + } + SetStats(metric.stats, metric_proto->mutable_stats()); + } + + return WriteMetricsToFile(options_.export_file_path, metrics_set); +#else + RTC_LOG(LS_ERROR) + << "Compile with protobuf support to properly use this class"; + return false; +#endif +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/metrics_set_proto_file_exporter.h b/third_party/libwebrtc/api/test/metrics/metrics_set_proto_file_exporter.h new file mode 100644 index 0000000000..f996e9e7b0 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_set_proto_file_exporter.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_ +#define API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_ + +#include <string> + +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_exporter.h" + +namespace webrtc { +namespace test { + +// Exports all collected metrics to the proto file using +// `webrtc::test_metrics::MetricsSet` format. +class MetricsSetProtoFileExporter : public MetricsExporter { + public: + struct Options { + explicit Options(absl::string_view export_file_path); + Options(absl::string_view export_file_path, bool export_whole_time_series); + + // File to export proto. + std::string export_file_path; + // If true will write all time series values to the output proto file, + // otherwise will write stats only. + bool export_whole_time_series = true; + }; + + explicit MetricsSetProtoFileExporter(const Options& options) + : options_(options) {} + + MetricsSetProtoFileExporter(const MetricsSetProtoFileExporter&) = delete; + MetricsSetProtoFileExporter& operator=(const MetricsSetProtoFileExporter&) = + delete; + + bool Export(rtc::ArrayView<const Metric> metrics) override; + + private: + const Options options_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_ diff --git a/third_party/libwebrtc/api/test/metrics/metrics_set_proto_file_exporter_test.cc b/third_party/libwebrtc/api/test/metrics/metrics_set_proto_file_exporter_test.cc new file mode 100644 index 0000000000..eb4d483068 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/metrics_set_proto_file_exporter_test.cc @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_set_proto_file_exporter.h" + +#include <fstream> +#include <map> +#include <string> +#include <vector> + +#include "api/test/metrics/metric.h" +#include "api/test/metrics/proto/metric.pb.h" +#include "api/units/timestamp.h" +#include "rtc_base/protobuf_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::Eq; +using ::testing::Test; + +namespace proto = ::webrtc::test_metrics; + +std::string ReadFileAsString(const std::string& filename) { + std::ifstream infile(filename, std::ios_base::binary); + auto buffer = std::vector<char>(std::istreambuf_iterator<char>(infile), + std::istreambuf_iterator<char>()); + return std::string(buffer.begin(), buffer.end()); +} + +std::map<std::string, std::string> DefaultMetadata() { + return std::map<std::string, std::string>{{"key", "value"}}; +} + +Metric::TimeSeries::Sample Sample(double value) { + return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1), + .value = value, + .sample_metadata = DefaultMetadata()}; +} + +void AssertSamplesEqual(const proto::Metric::TimeSeries::Sample& actual_sample, + const Metric::TimeSeries::Sample& expected_sample) { + EXPECT_THAT(actual_sample.value(), Eq(expected_sample.value)); + EXPECT_THAT(actual_sample.timestamp_us(), Eq(expected_sample.timestamp.us())); + EXPECT_THAT(actual_sample.sample_metadata().size(), + Eq(expected_sample.sample_metadata.size())); + for (const auto& [key, value] : expected_sample.sample_metadata) { + EXPECT_THAT(actual_sample.sample_metadata().at(key), Eq(value)); + } +} + +class MetricsSetProtoFileExporterTest : public Test { + protected: + ~MetricsSetProtoFileExporterTest() override = default; + + void SetUp() override { + temp_filename_ = webrtc::test::TempFilename( + webrtc::test::OutputPath(), "metrics_set_proto_file_exporter_test"); + } + + void TearDown() override { + ASSERT_TRUE(webrtc::test::RemoveFile(temp_filename_)); + } + + std::string temp_filename_; +}; + +TEST_F(MetricsSetProtoFileExporterTest, MetricsAreExportedCorrectly) { + MetricsSetProtoFileExporter::Options options(temp_filename_); + MetricsSetProtoFileExporter exporter(options); + + Metric metric1{ + .name = "test_metric1", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name1", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}}, + .stats = + Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}}; + Metric metric2{ + .name = "test_metric2", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .test_case = "test_case_name2", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}}, + .stats = Metric::Stats{ + .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}}; + + ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2})); + webrtc::test_metrics::MetricsSet actual_metrics_set; + actual_metrics_set.ParseFromString(ReadFileAsString(temp_filename_)); + EXPECT_THAT(actual_metrics_set.metrics().size(), Eq(2)); + + EXPECT_THAT(actual_metrics_set.metrics(0).name(), Eq("test_metric1")); + EXPECT_THAT(actual_metrics_set.metrics(0).test_case(), Eq("test_case_name1")); + EXPECT_THAT(actual_metrics_set.metrics(0).unit(), + Eq(proto::Unit::MILLISECONDS)); + EXPECT_THAT(actual_metrics_set.metrics(0).improvement_direction(), + Eq(proto::ImprovementDirection::BIGGER_IS_BETTER)); + EXPECT_THAT(actual_metrics_set.metrics(0).metric_metadata().size(), Eq(1lu)); + EXPECT_THAT(actual_metrics_set.metrics(0).metric_metadata().at("key"), + Eq("value")); + EXPECT_THAT(actual_metrics_set.metrics(0).time_series().samples().size(), + Eq(2)); + AssertSamplesEqual(actual_metrics_set.metrics(0).time_series().samples(0), + Sample(10.0)); + AssertSamplesEqual(actual_metrics_set.metrics(0).time_series().samples(1), + Sample(20.0)); + EXPECT_THAT(actual_metrics_set.metrics(0).stats().mean(), Eq(15.0)); + EXPECT_THAT(actual_metrics_set.metrics(0).stats().stddev(), Eq(5.0)); + EXPECT_THAT(actual_metrics_set.metrics(0).stats().min(), Eq(10.0)); + EXPECT_THAT(actual_metrics_set.metrics(0).stats().max(), Eq(20.0)); + + EXPECT_THAT(actual_metrics_set.metrics(1).name(), Eq("test_metric2")); + EXPECT_THAT(actual_metrics_set.metrics(1).test_case(), Eq("test_case_name2")); + EXPECT_THAT(actual_metrics_set.metrics(1).unit(), + Eq(proto::Unit::KILOBITS_PER_SECOND)); + EXPECT_THAT(actual_metrics_set.metrics(1).improvement_direction(), + Eq(proto::ImprovementDirection::SMALLER_IS_BETTER)); + EXPECT_THAT(actual_metrics_set.metrics(1).metric_metadata().size(), Eq(1lu)); + EXPECT_THAT(actual_metrics_set.metrics(1).metric_metadata().at("key"), + Eq("value")); + EXPECT_THAT(actual_metrics_set.metrics(1).time_series().samples().size(), + Eq(2)); + AssertSamplesEqual(actual_metrics_set.metrics(1).time_series().samples(0), + Sample(20.0)); + AssertSamplesEqual(actual_metrics_set.metrics(1).time_series().samples(1), + Sample(40.0)); + EXPECT_THAT(actual_metrics_set.metrics(1).stats().mean(), Eq(30.0)); + EXPECT_THAT(actual_metrics_set.metrics(1).stats().stddev(), Eq(10.0)); + EXPECT_THAT(actual_metrics_set.metrics(1).stats().min(), Eq(20.0)); + EXPECT_THAT(actual_metrics_set.metrics(1).stats().max(), Eq(40.0)); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/print_result_proxy_metrics_exporter.cc b/third_party/libwebrtc/api/test/metrics/print_result_proxy_metrics_exporter.cc new file mode 100644 index 0000000000..1ce1e63892 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/print_result_proxy_metrics_exporter.cc @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/print_result_proxy_metrics_exporter.h" + +#include <string> +#include <unordered_set> + +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace test { +namespace { + +std::string ToPrintResultUnit(Unit unit) { + switch (unit) { + case Unit::kMilliseconds: + return "msBestFitFormat"; + case Unit::kPercent: + return "n%"; + case Unit::kBytes: + return "sizeInBytes"; + case Unit::kKilobitsPerSecond: + // PrintResults prefer Chrome Perf Dashboard units, which doesn't have + // kpbs units, so we change the unit and value accordingly. + return "bytesPerSecond"; + case Unit::kHertz: + return "Hz"; + case Unit::kUnitless: + return "unitless"; + case Unit::kCount: + return "count"; + } +} + +double ToPrintResultValue(double value, Unit unit) { + switch (unit) { + case Unit::kKilobitsPerSecond: + // PrintResults prefer Chrome Perf Dashboard units, which doesn't have + // kpbs units, so we change the unit and value accordingly. + return value * 1000 / 8; + default: + return value; + } +} + +ImproveDirection ToPrintResultImproveDirection(ImprovementDirection direction) { + switch (direction) { + case ImprovementDirection::kBiggerIsBetter: + return ImproveDirection::kBiggerIsBetter; + case ImprovementDirection::kNeitherIsBetter: + return ImproveDirection::kNone; + case ImprovementDirection::kSmallerIsBetter: + return ImproveDirection::kSmallerIsBetter; + } +} + +bool IsEmpty(const Metric::Stats& stats) { + return !stats.mean.has_value() && !stats.stddev.has_value() && + !stats.min.has_value() && !stats.max.has_value(); +} + +bool NameEndsWithConnected(const std::string& name) { + static const std::string suffix = "_connected"; + return name.size() >= suffix.size() && + 0 == name.compare(name.size() - suffix.size(), suffix.size(), suffix); +} + +} // namespace + +bool PrintResultProxyMetricsExporter::Export( + rtc::ArrayView<const Metric> metrics) { + static const std::unordered_set<std::string> per_call_metrics{ + "actual_encode_bitrate", + "encode_frame_rate", + "harmonic_framerate", + "max_skipped", + "min_psnr_dB", + "retransmission_bitrate", + "sent_packets_loss", + "transmission_bitrate", + "dropped_frames", + "frames_in_flight", + "rendered_frames", + "average_receive_rate", + "average_send_rate", + "bytes_discarded_no_receiver", + "bytes_received", + "bytes_sent", + "packets_discarded_no_receiver", + "packets_received", + "packets_sent", + "payload_bytes_received", + "payload_bytes_sent", + "cpu_usage"}; + + for (const Metric& metric : metrics) { + if (metric.time_series.samples.empty() && IsEmpty(metric.stats)) { + // If there were no data collected for the metric it is expected that 0 + // will be exported, so add 0 to the samples. + PrintResult(metric.name, /*modifier=*/"", metric.test_case, + ToPrintResultValue(0, metric.unit), + ToPrintResultUnit(metric.unit), /*important=*/false, + ToPrintResultImproveDirection(metric.improvement_direction)); + continue; + } + + if (metric.time_series.samples.empty()) { + PrintResultMeanAndError( + metric.name, /*modifier=*/"", metric.test_case, + ToPrintResultValue(*metric.stats.mean, metric.unit), + ToPrintResultValue(*metric.stats.stddev, metric.unit), + ToPrintResultUnit(metric.unit), + /*important=*/false, + ToPrintResultImproveDirection(metric.improvement_direction)); + continue; + } + + if (metric.time_series.samples.size() == 1lu && + (per_call_metrics.count(metric.name) > 0 || + NameEndsWithConnected(metric.name))) { + // Increase backwards compatibility for 1 value use case. + PrintResult( + metric.name, /*modifier=*/"", metric.test_case, + ToPrintResultValue(metric.time_series.samples[0].value, metric.unit), + ToPrintResultUnit(metric.unit), /*important=*/false, + ToPrintResultImproveDirection(metric.improvement_direction)); + continue; + } + + SamplesStatsCounter counter; + for (size_t i = 0; i < metric.time_series.samples.size(); ++i) { + counter.AddSample(SamplesStatsCounter::StatsSample{ + .value = ToPrintResultValue(metric.time_series.samples[i].value, + metric.unit), + .time = metric.time_series.samples[i].timestamp, + .metadata = metric.time_series.samples[i].sample_metadata}); + } + + PrintResult(metric.name, /*modifier=*/"", metric.test_case, counter, + ToPrintResultUnit(metric.unit), + /*important=*/false, + ToPrintResultImproveDirection(metric.improvement_direction)); + } + return true; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/print_result_proxy_metrics_exporter.h b/third_party/libwebrtc/api/test/metrics/print_result_proxy_metrics_exporter.h new file mode 100644 index 0000000000..bad0594972 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/print_result_proxy_metrics_exporter.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_PRINT_RESULT_PROXY_METRICS_EXPORTER_H_ +#define API_TEST_METRICS_PRINT_RESULT_PROXY_METRICS_EXPORTER_H_ + +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_exporter.h" + +namespace webrtc { +namespace test { + +// Proxies all exported metrics to the `webrtc::test::PrintResult` API. +class PrintResultProxyMetricsExporter : public MetricsExporter { + public: + ~PrintResultProxyMetricsExporter() override = default; + + bool Export(rtc::ArrayView<const Metric> metrics) override; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_PRINT_RESULT_PROXY_METRICS_EXPORTER_H_ diff --git a/third_party/libwebrtc/api/test/metrics/print_result_proxy_metrics_exporter_test.cc b/third_party/libwebrtc/api/test/metrics/print_result_proxy_metrics_exporter_test.cc new file mode 100644 index 0000000000..768c794b40 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/print_result_proxy_metrics_exporter_test.cc @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/print_result_proxy_metrics_exporter.h" + +#include <map> +#include <string> +#include <vector> + +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::TestWithParam; + +std::map<std::string, std::string> DefaultMetadata() { + return std::map<std::string, std::string>{{"key", "value"}}; +} + +Metric::TimeSeries::Sample Sample(double value) { + return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1), + .value = value, + .sample_metadata = DefaultMetadata()}; +} + +TEST(PrintResultProxyMetricsExporterTest, + ExportMetricsWithTimeSeriesFormatCorrect) { + Metric metric1{ + .name = "test_metric1", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name1", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}}, + .stats = + Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}}; + Metric metric2{ + .name = "test_metric2", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .test_case = "test_case_name2", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}}, + .stats = Metric::Stats{ + .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}}; + + testing::internal::CaptureStdout(); + PrintResultProxyMetricsExporter exporter; + + std::string expected = + "RESULT test_metric1: test_case_name1= {15,5} " + "msBestFitFormat_biggerIsBetter\n" + "RESULT test_metric2: test_case_name2= {3750,1250} " + "bytesPerSecond_smallerIsBetter\n"; + + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(PrintResultProxyMetricsExporterTest, + ExportMetricsTimeSeriesOfSingleValueBackwardCompatibleFormat) { + // This should be printed as {mean, stddev} despite only being a single data + // point. + Metric metric1{ + .name = "available_send_bandwidth", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case/alice", + .metric_metadata = DefaultMetadata(), + .time_series = Metric::TimeSeries{.samples = std::vector{Sample(1000)}}, + .stats = Metric::Stats{ + .mean = 1000.0, .stddev = 0.0, .min = 1000.0, .max = 1000.0}}; + // This is a per-call metric that shouldn't have a stddev estimate. + Metric metric2{ + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case/alice-video", + .metric_metadata = DefaultMetadata(), + .time_series = Metric::TimeSeries{.samples = std::vector{Sample(10)}}, + .stats = + Metric::Stats{.mean = 10.0, .stddev = 0.0, .min = 10.0, .max = 10.0}}; + // This is a per-call metric that shouldn't have a stddev estimate. + Metric metric3{ + .name = "alice_connected", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case", + .metric_metadata = DefaultMetadata(), + .time_series = Metric::TimeSeries{.samples = std::vector{Sample(1)}}, + .stats = + Metric::Stats{.mean = 1.0, .stddev = 0.0, .min = 1.0, .max = 1.0}}; + + testing::internal::CaptureStdout(); + PrintResultProxyMetricsExporter exporter; + + std::string expected = + "RESULT available_send_bandwidth: test_case/alice= {125000,0} " + "bytesPerSecond_biggerIsBetter\n" + "RESULT min_psnr_dB: test_case/alice-video= 10 " + "unitless_biggerIsBetter\n" + "RESULT alice_connected: test_case= 1 " + "unitless_biggerIsBetter\n"; + + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2, metric3})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(PrintResultProxyMetricsExporterTest, + ExportMetricsWithStatsOnlyFormatCorrect) { + Metric metric1{.name = "test_metric1", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name1", + .metric_metadata = DefaultMetadata(), + .time_series = Metric::TimeSeries{.samples = {}}, + .stats = Metric::Stats{ + .mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}}; + Metric metric2{ + .name = "test_metric2", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .test_case = "test_case_name2", + .metric_metadata = DefaultMetadata(), + .time_series = Metric::TimeSeries{.samples = {}}, + .stats = Metric::Stats{ + .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}}; + + testing::internal::CaptureStdout(); + PrintResultProxyMetricsExporter exporter; + + std::string expected = + "RESULT test_metric1: test_case_name1= {15,5} " + "msBestFitFormat_biggerIsBetter\n" + "RESULT test_metric2: test_case_name2= {3750,1250} " + "bytesPerSecond_smallerIsBetter\n"; + + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(PrintResultProxyMetricsExporterTest, ExportEmptyMetricOnlyFormatCorrect) { + Metric metric{.name = "test_metric", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name", + .metric_metadata = DefaultMetadata(), + .time_series = Metric::TimeSeries{.samples = {}}, + .stats = Metric::Stats{}}; + + testing::internal::CaptureStdout(); + PrintResultProxyMetricsExporter exporter; + + std::string expected = + "RESULT test_metric: test_case_name= 0 " + "msBestFitFormat_biggerIsBetter\n"; + + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/proto/metric.proto b/third_party/libwebrtc/api/test/metrics/proto/metric.proto new file mode 100644 index 0000000000..cdd312589d --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/proto/metric.proto @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +syntax = "proto3"; + +package webrtc.test_metrics; + +// Root message of the proto file. Contains collection of all the metrics. +message MetricsSet { + repeated Metric metrics = 1; +} + +enum Unit { + // Default value that has to be defined. + UNDEFINED_UNIT = 0; + // General unitless value. Can be used either for dimensionless quantities + // (ex ratio) or for units not presented in this enum and too specific to add + // to this enum. + UNITLESS = 1; + MILLISECONDS = 2; + PERCENT = 3; + BYTES = 4; + KILOBITS_PER_SECOND = 5; + HERTZ = 6; + COUNT = 7; +} + +enum ImprovementDirection { + // Default value that has to be defined. + UNDEFINED_IMPROVEMENT_DIRECTION = 0; + BIGGER_IS_BETTER = 1; + NEITHER_IS_BETTER = 2; + SMALLER_IS_BETTER = 3; +} + +// Single performance metric with all related metadata. +message Metric { + // Metric name, for example PSNR, SSIM, decode_time, etc. + string name = 1; + Unit unit = 2; + ImprovementDirection improvement_direction = 3; + // If the metric is generated by a test, this field can be used to specify + // this information. + string test_case = 4; + // Metadata associated with the whole metric. + map<string, string> metric_metadata = 5; + + message TimeSeries { + message Sample { + // Timestamp in microseconds associated with a sample. For example, + // the timestamp when the sample was collected. + int64 timestamp_us = 1; + double value = 2; + // Metadata associated with this particular sample. + map<string, string> sample_metadata = 3; + } + // All samples collected for this metric. It can be empty if the Metric + // object only contains `stats`. + repeated Sample samples = 1; + } + // Contains all samples of the metric collected during test execution. + // It can be empty if the user only stores precomputed statistics into + // `stats`. + TimeSeries time_series = 6; + + // Contains metric's precomputed statistics based on the `time_series` or if + // `time_series` is omitted (has 0 samples) contains precomputed statistics + // provided by the metric's calculator. + message Stats { + // Sample mean of the metric + // (https://en.wikipedia.org/wiki/Sample_mean_and_covariance). + optional double mean = 1; + // Standard deviation (https://en.wikipedia.org/wiki/Standard_deviation). + // Is undefined if `time_series` contains only a single sample. + optional double stddev = 2; + optional double min = 3; + optional double max = 4; + } + Stats stats = 7; +} diff --git a/third_party/libwebrtc/api/test/metrics/stdout_metrics_exporter.cc b/third_party/libwebrtc/api/test/metrics/stdout_metrics_exporter.cc new file mode 100644 index 0000000000..22243e73e8 --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/stdout_metrics_exporter.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/stdout_metrics_exporter.h" + +#include <stdio.h> + +#include <cmath> +#include <string> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace test { +namespace { + +// Returns positive integral part of the number. +int64_t IntegralPart(double value) { + return std::lround(std::floor(std::abs(value))); +} + +void AppendWithPrecision(double value, + int digits_after_comma, + rtc::StringBuilder& out) { + int64_t multiplier = std::lround(std::pow(10, digits_after_comma)); + int64_t integral_part = IntegralPart(value); + double decimal_part = std::abs(value) - integral_part; + + // If decimal part has leading zeros then when it will be multiplied on + // `multiplier`, leading zeros will be lost. To preserve them we add "1" + // so then leading digit will be greater than 0 and won't be removed. + // + // During conversion to the string leading digit has to be stripped. + // + // Also due to rounding it may happen that leading digit may be incremented, + // like with `digits_after_comma` 3 number 1.9995 will be rounded to 2. In + // such case this increment has to be propagated to the `integral_part`. + int64_t decimal_holder = std::lround((1 + decimal_part) * multiplier); + if (decimal_holder >= 2 * multiplier) { + // Rounding incremented added leading digit, so we need to transfer 1 to + // integral part. + integral_part++; + decimal_holder -= multiplier; + } + // Remove trailing zeros. + while (decimal_holder % 10 == 0) { + decimal_holder /= 10; + } + + // Print serialized number to output. + if (value < 0) { + out << "-"; + } + out << integral_part; + if (decimal_holder != 1) { + out << "." << std::to_string(decimal_holder).substr(1, digits_after_comma); + } +} + +} // namespace + +StdoutMetricsExporter::StdoutMetricsExporter() : output_(stdout) {} + +bool StdoutMetricsExporter::Export(rtc::ArrayView<const Metric> metrics) { + for (const Metric& metric : metrics) { + PrintMetric(metric); + } + return true; +} + +void StdoutMetricsExporter::PrintMetric(const Metric& metric) { + rtc::StringBuilder value_stream; + value_stream << metric.test_case << " / " << metric.name << "= {mean="; + if (metric.stats.mean.has_value()) { + AppendWithPrecision(*metric.stats.mean, 8, value_stream); + } else { + value_stream << "-"; + } + value_stream << ", stddev="; + if (metric.stats.stddev.has_value()) { + AppendWithPrecision(*metric.stats.stddev, 8, value_stream); + } else { + value_stream << "-"; + } + value_stream << "} " << ToString(metric.unit) << " (" + << ToString(metric.improvement_direction) << ")"; + + fprintf(output_, "RESULT: %s\n", value_stream.str().c_str()); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/metrics/stdout_metrics_exporter.h b/third_party/libwebrtc/api/test/metrics/stdout_metrics_exporter.h new file mode 100644 index 0000000000..2c572cb2ea --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/stdout_metrics_exporter.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_STDOUT_METRICS_EXPORTER_H_ +#define API_TEST_METRICS_STDOUT_METRICS_EXPORTER_H_ + +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_exporter.h" + +namespace webrtc { +namespace test { + +// Exports all collected metrics to stdout. +class StdoutMetricsExporter : public MetricsExporter { + public: + StdoutMetricsExporter(); + ~StdoutMetricsExporter() override = default; + + StdoutMetricsExporter(const StdoutMetricsExporter&) = delete; + StdoutMetricsExporter& operator=(const StdoutMetricsExporter&) = delete; + + bool Export(rtc::ArrayView<const Metric> metrics) override; + + private: + void PrintMetric(const Metric& metric); + + FILE* const output_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_STDOUT_METRICS_EXPORTER_H_ diff --git a/third_party/libwebrtc/api/test/metrics/stdout_metrics_exporter_test.cc b/third_party/libwebrtc/api/test/metrics/stdout_metrics_exporter_test.cc new file mode 100644 index 0000000000..91c06fac5b --- /dev/null +++ b/third_party/libwebrtc/api/test/metrics/stdout_metrics_exporter_test.cc @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/stdout_metrics_exporter.h" + +#include <map> +#include <string> +#include <vector> + +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::TestWithParam; + +std::map<std::string, std::string> DefaultMetadata() { + return std::map<std::string, std::string>{{"key", "value"}}; +} + +Metric::TimeSeries::Sample Sample(double value) { + return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1), + .value = value, + .sample_metadata = DefaultMetadata()}; +} + +Metric PsnrForTestFoo(double mean, double stddev) { + return Metric{.name = "psnr", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "foo", + .time_series = Metric::TimeSeries{}, + .stats = Metric::Stats{.mean = mean, .stddev = stddev}}; +} + +TEST(StdoutMetricsExporterTest, ExportMetricFormatCorrect) { + Metric metric1{ + .name = "test_metric1", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name1", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}}, + .stats = + Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}}; + Metric metric2{ + .name = "test_metric2", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .test_case = "test_case_name2", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}}, + .stats = Metric::Stats{ + .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}}; + + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + std::string expected = + "RESULT: test_case_name1 / test_metric1= " + "{mean=15, stddev=5} Milliseconds (BiggerIsBetter)\n" + "RESULT: test_case_name2 / test_metric2= " + "{mean=30, stddev=10} KilobitsPerSecond (SmallerIsBetter)\n"; + + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, PositiveNumberMaxPrecision) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.00000001, 0.00000001); + std::string expected = + "RESULT: foo / psnr= " + "{mean=15.00000001, stddev=0.00000001} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + PositiveNumberTrailingZeroNotAdded) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.12345, 0.12); + std::string expected = + "RESULT: foo / psnr= " + "{mean=15.12345, stddev=0.12} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + PositiveNumberTrailingZeroAreRemoved) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.123450000, 0.120000000); + std::string expected = + "RESULT: foo / psnr= " + "{mean=15.12345, stddev=0.12} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + PositiveNumberRoundsUpOnPrecisionCorrectly) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.000000009, 0.999999999); + std::string expected = + "RESULT: foo / psnr= " + "{mean=15.00000001, stddev=1} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + PositiveNumberRoundsDownOnPrecisionCorrectly) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(15.0000000049, 0.9999999949); + std::string expected = + "RESULT: foo / psnr= " + "{mean=15, stddev=0.99999999} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, NegativeNumberMaxPrecision) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.00000001, -0.00000001); + std::string expected = + "RESULT: foo / psnr= " + "{mean=-15.00000001, stddev=-0.00000001} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + NegativeNumberTrailingZeroNotAdded) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.12345, -0.12); + std::string expected = + "RESULT: foo / psnr= " + "{mean=-15.12345, stddev=-0.12} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + NegativeNumberTrailingZeroAreRemoved) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.123450000, -0.120000000); + std::string expected = + "RESULT: foo / psnr= " + "{mean=-15.12345, stddev=-0.12} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + NegativeNumberRoundsUpOnPrecisionCorrectly) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.000000009, -0.999999999); + std::string expected = + "RESULT: foo / psnr= " + "{mean=-15.00000001, stddev=-1} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +TEST(StdoutMetricsExporterNumberFormatTest, + NegativeNumberRoundsDownOnPrecisionCorrectly) { + testing::internal::CaptureStdout(); + StdoutMetricsExporter exporter; + + Metric metric = PsnrForTestFoo(-15.0000000049, -0.9999999949); + std::string expected = + "RESULT: foo / psnr= " + "{mean=-15, stddev=-0.99999999} Unitless (BiggerIsBetter)\n"; + EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric})); + EXPECT_EQ(expected, testing::internal::GetCapturedStdout()); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/mock_async_dns_resolver.h b/third_party/libwebrtc/api/test/mock_async_dns_resolver.h new file mode 100644 index 0000000000..81132c96a5 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_async_dns_resolver.h @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_ASYNC_DNS_RESOLVER_H_ +#define API_TEST_MOCK_ASYNC_DNS_RESOLVER_H_ + +#include <functional> +#include <memory> + +#include "api/async_dns_resolver.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockAsyncDnsResolverResult : public AsyncDnsResolverResult { + public: + MOCK_METHOD(bool, + GetResolvedAddress, + (int, rtc::SocketAddress*), + (const, override)); + MOCK_METHOD(int, GetError, (), (const, override)); +}; + +class MockAsyncDnsResolver : public AsyncDnsResolverInterface { + public: + MOCK_METHOD(void, + Start, + (const rtc::SocketAddress&, std::function<void()>), + (override)); + MOCK_METHOD(void, + Start, + (const rtc::SocketAddress&, int family, std::function<void()>), + (override)); + MOCK_METHOD(AsyncDnsResolverResult&, result, (), (const, override)); +}; + +class MockAsyncDnsResolverFactory : public AsyncDnsResolverFactoryInterface { + public: + MOCK_METHOD(std::unique_ptr<webrtc::AsyncDnsResolverInterface>, + CreateAndResolve, + (const rtc::SocketAddress&, std::function<void()>), + (override)); + MOCK_METHOD(std::unique_ptr<webrtc::AsyncDnsResolverInterface>, + CreateAndResolve, + (const rtc::SocketAddress&, int, std::function<void()>), + (override)); + MOCK_METHOD(std::unique_ptr<webrtc::AsyncDnsResolverInterface>, + Create, + (), + (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_ASYNC_DNS_RESOLVER_H_ diff --git a/third_party/libwebrtc/api/test/mock_audio_mixer.h b/third_party/libwebrtc/api/test/mock_audio_mixer.h new file mode 100644 index 0000000000..88dc108ca3 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_audio_mixer.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_AUDIO_MIXER_H_ +#define API_TEST_MOCK_AUDIO_MIXER_H_ + +#include "api/audio/audio_mixer.h" +#include "test/gmock.h" + +namespace webrtc { +namespace test { + +class MockAudioMixer : public AudioMixer { + public: + MOCK_METHOD(bool, AddSource, (Source*), (override)); + MOCK_METHOD(void, RemoveSource, (Source*), (override)); + MOCK_METHOD(void, Mix, (size_t number_of_channels, AudioFrame*), (override)); +}; +} // namespace test +} // namespace webrtc + +#endif // API_TEST_MOCK_AUDIO_MIXER_H_ diff --git a/third_party/libwebrtc/api/test/mock_audio_sink.h b/third_party/libwebrtc/api/test/mock_audio_sink.h new file mode 100644 index 0000000000..88f38a3c57 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_audio_sink.h @@ -0,0 +1,44 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_AUDIO_SINK_H_ +#define API_TEST_MOCK_AUDIO_SINK_H_ + +#include "absl/types/optional.h" +#include "api/media_stream_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockAudioSink : public webrtc::AudioTrackSinkInterface { + public: + MOCK_METHOD(void, + OnData, + (const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames), + (override)); + + MOCK_METHOD(void, + OnData, + (const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional<int64_t> absolute_capture_timestamp_ms), + (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_AUDIO_SINK_H_ diff --git a/third_party/libwebrtc/api/test/mock_data_channel.h b/third_party/libwebrtc/api/test/mock_data_channel.h new file mode 100644 index 0000000000..38730eaa51 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_data_channel.h @@ -0,0 +1,61 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_DATA_CHANNEL_H_ +#define API_TEST_MOCK_DATA_CHANNEL_H_ + +#include <string> + +#include "api/data_channel_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDataChannelInterface + : public rtc::RefCountedObject<webrtc::DataChannelInterface> { + public: + static rtc::scoped_refptr<MockDataChannelInterface> Create() { + return rtc::scoped_refptr<MockDataChannelInterface>( + new MockDataChannelInterface()); + } + + MOCK_METHOD(void, + RegisterObserver, + (DataChannelObserver * observer), + (override)); + MOCK_METHOD(void, UnregisterObserver, (), (override)); + MOCK_METHOD(std::string, label, (), (const, override)); + MOCK_METHOD(bool, reliable, (), (const, override)); + MOCK_METHOD(bool, ordered, (), (const, override)); + MOCK_METHOD(uint16_t, maxRetransmitTime, (), (const, override)); + MOCK_METHOD(uint16_t, maxRetransmits, (), (const, override)); + MOCK_METHOD(absl::optional<int>, maxRetransmitsOpt, (), (const, override)); + MOCK_METHOD(absl::optional<int>, maxPacketLifeTime, (), (const, override)); + MOCK_METHOD(std::string, protocol, (), (const, override)); + MOCK_METHOD(bool, negotiated, (), (const, override)); + MOCK_METHOD(int, id, (), (const, override)); + MOCK_METHOD(Priority, priority, (), (const, override)); + MOCK_METHOD(DataState, state, (), (const, override)); + MOCK_METHOD(RTCError, error, (), (const, override)); + MOCK_METHOD(uint32_t, messages_sent, (), (const, override)); + MOCK_METHOD(uint64_t, bytes_sent, (), (const, override)); + MOCK_METHOD(uint32_t, messages_received, (), (const, override)); + MOCK_METHOD(uint64_t, bytes_received, (), (const, override)); + MOCK_METHOD(uint64_t, buffered_amount, (), (const, override)); + MOCK_METHOD(void, Close, (), (override)); + MOCK_METHOD(bool, Send, (const DataBuffer& buffer), (override)); + + protected: + MockDataChannelInterface() = default; +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_DATA_CHANNEL_H_ diff --git a/third_party/libwebrtc/api/test/mock_dtmf_sender.h b/third_party/libwebrtc/api/test/mock_dtmf_sender.h new file mode 100644 index 0000000000..9029195025 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_dtmf_sender.h @@ -0,0 +1,56 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_DTMF_SENDER_H_ +#define API_TEST_MOCK_DTMF_SENDER_H_ + +#include <string> + +#include "api/dtmf_sender_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDtmfSenderObserver : public DtmfSenderObserverInterface { + public: + MOCK_METHOD(void, + OnToneChange, + (const std::string&, const std::string&), + (override)); + MOCK_METHOD(void, OnToneChange, (const std::string&), (override)); +}; + +static_assert(!std::is_abstract_v<MockDtmfSenderObserver>, ""); + +class MockDtmfSender : public DtmfSenderInterface { + public: + static rtc::scoped_refptr<MockDtmfSender> Create() { + return rtc::make_ref_counted<MockDtmfSender>(); + } + + MOCK_METHOD(void, + RegisterObserver, + (DtmfSenderObserverInterface * observer), + (override)); + MOCK_METHOD(void, UnregisterObserver, (), (override)); + MOCK_METHOD(bool, CanInsertDtmf, (), (override)); + MOCK_METHOD(std::string, tones, (), (const override)); + MOCK_METHOD(int, duration, (), (const override)); + MOCK_METHOD(int, inter_tone_gap, (), (const override)); + + protected: + MockDtmfSender() = default; +}; + +static_assert(!std::is_abstract_v<rtc::RefCountedObject<MockDtmfSender>>, ""); + +} // namespace webrtc + +#endif // API_TEST_MOCK_DTMF_SENDER_H_ diff --git a/third_party/libwebrtc/api/test/mock_encoder_selector.h b/third_party/libwebrtc/api/test/mock_encoder_selector.h new file mode 100644 index 0000000000..2e018d57ba --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_encoder_selector.h @@ -0,0 +1,42 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_ENCODER_SELECTOR_H_ +#define API_TEST_MOCK_ENCODER_SELECTOR_H_ + +#include "api/video_codecs/video_encoder_factory.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockEncoderSelector + : public VideoEncoderFactory::EncoderSelectorInterface { + public: + MOCK_METHOD(void, + OnCurrentEncoder, + (const SdpVideoFormat& format), + (override)); + + MOCK_METHOD(absl::optional<SdpVideoFormat>, + OnAvailableBitrate, + (const DataRate& rate), + (override)); + + MOCK_METHOD(absl::optional<SdpVideoFormat>, + OnResolutionChange, + (const RenderResolution& resolution), + (override)); + + MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_ENCODER_SELECTOR_H_ diff --git a/third_party/libwebrtc/api/test/mock_fec_controller_override.h b/third_party/libwebrtc/api/test/mock_fec_controller_override.h new file mode 100644 index 0000000000..8f3accbc03 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_fec_controller_override.h @@ -0,0 +1,26 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_FEC_CONTROLLER_OVERRIDE_H_ +#define API_TEST_MOCK_FEC_CONTROLLER_OVERRIDE_H_ + +#include "api/fec_controller_override.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockFecControllerOverride : public FecControllerOverride { + public: + MOCK_METHOD(void, SetFecAllowed, (bool fec_allowed), (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_FEC_CONTROLLER_OVERRIDE_H_ diff --git a/third_party/libwebrtc/api/test/mock_frame_decryptor.h b/third_party/libwebrtc/api/test/mock_frame_decryptor.h new file mode 100644 index 0000000000..9604b96cc2 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_frame_decryptor.h @@ -0,0 +1,40 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_FRAME_DECRYPTOR_H_ +#define API_TEST_MOCK_FRAME_DECRYPTOR_H_ + +#include <vector> + +#include "api/crypto/frame_decryptor_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockFrameDecryptor : public FrameDecryptorInterface { + public: + MOCK_METHOD(Result, + Decrypt, + (cricket::MediaType, + const std::vector<uint32_t>&, + rtc::ArrayView<const uint8_t>, + rtc::ArrayView<const uint8_t>, + rtc::ArrayView<uint8_t>), + (override)); + + MOCK_METHOD(size_t, + GetMaxPlaintextByteSize, + (cricket::MediaType, size_t encrypted_frame_size), + (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_FRAME_DECRYPTOR_H_ diff --git a/third_party/libwebrtc/api/test/mock_frame_encryptor.h b/third_party/libwebrtc/api/test/mock_frame_encryptor.h new file mode 100644 index 0000000000..e47321f801 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_frame_encryptor.h @@ -0,0 +1,39 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_FRAME_ENCRYPTOR_H_ +#define API_TEST_MOCK_FRAME_ENCRYPTOR_H_ + +#include "api/crypto/frame_encryptor_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockFrameEncryptor : public FrameEncryptorInterface { + public: + MOCK_METHOD(int, + Encrypt, + (cricket::MediaType, + uint32_t, + rtc::ArrayView<const uint8_t>, + rtc::ArrayView<const uint8_t>, + rtc::ArrayView<uint8_t>, + size_t*), + (override)); + + MOCK_METHOD(size_t, + GetMaxCiphertextByteSize, + (cricket::MediaType media_type, size_t frame_size), + (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_FRAME_ENCRYPTOR_H_ diff --git a/third_party/libwebrtc/api/test/mock_media_stream_interface.h b/third_party/libwebrtc/api/test/mock_media_stream_interface.h new file mode 100644 index 0000000000..dfdbab35e9 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_media_stream_interface.h @@ -0,0 +1,134 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_MEDIA_STREAM_INTERFACE_H_ +#define API_TEST_MOCK_MEDIA_STREAM_INTERFACE_H_ + +#include <string> + +#include "api/media_stream_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockAudioSource : public rtc::RefCountedObject<AudioSourceInterface> { + public: + static rtc::scoped_refptr<MockAudioSource> Create() { + return rtc::scoped_refptr<MockAudioSource>(new MockAudioSource()); + } + + MOCK_METHOD(void, + RegisterObserver, + (ObserverInterface * observer), + (override)); + MOCK_METHOD(void, + UnregisterObserver, + (ObserverInterface * observer), + (override)); + MOCK_METHOD(SourceState, state, (), (const, override)); + MOCK_METHOD(bool, remote, (), (const, override)); + MOCK_METHOD(void, SetVolume, (double volume), (override)); + MOCK_METHOD(void, + RegisterAudioObserver, + (AudioObserver * observer), + (override)); + MOCK_METHOD(void, + UnregisterAudioObserver, + (AudioObserver * observer), + (override)); + MOCK_METHOD(void, AddSink, (AudioTrackSinkInterface * sink), (override)); + MOCK_METHOD(void, RemoveSink, (AudioTrackSinkInterface * sink), (override)); + MOCK_METHOD(const cricket::AudioOptions, options, (), (const, override)); + + private: + MockAudioSource() = default; +}; + +class MockAudioTrack : public rtc::RefCountedObject<AudioTrackInterface> { + public: + static rtc::scoped_refptr<MockAudioTrack> Create() { + return rtc::scoped_refptr<MockAudioTrack>(new MockAudioTrack()); + } + + MOCK_METHOD(void, + RegisterObserver, + (ObserverInterface * observer), + (override)); + MOCK_METHOD(void, + UnregisterObserver, + (ObserverInterface * observer), + (override)); + MOCK_METHOD(std::string, kind, (), (const, override)); + MOCK_METHOD(std::string, id, (), (const, override)); + MOCK_METHOD(bool, enabled, (), (const, override)); + MOCK_METHOD(bool, set_enabled, (bool enable), (override)); + MOCK_METHOD(TrackState, state, (), (const, override)); + MOCK_METHOD(AudioSourceInterface*, GetSource, (), (const, override)); + MOCK_METHOD(void, AddSink, (AudioTrackSinkInterface * sink), (override)); + MOCK_METHOD(void, RemoveSink, (AudioTrackSinkInterface * sink), (override)); + MOCK_METHOD(bool, GetSignalLevel, (int* level), (override)); + MOCK_METHOD(rtc::scoped_refptr<AudioProcessorInterface>, + GetAudioProcessor, + (), + (override)); + + private: + MockAudioTrack() = default; +}; + +class MockMediaStream : public MediaStreamInterface { + public: + MOCK_METHOD(std::string, id, (), (const override)); + MOCK_METHOD(AudioTrackVector, GetAudioTracks, (), (override)); + MOCK_METHOD(VideoTrackVector, GetVideoTracks, (), (override)); + MOCK_METHOD(rtc::scoped_refptr<AudioTrackInterface>, + FindAudioTrack, + (const std::string& track_id), + (override)); + MOCK_METHOD(rtc::scoped_refptr<VideoTrackInterface>, + FindVideoTrack, + (const std::string& track_id), + (override)); + MOCK_METHOD(bool, + AddTrack, + (rtc::scoped_refptr<AudioTrackInterface> track), + (override)); + MOCK_METHOD(bool, + AddTrack, + (rtc::scoped_refptr<VideoTrackInterface> track), + (override)); + MOCK_METHOD(bool, + RemoveTrack, + (rtc::scoped_refptr<AudioTrackInterface> track), + (override)); + MOCK_METHOD(bool, + RemoveTrack, + (rtc::scoped_refptr<VideoTrackInterface> track), + (override)); + // Old AddTrack/RemoveTrack methods - slated for removal + MOCK_METHOD(bool, AddTrack, (AudioTrackInterface * track), (override)); + MOCK_METHOD(bool, AddTrack, (VideoTrackInterface * track), (override)); + MOCK_METHOD(bool, RemoveTrack, (AudioTrackInterface * track), (override)); + MOCK_METHOD(bool, RemoveTrack, (VideoTrackInterface * track), (override)); + MOCK_METHOD(void, + RegisterObserver, + (ObserverInterface * observer), + (override)); + MOCK_METHOD(void, + UnregisterObserver, + (ObserverInterface * observer), + (override)); +}; + +static_assert(!std::is_abstract_v<rtc::RefCountedObject<MockMediaStream>>, ""); + +} // namespace webrtc + +#endif // API_TEST_MOCK_MEDIA_STREAM_INTERFACE_H_ diff --git a/third_party/libwebrtc/api/test/mock_packet_socket_factory.h b/third_party/libwebrtc/api/test/mock_packet_socket_factory.h new file mode 100644 index 0000000000..7e59556385 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_packet_socket_factory.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_PACKET_SOCKET_FACTORY_H_ +#define API_TEST_MOCK_PACKET_SOCKET_FACTORY_H_ + +#include <memory> +#include <string> + +#include "api/packet_socket_factory.h" +#include "test/gmock.h" + +namespace rtc { +class MockPacketSocketFactory : public PacketSocketFactory { + public: + MOCK_METHOD(AsyncPacketSocket*, + CreateUdpSocket, + (const SocketAddress&, uint16_t, uint16_t), + (override)); + MOCK_METHOD(AsyncListenSocket*, + CreateServerTcpSocket, + (const SocketAddress&, uint16_t, uint16_t, int opts), + (override)); + MOCK_METHOD(AsyncPacketSocket*, + CreateClientTcpSocket, + (const SocketAddress& local_address, + const SocketAddress&, + const ProxyInfo&, + const std::string&, + const PacketSocketTcpOptions&), + (override)); + MOCK_METHOD(std::unique_ptr<webrtc::AsyncDnsResolverInterface>, + CreateAsyncDnsResolver, + (), + (override)); +}; + +static_assert(!std::is_abstract_v<MockPacketSocketFactory>, ""); + +} // namespace rtc + +#endif // API_TEST_MOCK_PACKET_SOCKET_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/mock_peer_connection_factory_interface.h b/third_party/libwebrtc/api/test/mock_peer_connection_factory_interface.h new file mode 100644 index 0000000000..ae1fbfbbb7 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_peer_connection_factory_interface.h @@ -0,0 +1,81 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_PEER_CONNECTION_FACTORY_INTERFACE_H_ +#define API_TEST_MOCK_PEER_CONNECTION_FACTORY_INTERFACE_H_ + +#include <memory> +#include <string> + +#include "api/peer_connection_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockPeerConnectionFactoryInterface + : public rtc::RefCountedObject<webrtc::PeerConnectionFactoryInterface> { + public: + static rtc::scoped_refptr<MockPeerConnectionFactoryInterface> Create() { + return rtc::scoped_refptr<MockPeerConnectionFactoryInterface>( + new MockPeerConnectionFactoryInterface()); + } + + MOCK_METHOD(void, SetOptions, (const Options&), (override)); + MOCK_METHOD(rtc::scoped_refptr<PeerConnectionInterface>, + CreatePeerConnection, + (const PeerConnectionInterface::RTCConfiguration&, + PeerConnectionDependencies), + (override)); + MOCK_METHOD(RTCErrorOr<rtc::scoped_refptr<PeerConnectionInterface>>, + CreatePeerConnectionOrError, + (const PeerConnectionInterface::RTCConfiguration&, + PeerConnectionDependencies), + (override)); + MOCK_METHOD(rtc::scoped_refptr<PeerConnectionInterface>, + CreatePeerConnection, + (const PeerConnectionInterface::RTCConfiguration&, + std::unique_ptr<cricket::PortAllocator>, + std::unique_ptr<rtc::RTCCertificateGeneratorInterface>, + PeerConnectionObserver*), + (override)); + MOCK_METHOD(RtpCapabilities, + GetRtpSenderCapabilities, + (cricket::MediaType), + (const, override)); + MOCK_METHOD(RtpCapabilities, + GetRtpReceiverCapabilities, + (cricket::MediaType), + (const, override)); + MOCK_METHOD(rtc::scoped_refptr<MediaStreamInterface>, + CreateLocalMediaStream, + (const std::string&), + (override)); + MOCK_METHOD(rtc::scoped_refptr<AudioSourceInterface>, + CreateAudioSource, + (const cricket::AudioOptions&), + (override)); + MOCK_METHOD(rtc::scoped_refptr<VideoTrackInterface>, + CreateVideoTrack, + (const std::string&, VideoTrackSourceInterface*), + (override)); + MOCK_METHOD(rtc::scoped_refptr<AudioTrackInterface>, + CreateAudioTrack, + (const std::string&, AudioSourceInterface*), + (override)); + MOCK_METHOD(bool, StartAecDump, (FILE*, int64_t), (override)); + MOCK_METHOD(void, StopAecDump, (), (override)); + + protected: + MockPeerConnectionFactoryInterface() = default; +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_PEER_CONNECTION_FACTORY_INTERFACE_H_ diff --git a/third_party/libwebrtc/api/test/mock_peerconnectioninterface.h b/third_party/libwebrtc/api/test/mock_peerconnectioninterface.h new file mode 100644 index 0000000000..ccc6ce46b1 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_peerconnectioninterface.h @@ -0,0 +1,213 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_PEERCONNECTIONINTERFACE_H_ +#define API_TEST_MOCK_PEERCONNECTIONINTERFACE_H_ + +#include <memory> +#include <string> +#include <type_traits> +#include <utility> +#include <vector> + +#include "api/peer_connection_interface.h" +#include "api/scoped_refptr.h" +#include "api/sctp_transport_interface.h" +#include "rtc_base/ref_counted_object.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockPeerConnectionInterface : public webrtc::PeerConnectionInterface { + public: + static rtc::scoped_refptr<MockPeerConnectionInterface> Create() { + return rtc::make_ref_counted<MockPeerConnectionInterface>(); + } + + // PeerConnectionInterface + MOCK_METHOD(rtc::scoped_refptr<StreamCollectionInterface>, + local_streams, + (), + (override)); + MOCK_METHOD(rtc::scoped_refptr<StreamCollectionInterface>, + remote_streams, + (), + (override)); + MOCK_METHOD(bool, AddStream, (MediaStreamInterface*), (override)); + MOCK_METHOD(void, RemoveStream, (MediaStreamInterface*), (override)); + MOCK_METHOD(RTCErrorOr<rtc::scoped_refptr<RtpSenderInterface>>, + AddTrack, + (rtc::scoped_refptr<MediaStreamTrackInterface>, + const std::vector<std::string>&), + (override)); + MOCK_METHOD(RTCErrorOr<rtc::scoped_refptr<RtpSenderInterface>>, + AddTrack, + (rtc::scoped_refptr<MediaStreamTrackInterface>, + const std::vector<std::string>&, + const std::vector<RtpEncodingParameters>&), + (override)); + MOCK_METHOD(RTCError, + RemoveTrackOrError, + (rtc::scoped_refptr<RtpSenderInterface>), + (override)); + MOCK_METHOD(RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>>, + AddTransceiver, + (rtc::scoped_refptr<MediaStreamTrackInterface>), + (override)); + MOCK_METHOD(RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>>, + AddTransceiver, + (rtc::scoped_refptr<MediaStreamTrackInterface>, + const RtpTransceiverInit&), + (override)); + MOCK_METHOD(RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>>, + AddTransceiver, + (cricket::MediaType), + (override)); + MOCK_METHOD(RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>>, + AddTransceiver, + (cricket::MediaType, const RtpTransceiverInit&), + (override)); + MOCK_METHOD(rtc::scoped_refptr<RtpSenderInterface>, + CreateSender, + (const std::string&, const std::string&), + (override)); + MOCK_METHOD(std::vector<rtc::scoped_refptr<RtpSenderInterface>>, + GetSenders, + (), + (const, override)); + MOCK_METHOD(std::vector<rtc::scoped_refptr<RtpReceiverInterface>>, + GetReceivers, + (), + (const, override)); + MOCK_METHOD(std::vector<rtc::scoped_refptr<RtpTransceiverInterface>>, + GetTransceivers, + (), + (const, override)); + MOCK_METHOD(bool, + GetStats, + (StatsObserver*, MediaStreamTrackInterface*, StatsOutputLevel), + (override)); + MOCK_METHOD(void, GetStats, (RTCStatsCollectorCallback*), (override)); + MOCK_METHOD(void, + GetStats, + (rtc::scoped_refptr<RtpSenderInterface>, + rtc::scoped_refptr<RTCStatsCollectorCallback>), + (override)); + MOCK_METHOD(void, + GetStats, + (rtc::scoped_refptr<RtpReceiverInterface>, + rtc::scoped_refptr<RTCStatsCollectorCallback>), + (override)); + MOCK_METHOD(void, ClearStatsCache, (), (override)); + MOCK_METHOD(rtc::scoped_refptr<SctpTransportInterface>, + GetSctpTransport, + (), + (const, override)); + MOCK_METHOD(RTCErrorOr<rtc::scoped_refptr<DataChannelInterface>>, + CreateDataChannelOrError, + (const std::string&, const DataChannelInit*), + (override)); + MOCK_METHOD(const SessionDescriptionInterface*, + local_description, + (), + (const, override)); + MOCK_METHOD(const SessionDescriptionInterface*, + remote_description, + (), + (const, override)); + MOCK_METHOD(const SessionDescriptionInterface*, + current_local_description, + (), + (const, override)); + MOCK_METHOD(const SessionDescriptionInterface*, + current_remote_description, + (), + (const, override)); + MOCK_METHOD(const SessionDescriptionInterface*, + pending_local_description, + (), + (const, override)); + MOCK_METHOD(const SessionDescriptionInterface*, + pending_remote_description, + (), + (const, override)); + MOCK_METHOD(void, RestartIce, (), (override)); + MOCK_METHOD(void, + CreateOffer, + (CreateSessionDescriptionObserver*, const RTCOfferAnswerOptions&), + (override)); + MOCK_METHOD(void, + CreateAnswer, + (CreateSessionDescriptionObserver*, const RTCOfferAnswerOptions&), + (override)); + MOCK_METHOD(void, + SetLocalDescription, + (SetSessionDescriptionObserver*, SessionDescriptionInterface*), + (override)); + MOCK_METHOD(void, + SetRemoteDescription, + (SetSessionDescriptionObserver*, SessionDescriptionInterface*), + (override)); + MOCK_METHOD(void, + SetRemoteDescription, + (std::unique_ptr<SessionDescriptionInterface>, + rtc::scoped_refptr<SetRemoteDescriptionObserverInterface>), + (override)); + MOCK_METHOD(PeerConnectionInterface::RTCConfiguration, + GetConfiguration, + (), + (override)); + MOCK_METHOD(RTCError, + SetConfiguration, + (const PeerConnectionInterface::RTCConfiguration&), + (override)); + MOCK_METHOD(bool, + AddIceCandidate, + (const IceCandidateInterface*), + (override)); + MOCK_METHOD(bool, + RemoveIceCandidates, + (const std::vector<cricket::Candidate>&), + (override)); + MOCK_METHOD(RTCError, SetBitrate, (const BitrateSettings&), (override)); + MOCK_METHOD(void, SetAudioPlayout, (bool), (override)); + MOCK_METHOD(void, SetAudioRecording, (bool), (override)); + MOCK_METHOD(rtc::scoped_refptr<DtlsTransportInterface>, + LookupDtlsTransportByMid, + (const std::string&), + (override)); + MOCK_METHOD(SignalingState, signaling_state, (), (override)); + MOCK_METHOD(IceConnectionState, ice_connection_state, (), (override)); + MOCK_METHOD(IceConnectionState, + standardized_ice_connection_state, + (), + (override)); + MOCK_METHOD(PeerConnectionState, peer_connection_state, (), (override)); + MOCK_METHOD(IceGatheringState, ice_gathering_state, (), (override)); + MOCK_METHOD(absl::optional<bool>, can_trickle_ice_candidates, (), (override)); + MOCK_METHOD(bool, + StartRtcEventLog, + (std::unique_ptr<RtcEventLogOutput>, int64_t), + (override)); + MOCK_METHOD(bool, + StartRtcEventLog, + (std::unique_ptr<RtcEventLogOutput>), + (override)); + MOCK_METHOD(void, StopRtcEventLog, (), (override)); + MOCK_METHOD(void, Close, (), (override)); +}; + +static_assert( + !std::is_abstract_v<rtc::RefCountedObject<MockPeerConnectionInterface>>, + ""); + +} // namespace webrtc + +#endif // API_TEST_MOCK_PEERCONNECTIONINTERFACE_H_ diff --git a/third_party/libwebrtc/api/test/mock_rtp_transceiver.h b/third_party/libwebrtc/api/test/mock_rtp_transceiver.h new file mode 100644 index 0000000000..1d21bce5eb --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_rtp_transceiver.h @@ -0,0 +1,87 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_RTP_TRANSCEIVER_H_ +#define API_TEST_MOCK_RTP_TRANSCEIVER_H_ + +#include <string> +#include <vector> + +#include "api/rtp_transceiver_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockRtpTransceiver : public RtpTransceiverInterface { + public: + MockRtpTransceiver() = default; + + static rtc::scoped_refptr<MockRtpTransceiver> Create() { + return rtc::make_ref_counted<MockRtpTransceiver>(); + } + + MOCK_METHOD(cricket::MediaType, media_type, (), (const, override)); + MOCK_METHOD(absl::optional<std::string>, mid, (), (const, override)); + MOCK_METHOD(rtc::scoped_refptr<RtpSenderInterface>, + sender, + (), + (const, override)); + MOCK_METHOD(rtc::scoped_refptr<RtpReceiverInterface>, + receiver, + (), + (const, override)); + MOCK_METHOD(bool, stopped, (), (const, override)); + MOCK_METHOD(bool, stopping, (), (const, override)); + MOCK_METHOD(RtpTransceiverDirection, direction, (), (const, override)); + MOCK_METHOD(void, + SetDirection, + (RtpTransceiverDirection new_direction), + (override)); + MOCK_METHOD(RTCError, + SetDirectionWithError, + (RtpTransceiverDirection new_direction), + (override)); + MOCK_METHOD(absl::optional<RtpTransceiverDirection>, + current_direction, + (), + (const, override)); + MOCK_METHOD(absl::optional<RtpTransceiverDirection>, + fired_direction, + (), + (const, override)); + MOCK_METHOD(RTCError, StopStandard, (), (override)); + MOCK_METHOD(void, StopInternal, (), (override)); + MOCK_METHOD(void, Stop, (), (override)); + MOCK_METHOD(RTCError, + SetCodecPreferences, + (rtc::ArrayView<RtpCodecCapability> codecs), + (override)); + MOCK_METHOD(std::vector<RtpCodecCapability>, + codec_preferences, + (), + (const, override)); + MOCK_METHOD(std::vector<RtpHeaderExtensionCapability>, + HeaderExtensionsToOffer, + (), + (const, override)); + MOCK_METHOD(std::vector<RtpHeaderExtensionCapability>, + HeaderExtensionsNegotiated, + (), + (const, override)); + MOCK_METHOD(webrtc::RTCError, + SetOfferedRtpHeaderExtensions, + (rtc::ArrayView<const RtpHeaderExtensionCapability> + header_extensions_to_offer), + (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_RTP_TRANSCEIVER_H_ diff --git a/third_party/libwebrtc/api/test/mock_rtpreceiver.h b/third_party/libwebrtc/api/test/mock_rtpreceiver.h new file mode 100644 index 0000000000..63318dc32d --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_rtpreceiver.h @@ -0,0 +1,58 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_RTPRECEIVER_H_ +#define API_TEST_MOCK_RTPRECEIVER_H_ + +#include <string> +#include <vector> + +#include "api/crypto/frame_decryptor_interface.h" +#include "api/rtp_receiver_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockRtpReceiver : public rtc::RefCountedObject<RtpReceiverInterface> { + public: + MOCK_METHOD(rtc::scoped_refptr<MediaStreamTrackInterface>, + track, + (), + (const, override)); + MOCK_METHOD(std::vector<rtc::scoped_refptr<MediaStreamInterface>>, + streams, + (), + (const, override)); + MOCK_METHOD(cricket::MediaType, media_type, (), (const, override)); + MOCK_METHOD(std::string, id, (), (const, override)); + MOCK_METHOD(RtpParameters, GetParameters, (), (const, override)); + MOCK_METHOD(bool, + SetParameters, + (const webrtc::RtpParameters& parameters), + (override)); + MOCK_METHOD(void, SetObserver, (RtpReceiverObserverInterface*), (override)); + MOCK_METHOD(void, + SetJitterBufferMinimumDelay, + (absl::optional<double>), + (override)); + MOCK_METHOD(std::vector<RtpSource>, GetSources, (), (const, override)); + MOCK_METHOD(void, + SetFrameDecryptor, + (rtc::scoped_refptr<webrtc::FrameDecryptorInterface>), + (override)); + MOCK_METHOD(rtc::scoped_refptr<webrtc::FrameDecryptorInterface>, + GetFrameDecryptor, + (), + (const, override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_RTPRECEIVER_H_ diff --git a/third_party/libwebrtc/api/test/mock_rtpsender.h b/third_party/libwebrtc/api/test/mock_rtpsender.h new file mode 100644 index 0000000000..22113678b9 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_rtpsender.h @@ -0,0 +1,78 @@ +/* + * Copyright 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_RTPSENDER_H_ +#define API_TEST_MOCK_RTPSENDER_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "api/rtp_sender_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockRtpSender : public RtpSenderInterface { + public: + static rtc::scoped_refptr<MockRtpSender> Create() { + return rtc::make_ref_counted<MockRtpSender>(); + } + + MOCK_METHOD(bool, SetTrack, (MediaStreamTrackInterface*), (override)); + MOCK_METHOD(rtc::scoped_refptr<MediaStreamTrackInterface>, + track, + (), + (const, override)); + MOCK_METHOD(rtc::scoped_refptr<DtlsTransportInterface>, + dtls_transport, + (), + (const override)); + MOCK_METHOD(uint32_t, ssrc, (), (const, override)); + MOCK_METHOD(cricket::MediaType, media_type, (), (const, override)); + MOCK_METHOD(std::string, id, (), (const, override)); + MOCK_METHOD(std::vector<std::string>, stream_ids, (), (const, override)); + MOCK_METHOD(void, SetStreams, (const std::vector<std::string>&), (override)); + MOCK_METHOD(std::vector<RtpEncodingParameters>, + init_send_encodings, + (), + (const, override)); + MOCK_METHOD(RtpParameters, GetParameters, (), (const, override)); + MOCK_METHOD(RTCError, SetParameters, (const RtpParameters&), (override)); + MOCK_METHOD(void, + SetParametersAsync, + (const RtpParameters&, SetParametersCallback), + (override)); + MOCK_METHOD(rtc::scoped_refptr<DtmfSenderInterface>, + GetDtmfSender, + (), + (const, override)); + MOCK_METHOD(void, + SetFrameEncryptor, + (rtc::scoped_refptr<FrameEncryptorInterface>), + (override)); + MOCK_METHOD(rtc::scoped_refptr<FrameEncryptorInterface>, + GetFrameEncryptor, + (), + (const, override)); + MOCK_METHOD(void, + SetEncoderToPacketizerFrameTransformer, + (rtc::scoped_refptr<FrameTransformerInterface>), + (override)); + MOCK_METHOD(void, + SetEncoderSelector, + (std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface>), + (override)); +}; + +static_assert(!std::is_abstract_v<rtc::RefCountedObject<MockRtpSender>>, ""); +} // namespace webrtc + +#endif // API_TEST_MOCK_RTPSENDER_H_ diff --git a/third_party/libwebrtc/api/test/mock_session_description_interface.h b/third_party/libwebrtc/api/test/mock_session_description_interface.h new file mode 100644 index 0000000000..f0346ceb11 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_session_description_interface.h @@ -0,0 +1,56 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_SESSION_DESCRIPTION_INTERFACE_H_ +#define API_TEST_MOCK_SESSION_DESCRIPTION_INTERFACE_H_ + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "api/jsep.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockSessionDescriptionInterface : public SessionDescriptionInterface { + public: + MOCK_METHOD(std::unique_ptr<SessionDescriptionInterface>, + Clone, + (), + (const, override)); + MOCK_METHOD(cricket::SessionDescription*, description, (), (override)); + MOCK_METHOD(const cricket::SessionDescription*, + description, + (), + (const, override)); + MOCK_METHOD(std::string, session_id, (), (const, override)); + MOCK_METHOD(std::string, session_version, (), (const, override)); + MOCK_METHOD(SdpType, GetType, (), (const, override)); + MOCK_METHOD(std::string, type, (), (const, override)); + MOCK_METHOD(bool, AddCandidate, (const IceCandidateInterface*), (override)); + MOCK_METHOD(size_t, + RemoveCandidates, + (const std::vector<cricket::Candidate>&), + (override)); + MOCK_METHOD(size_t, number_of_mediasections, (), (const, override)); + MOCK_METHOD(const IceCandidateCollection*, + candidates, + (size_t), + (const, override)); + MOCK_METHOD(bool, ToString, (std::string*), (const, override)); +}; + +static_assert(!std::is_abstract_v<MockSessionDescriptionInterface>); + +} // namespace webrtc + +#endif // API_TEST_MOCK_SESSION_DESCRIPTION_INTERFACE_H_ diff --git a/third_party/libwebrtc/api/test/mock_transformable_video_frame.h b/third_party/libwebrtc/api/test/mock_transformable_video_frame.h new file mode 100644 index 0000000000..18b3de534e --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_transformable_video_frame.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_TRANSFORMABLE_VIDEO_FRAME_H_ +#define API_TEST_MOCK_TRANSFORMABLE_VIDEO_FRAME_H_ + +#include <vector> + +#include "api/frame_transformer_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockTransformableVideoFrame + : public webrtc::TransformableVideoFrameInterface { + public: + MOCK_METHOD(rtc::ArrayView<const uint8_t>, GetData, (), (const, override)); + MOCK_METHOD(void, SetData, (rtc::ArrayView<const uint8_t> data), (override)); + MOCK_METHOD(uint32_t, GetTimestamp, (), (const, override)); + MOCK_METHOD(uint32_t, GetSsrc, (), (const, override)); + MOCK_METHOD(bool, IsKeyFrame, (), (const, override)); + MOCK_METHOD(std::vector<uint8_t>, GetAdditionalData, (), (const, override)); + MOCK_METHOD(const webrtc::VideoFrameMetadata&, + GetMetadata, + (), + (const, override)); + MOCK_METHOD(void, + SetMetadata, + (const webrtc::VideoFrameMetadata&), + (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_TRANSFORMABLE_VIDEO_FRAME_H_ diff --git a/third_party/libwebrtc/api/test/mock_video_bitrate_allocator.h b/third_party/libwebrtc/api/test/mock_video_bitrate_allocator.h new file mode 100644 index 0000000000..76cf49e955 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_video_bitrate_allocator.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_VIDEO_BITRATE_ALLOCATOR_H_ +#define API_TEST_MOCK_VIDEO_BITRATE_ALLOCATOR_H_ + +#include "api/video/video_bitrate_allocator.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockVideoBitrateAllocator : public webrtc::VideoBitrateAllocator { + MOCK_METHOD(VideoBitrateAllocation, + Allocate, + (VideoBitrateAllocationParameters parameters), + (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_VIDEO_BITRATE_ALLOCATOR_H_ diff --git a/third_party/libwebrtc/api/test/mock_video_bitrate_allocator_factory.h b/third_party/libwebrtc/api/test/mock_video_bitrate_allocator_factory.h new file mode 100644 index 0000000000..16af191970 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_video_bitrate_allocator_factory.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_VIDEO_BITRATE_ALLOCATOR_FACTORY_H_ +#define API_TEST_MOCK_VIDEO_BITRATE_ALLOCATOR_FACTORY_H_ + +#include <memory> + +#include "api/video/video_bitrate_allocator_factory.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockVideoBitrateAllocatorFactory + : public webrtc::VideoBitrateAllocatorFactory { + public: + ~MockVideoBitrateAllocatorFactory() override { Die(); } + MOCK_METHOD(std::unique_ptr<VideoBitrateAllocator>, + CreateVideoBitrateAllocator, + (const VideoCodec&), + (override)); + MOCK_METHOD(void, Die, ()); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_VIDEO_BITRATE_ALLOCATOR_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/mock_video_decoder.h b/third_party/libwebrtc/api/test/mock_video_decoder.h new file mode 100644 index 0000000000..34f732ca4d --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_video_decoder.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_VIDEO_DECODER_H_ +#define API_TEST_MOCK_VIDEO_DECODER_H_ + +#include <utility> + +#include "api/video_codecs/video_decoder.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDecodedImageCallback : public DecodedImageCallback { + public: + MOCK_METHOD(int32_t, + Decoded, + (VideoFrame & decoded_image), // NOLINT + (override)); + MOCK_METHOD(int32_t, + Decoded, + (VideoFrame & decoded_image, // NOLINT + int64_t decode_time_ms), + (override)); + MOCK_METHOD(void, + Decoded, + (VideoFrame & decoded_image, // NOLINT + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp), + (override)); +}; + +class MockVideoDecoder : public VideoDecoder { + public: + MockVideoDecoder() { + // Make `Configure` succeed by default, so that individual tests that + // verify other methods wouldn't need to stub `Configure`. + ON_CALL(*this, Configure).WillByDefault(testing::Return(true)); + } + + ~MockVideoDecoder() override { Destruct(); } + + MOCK_METHOD(bool, Configure, (const Settings& settings), (override)); + MOCK_METHOD(int32_t, + Decode, + (const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms), + (override)); + MOCK_METHOD(int32_t, + RegisterDecodeCompleteCallback, + (DecodedImageCallback * callback), + (override)); + MOCK_METHOD(int32_t, Release, (), (override)); + + // Special utility method that allows a test to monitor/verify when + // destruction of the decoder instance occurs. + MOCK_METHOD(void, Destruct, (), ()); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_VIDEO_DECODER_H_ diff --git a/third_party/libwebrtc/api/test/mock_video_decoder_factory.h b/third_party/libwebrtc/api/test/mock_video_decoder_factory.h new file mode 100644 index 0000000000..6150d9f8b5 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_video_decoder_factory.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_VIDEO_DECODER_FACTORY_H_ +#define API_TEST_MOCK_VIDEO_DECODER_FACTORY_H_ + +#include <memory> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockVideoDecoderFactory : public webrtc::VideoDecoderFactory { + public: + ~MockVideoDecoderFactory() override { Die(); } + + MOCK_METHOD(std::vector<webrtc::SdpVideoFormat>, + GetSupportedFormats, + (), + (const, override)); + MOCK_METHOD(std::unique_ptr<webrtc::VideoDecoder>, + CreateVideoDecoder, + (const webrtc::SdpVideoFormat&), + (override)); + MOCK_METHOD(void, Die, ()); +}; +} // namespace webrtc + +#endif // API_TEST_MOCK_VIDEO_DECODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/mock_video_encoder.h b/third_party/libwebrtc/api/test/mock_video_encoder.h new file mode 100644 index 0000000000..11e0f64b3f --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_video_encoder.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_VIDEO_ENCODER_H_ +#define API_TEST_MOCK_VIDEO_ENCODER_H_ + +#include <vector> + +#include "api/video_codecs/video_encoder.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockEncodedImageCallback : public EncodedImageCallback { + public: + MOCK_METHOD(Result, + OnEncodedImage, + (const EncodedImage&, const CodecSpecificInfo*), + (override)); + MOCK_METHOD(void, OnDroppedFrame, (DropReason reason), (override)); +}; + +class MockVideoEncoder : public VideoEncoder { + public: + MOCK_METHOD(void, + SetFecControllerOverride, + (FecControllerOverride*), + (override)); + MOCK_METHOD(int32_t, + InitEncode, + (const VideoCodec*, int32_t numberOfCores, size_t maxPayloadSize), + (override)); + MOCK_METHOD(int32_t, + InitEncode, + (const VideoCodec*, const VideoEncoder::Settings& settings), + (override)); + + MOCK_METHOD(int32_t, + Encode, + (const VideoFrame& inputImage, + const std::vector<VideoFrameType>*), + (override)); + MOCK_METHOD(int32_t, + RegisterEncodeCompleteCallback, + (EncodedImageCallback*), + (override)); + MOCK_METHOD(int32_t, Release, (), (override)); + MOCK_METHOD(void, + SetRates, + (const RateControlParameters& parameters), + (override)); + MOCK_METHOD(void, + OnPacketLossRateUpdate, + (float packet_loss_rate), + (override)); + MOCK_METHOD(void, OnRttUpdate, (int64_t rtt_ms), (override)); + MOCK_METHOD(void, + OnLossNotification, + (const LossNotification& loss_notification), + (override)); + MOCK_METHOD(EncoderInfo, GetEncoderInfo, (), (const, override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_VIDEO_ENCODER_H_ diff --git a/third_party/libwebrtc/api/test/mock_video_encoder_factory.h b/third_party/libwebrtc/api/test/mock_video_encoder_factory.h new file mode 100644 index 0000000000..02ee7aa15e --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_video_encoder_factory.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_VIDEO_ENCODER_FACTORY_H_ +#define API_TEST_MOCK_VIDEO_ENCODER_FACTORY_H_ + +#include <memory> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockVideoEncoderFactory : public webrtc::VideoEncoderFactory { + public: + ~MockVideoEncoderFactory() override { Die(); } + + MOCK_METHOD(std::vector<SdpVideoFormat>, + GetSupportedFormats, + (), + (const, override)); + MOCK_METHOD(std::unique_ptr<VideoEncoder>, + CreateVideoEncoder, + (const SdpVideoFormat&), + (override)); + + MOCK_METHOD(void, Die, ()); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_VIDEO_ENCODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/mock_video_track.h b/third_party/libwebrtc/api/test/mock_video_track.h new file mode 100644 index 0000000000..1212a32527 --- /dev/null +++ b/third_party/libwebrtc/api/test/mock_video_track.h @@ -0,0 +1,69 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_VIDEO_TRACK_H_ +#define API_TEST_MOCK_VIDEO_TRACK_H_ + +#include <string> + +#include "api/media_stream_interface.h" +#include "api/scoped_refptr.h" +#include "rtc_base/ref_counted_object.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockVideoTrack + : public rtc::RefCountedObject<webrtc::VideoTrackInterface> { + public: + static rtc::scoped_refptr<MockVideoTrack> Create() { + return rtc::scoped_refptr<MockVideoTrack>(new MockVideoTrack()); + } + + // NotifierInterface + MOCK_METHOD(void, + RegisterObserver, + (ObserverInterface * observer), + (override)); + MOCK_METHOD(void, + UnregisterObserver, + (ObserverInterface * observer), + (override)); + + // MediaStreamTrackInterface + MOCK_METHOD(std::string, kind, (), (const, override)); + MOCK_METHOD(std::string, id, (), (const, override)); + MOCK_METHOD(bool, enabled, (), (const, override)); + MOCK_METHOD(bool, set_enabled, (bool enable), (override)); + MOCK_METHOD(TrackState, state, (), (const, override)); + + // VideoSourceInterface + MOCK_METHOD(void, + AddOrUpdateSink, + (rtc::VideoSinkInterface<VideoFrame> * sink, + const rtc::VideoSinkWants& wants), + (override)); + // RemoveSink must guarantee that at the time the method returns, + // there is no current and no future calls to VideoSinkInterface::OnFrame. + MOCK_METHOD(void, + RemoveSink, + (rtc::VideoSinkInterface<VideoFrame> * sink), + (override)); + + // VideoTrackInterface + MOCK_METHOD(VideoTrackSourceInterface*, GetSource, (), (const, override)); + + MOCK_METHOD(ContentHint, content_hint, (), (const, override)); + MOCK_METHOD(void, set_content_hint, (ContentHint hint), (override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_VIDEO_TRACK_H_ diff --git a/third_party/libwebrtc/api/test/neteq_simulator.cc b/third_party/libwebrtc/api/test/neteq_simulator.cc new file mode 100644 index 0000000000..980db96d32 --- /dev/null +++ b/third_party/libwebrtc/api/test/neteq_simulator.cc @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/neteq_simulator.h" + +namespace webrtc { +namespace test { + +NetEqSimulator::SimulationStepResult::SimulationStepResult() = default; +NetEqSimulator::SimulationStepResult::SimulationStepResult( + const NetEqSimulator::SimulationStepResult& other) = default; +NetEqSimulator::SimulationStepResult::~SimulationStepResult() = default; + +NetEqSimulator::NetEqState::NetEqState() = default; +NetEqSimulator::NetEqState::NetEqState(const NetEqState& other) = default; +NetEqSimulator::NetEqState::~NetEqState() = default; + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/neteq_simulator.h b/third_party/libwebrtc/api/test/neteq_simulator.h new file mode 100644 index 0000000000..88c7ffa681 --- /dev/null +++ b/third_party/libwebrtc/api/test/neteq_simulator.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_NETEQ_SIMULATOR_H_ +#define API_TEST_NETEQ_SIMULATOR_H_ + +#include <stdint.h> + +#include <map> +#include <vector> + +namespace webrtc { +namespace test { + +class NetEqSimulator { + public: + virtual ~NetEqSimulator() = default; + + enum class Action { kNormal, kExpand, kAccelerate, kPreemptiveExpand }; + + // The results of one simulation step. + struct SimulationStepResult { + SimulationStepResult(); + SimulationStepResult(const SimulationStepResult& other); + ~SimulationStepResult(); + + bool is_simulation_finished = false; + // The amount of audio produced (in ms) with the actions in this time step. + std::map<Action, int> action_times_ms; + // The amount of wall clock time (in ms) that elapsed since the previous + // event. This is not necessarily equal to the sum of the values in + // action_times_ms. + int64_t simulation_step_ms = 0; + }; + + struct NetEqState { + NetEqState(); + NetEqState(const NetEqState& other); + ~NetEqState(); + // The sum of the packet buffer and sync buffer delay. + int current_delay_ms = 0; + // An indicator that packet loss occurred since the last GetAudio event. + bool packet_loss_occurred = false; + // An indicator that the packet buffer has been flushed since the last + // GetAudio event. + bool packet_buffer_flushed = false; + // Indicates if the next needed packet is available in the buffer. + bool next_packet_available = false; + // The inter-arrival times in ms of the packets that have arrived since the + // last GetAudio event. + std::vector<int> packet_iat_ms; + // The current packet size in ms. + int packet_size_ms = 0; + }; + + // Runs the simulation until the end. Returns the duration of the produced + // audio in ms. + virtual int64_t Run() = 0; + // Runs the simulation until we hit the next GetAudio event. If the simulation + // is finished, is_simulation_finished will be set to true in the returned + // SimulationStepResult. + virtual SimulationStepResult RunToNextGetAudio() = 0; + + // Set the next action to be taken by NetEq. This will override any action + // that NetEq would normally decide to take. + virtual void SetNextAction(Action next_operation) = 0; + + // Get the current state of NetEq. + virtual NetEqState GetNetEqState() = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_NETEQ_SIMULATOR_H_ diff --git a/third_party/libwebrtc/api/test/neteq_simulator_factory.cc b/third_party/libwebrtc/api/test/neteq_simulator_factory.cc new file mode 100644 index 0000000000..82b27e546d --- /dev/null +++ b/third_party/libwebrtc/api/test/neteq_simulator_factory.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/neteq_simulator_factory.h" + +#include <memory> +#include <string> +#include <vector> + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "modules/audio_coding/neteq/tools/neteq_test_factory.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { +namespace { +NetEqTestFactory::Config convertConfig( + const NetEqSimulatorFactory::Config& simulation_config, + absl::string_view replacement_audio_filename) { + NetEqTestFactory::Config config; + config.replacement_audio_file = std::string(replacement_audio_filename); + config.max_nr_packets_in_buffer = simulation_config.max_nr_packets_in_buffer; + config.initial_dummy_packets = simulation_config.initial_dummy_packets; + config.skip_get_audio_events = simulation_config.skip_get_audio_events; + config.field_trial_string = simulation_config.field_trial_string; + config.output_audio_filename = simulation_config.output_audio_filename; + config.pythonplot = simulation_config.python_plot_filename.has_value(); + config.plot_scripts_basename = simulation_config.python_plot_filename; + config.textlog = simulation_config.text_log_filename.has_value(); + config.textlog_filename = simulation_config.text_log_filename; + return config; +} +} // namespace + +NetEqSimulatorFactory::NetEqSimulatorFactory() + : factory_(std::make_unique<NetEqTestFactory>()) {} + +NetEqSimulatorFactory::~NetEqSimulatorFactory() = default; + +std::unique_ptr<NetEqSimulator> NetEqSimulatorFactory::CreateSimulatorFromFile( + absl::string_view event_log_filename, + absl::string_view replacement_audio_filename, + Config simulation_config) { + NetEqTestFactory::Config config = + convertConfig(simulation_config, replacement_audio_filename); + return factory_->InitializeTestFromFile( + std::string(event_log_filename), simulation_config.neteq_factory, config); +} + +std::unique_ptr<NetEqSimulator> +NetEqSimulatorFactory::CreateSimulatorFromString( + absl::string_view event_log_file_contents, + absl::string_view replacement_audio_filename, + Config simulation_config) { + NetEqTestFactory::Config config = + convertConfig(simulation_config, replacement_audio_filename); + return factory_->InitializeTestFromString( + std::string(event_log_file_contents), simulation_config.neteq_factory, + config); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/neteq_simulator_factory.h b/third_party/libwebrtc/api/test/neteq_simulator_factory.h new file mode 100644 index 0000000000..2a716e665e --- /dev/null +++ b/third_party/libwebrtc/api/test/neteq_simulator_factory.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_NETEQ_SIMULATOR_FACTORY_H_ +#define API_TEST_NETEQ_SIMULATOR_FACTORY_H_ + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/neteq/neteq_factory.h" +#include "api/test/neteq_simulator.h" + +namespace webrtc { +namespace test { + +class NetEqTestFactory; + +class NetEqSimulatorFactory { + public: + NetEqSimulatorFactory(); + ~NetEqSimulatorFactory(); + struct Config { + // The maximum allowed number of packets in the jitter buffer. + int max_nr_packets_in_buffer = 0; + // The number of audio packets to insert at the start of the simulation. + // Since the simulation is done with a replacement audio file, these + // artificial packets will take a small piece of that replacement audio. + int initial_dummy_packets = 0; + // The number of simulation steps to skip at the start of the simulation. + // This removes incoming packets and GetAudio events from the start of the + // simulation, until the requested number of GetAudio events has been + // removed. + int skip_get_audio_events = 0; + // A WebRTC field trial string to be used during the simulation. + std::string field_trial_string; + // A filename for the generated output audio file. + absl::optional<std::string> output_audio_filename; + // A filename for the python plot. + absl::optional<std::string> python_plot_filename; + // A filename for the text log. + absl::optional<std::string> text_log_filename; + // A custom NetEqFactory can be used. + NetEqFactory* neteq_factory = nullptr; + }; + std::unique_ptr<NetEqSimulator> CreateSimulatorFromFile( + absl::string_view event_log_filename, + absl::string_view replacement_audio_filename, + Config simulation_config); + // The same as above, but pass the file contents as a string. + std::unique_ptr<NetEqSimulator> CreateSimulatorFromString( + absl::string_view event_log_file_contents, + absl::string_view replacement_audio_file, + Config simulation_config); + + private: + std::unique_ptr<NetEqTestFactory> factory_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_NETEQ_SIMULATOR_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/network_emulation/BUILD.gn b/third_party/libwebrtc/api/test/network_emulation/BUILD.gn new file mode 100644 index 0000000000..d009d39a21 --- /dev/null +++ b/third_party/libwebrtc/api/test/network_emulation/BUILD.gn @@ -0,0 +1,52 @@ +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_library("network_emulation") { + visibility = [ "*" ] + + sources = [ + "cross_traffic.h", + "network_emulation_interfaces.cc", + "network_emulation_interfaces.h", + ] + + deps = [ + "../..:array_view", + "../../../rtc_base:checks", + "../../../rtc_base:copy_on_write_buffer", + "../../../rtc_base:ip_address", + "../../../rtc_base:net_helper", + "../../../rtc_base:socket_address", + "../../numerics", + "../../task_queue", + "../../units:data_rate", + "../../units:data_size", + "../../units:time_delta", + "../../units:timestamp", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("create_cross_traffic") { + visibility = [ "*" ] + testonly = true + + sources = [ + "create_cross_traffic.cc", + "create_cross_traffic.h", + ] + + deps = [ + ":network_emulation", + "../..:network_emulation_manager_api", + "../../../rtc_base/task_utils:repeating_task", + "../../../test/network:emulated_network", + ] +} diff --git a/third_party/libwebrtc/api/test/network_emulation/DEPS b/third_party/libwebrtc/api/test/network_emulation/DEPS new file mode 100644 index 0000000000..0cf128849d --- /dev/null +++ b/third_party/libwebrtc/api/test/network_emulation/DEPS @@ -0,0 +1,7 @@ +specific_include_rules = { + ".*": [ + "+rtc_base/socket_address.h", + "+rtc_base/ip_address.h", + "+rtc_base/copy_on_write_buffer.h", + ], +} diff --git a/third_party/libwebrtc/api/test/network_emulation/create_cross_traffic.cc b/third_party/libwebrtc/api/test/network_emulation/create_cross_traffic.cc new file mode 100644 index 0000000000..36a535cec6 --- /dev/null +++ b/third_party/libwebrtc/api/test/network_emulation/create_cross_traffic.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/network_emulation/create_cross_traffic.h" + +#include <memory> + +#include "rtc_base/task_utils/repeating_task.h" +#include "test/network/cross_traffic.h" + +namespace webrtc { + +std::unique_ptr<CrossTrafficGenerator> CreateRandomWalkCrossTraffic( + CrossTrafficRoute* traffic_route, + RandomWalkConfig config) { + return std::make_unique<test::RandomWalkCrossTraffic>(config, traffic_route); +} + +std::unique_ptr<CrossTrafficGenerator> CreatePulsedPeaksCrossTraffic( + CrossTrafficRoute* traffic_route, + PulsedPeaksConfig config) { + return std::make_unique<test::PulsedPeaksCrossTraffic>(config, traffic_route); +} + +std::unique_ptr<CrossTrafficGenerator> CreateFakeTcpCrossTraffic( + EmulatedRoute* send_route, + EmulatedRoute* ret_route, + FakeTcpConfig config) { + return std::make_unique<test::FakeTcpCrossTraffic>(config, send_route, + ret_route); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/network_emulation/create_cross_traffic.h b/third_party/libwebrtc/api/test/network_emulation/create_cross_traffic.h new file mode 100644 index 0000000000..42fc855392 --- /dev/null +++ b/third_party/libwebrtc/api/test/network_emulation/create_cross_traffic.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_NETWORK_EMULATION_CREATE_CROSS_TRAFFIC_H_ +#define API_TEST_NETWORK_EMULATION_CREATE_CROSS_TRAFFIC_H_ + +#include <memory> + +#include "api/test/network_emulation/cross_traffic.h" +#include "api/test/network_emulation_manager.h" + +namespace webrtc { + +// This API is still in development and can be changed without prior notice. + +std::unique_ptr<CrossTrafficGenerator> CreateRandomWalkCrossTraffic( + CrossTrafficRoute* traffic_route, + RandomWalkConfig config); + +std::unique_ptr<CrossTrafficGenerator> CreatePulsedPeaksCrossTraffic( + CrossTrafficRoute* traffic_route, + PulsedPeaksConfig config); + +std::unique_ptr<CrossTrafficGenerator> CreateFakeTcpCrossTraffic( + EmulatedRoute* send_route, + EmulatedRoute* ret_route, + FakeTcpConfig config); + +} // namespace webrtc + +#endif // API_TEST_NETWORK_EMULATION_CREATE_CROSS_TRAFFIC_H_ diff --git a/third_party/libwebrtc/api/test/network_emulation/cross_traffic.h b/third_party/libwebrtc/api/test/network_emulation/cross_traffic.h new file mode 100644 index 0000000000..737a93c2fd --- /dev/null +++ b/third_party/libwebrtc/api/test/network_emulation/cross_traffic.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_NETWORK_EMULATION_CROSS_TRAFFIC_H_ +#define API_TEST_NETWORK_EMULATION_CROSS_TRAFFIC_H_ + +#include "api/task_queue/task_queue_base.h" +#include "api/test/network_emulation/network_emulation_interfaces.h" +#include "api/units/data_rate.h" +#include "api/units/data_size.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" + +namespace webrtc { + +// This API is still in development and can be changed without prior notice. + +// Represents the endpoint for cross traffic that is going through the network. +// It can be used to emulate unexpected network load. +class CrossTrafficRoute { + public: + virtual ~CrossTrafficRoute() = default; + + // Triggers sending of dummy packets with size `packet_size` bytes. + virtual void TriggerPacketBurst(size_t num_packets, size_t packet_size) = 0; + // Sends a packet over the nodes. The content of the packet is unspecified; + // only the size metter for the emulation purposes. + virtual void SendPacket(size_t packet_size) = 0; + // Sends a packet over the nodes and runs `action` when it has been delivered. + virtual void NetworkDelayedAction(size_t packet_size, + std::function<void()> action) = 0; +}; + +// Describes a way of generating cross traffic on some route. Used by +// NetworkEmulationManager to produce cross traffic during some period of time. +class CrossTrafficGenerator { + public: + virtual ~CrossTrafficGenerator() = default; + + // Time between Process calls. + virtual TimeDelta GetProcessInterval() const = 0; + + // Called periodically by NetworkEmulationManager. Generates traffic on the + // route. + virtual void Process(Timestamp at_time) = 0; +}; + +// Config of a cross traffic generator. Generated traffic rises and falls +// randomly. +struct RandomWalkConfig { + int random_seed = 1; + DataRate peak_rate = DataRate::KilobitsPerSec(100); + DataSize min_packet_size = DataSize::Bytes(200); + TimeDelta min_packet_interval = TimeDelta::Millis(1); + TimeDelta update_interval = TimeDelta::Millis(200); + double variance = 0.6; + double bias = -0.1; +}; + +// Config of a cross traffic generator. Generated traffic has form of periodic +// peaks alternating with periods of silence. +struct PulsedPeaksConfig { + DataRate peak_rate = DataRate::KilobitsPerSec(100); + DataSize min_packet_size = DataSize::Bytes(200); + TimeDelta min_packet_interval = TimeDelta::Millis(1); + TimeDelta send_duration = TimeDelta::Millis(100); + TimeDelta hold_duration = TimeDelta::Millis(2000); +}; + +struct FakeTcpConfig { + DataSize packet_size = DataSize::Bytes(1200); + DataSize send_limit = DataSize::PlusInfinity(); + TimeDelta process_interval = TimeDelta::Millis(200); + TimeDelta packet_timeout = TimeDelta::Seconds(1); +}; + +} // namespace webrtc + +#endif // API_TEST_NETWORK_EMULATION_CROSS_TRAFFIC_H_ diff --git a/third_party/libwebrtc/api/test/network_emulation/network_emulation_interfaces.cc b/third_party/libwebrtc/api/test/network_emulation/network_emulation_interfaces.cc new file mode 100644 index 0000000000..0f3a7f8ffd --- /dev/null +++ b/third_party/libwebrtc/api/test/network_emulation/network_emulation_interfaces.cc @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/network_emulation/network_emulation_interfaces.h" + +#include "rtc_base/net_helper.h" + +namespace webrtc { + +EmulatedIpPacket::EmulatedIpPacket(const rtc::SocketAddress& from, + const rtc::SocketAddress& to, + rtc::CopyOnWriteBuffer data, + Timestamp arrival_time, + uint16_t application_overhead) + : from(from), + to(to), + data(data), + headers_size(to.ipaddr().overhead() + application_overhead + + cricket::kUdpHeaderSize), + arrival_time(arrival_time) { + RTC_DCHECK(to.family() == AF_INET || to.family() == AF_INET6); +} + +DataRate EmulatedNetworkOutgoingStats::AverageSendRate() const { + RTC_DCHECK_GE(packets_sent, 2); + RTC_DCHECK(first_packet_sent_time.IsFinite()); + RTC_DCHECK(last_packet_sent_time.IsFinite()); + return (bytes_sent - first_sent_packet_size) / + (last_packet_sent_time - first_packet_sent_time); +} + +DataRate EmulatedNetworkIncomingStats::AverageReceiveRate() const { + RTC_DCHECK_GE(packets_received, 2); + RTC_DCHECK(first_packet_received_time.IsFinite()); + RTC_DCHECK(last_packet_received_time.IsFinite()); + return (bytes_received - first_received_packet_size) / + (last_packet_received_time - first_packet_received_time); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/network_emulation/network_emulation_interfaces.h b/third_party/libwebrtc/api/test/network_emulation/network_emulation_interfaces.h new file mode 100644 index 0000000000..7cab07b75d --- /dev/null +++ b/third_party/libwebrtc/api/test/network_emulation/network_emulation_interfaces.h @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_NETWORK_EMULATION_NETWORK_EMULATION_INTERFACES_H_ +#define API_TEST_NETWORK_EMULATION_NETWORK_EMULATION_INTERFACES_H_ + +#include <map> +#include <memory> +#include <vector> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/units/data_rate.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/ip_address.h" +#include "rtc_base/socket_address.h" + +namespace webrtc { + +struct EmulatedIpPacket { + public: + EmulatedIpPacket(const rtc::SocketAddress& from, + const rtc::SocketAddress& to, + rtc::CopyOnWriteBuffer data, + Timestamp arrival_time, + uint16_t application_overhead = 0); + ~EmulatedIpPacket() = default; + // This object is not copyable or assignable. + EmulatedIpPacket(const EmulatedIpPacket&) = delete; + EmulatedIpPacket& operator=(const EmulatedIpPacket&) = delete; + // This object is only moveable. + EmulatedIpPacket(EmulatedIpPacket&&) = default; + EmulatedIpPacket& operator=(EmulatedIpPacket&&) = default; + + size_t size() const { return data.size(); } + const uint8_t* cdata() const { return data.cdata(); } + + size_t ip_packet_size() const { return size() + headers_size; } + rtc::SocketAddress from; + rtc::SocketAddress to; + // Holds the UDP payload. + rtc::CopyOnWriteBuffer data; + uint16_t headers_size; + Timestamp arrival_time; +}; + +// Interface for handling IP packets from an emulated network. This is used with +// EmulatedEndpoint to receive packets on a specific port. +class EmulatedNetworkReceiverInterface { + public: + virtual ~EmulatedNetworkReceiverInterface() = default; + + virtual void OnPacketReceived(EmulatedIpPacket packet) = 0; +}; + +struct EmulatedNetworkOutgoingStats { + int64_t packets_sent = 0; + + DataSize bytes_sent = DataSize::Zero(); + + // Sizes of all sent packets. + // Collected iff EmulatedNetworkStatsGatheringMode::kDebug is enabled. + SamplesStatsCounter sent_packets_size; + + DataSize first_sent_packet_size = DataSize::Zero(); + + // Time of the first packet sent or infinite value if no packets were sent. + Timestamp first_packet_sent_time = Timestamp::PlusInfinity(); + + // Time of the last packet sent or infinite value if no packets were sent. + Timestamp last_packet_sent_time = Timestamp::MinusInfinity(); + + // Returns average send rate. Requires that at least 2 packets were sent. + DataRate AverageSendRate() const; +}; + +struct EmulatedNetworkIncomingStats { + // Total amount of packets received with or without destination. + int64_t packets_received = 0; + + // Total amount of bytes in received packets. + DataSize bytes_received = DataSize::Zero(); + + // Sizes of all received packets. + // Collected iff EmulatedNetworkStatsGatheringMode::kDebug is enabled. + SamplesStatsCounter received_packets_size; + + // Total amount of packets that were received, but no destination was found. + int64_t packets_discarded_no_receiver = 0; + + // Total amount of bytes in discarded packets. + DataSize bytes_discarded_no_receiver = DataSize::Zero(); + + // Sizes of all packets that were received, but no destination was found. + // Collected iff EmulatedNetworkStatsGatheringMode::kDebug is enabled. + SamplesStatsCounter packets_discarded_no_receiver_size; + + DataSize first_received_packet_size = DataSize::Zero(); + + // Time of the first packet received or infinite value if no packets were + // received. + Timestamp first_packet_received_time = Timestamp::PlusInfinity(); + + // Time of the last packet received or infinite value if no packets were + // received. + Timestamp last_packet_received_time = Timestamp::MinusInfinity(); + + DataRate AverageReceiveRate() const; +}; + +struct EmulatedNetworkStats { + int64_t PacketsSent() const { return overall_outgoing_stats.packets_sent; } + + DataSize BytesSent() const { return overall_outgoing_stats.bytes_sent; } + + // Returns the timestamped sizes of all sent packets. + // Returned reference is valid until the next call to a non-const method. + // Collected iff EmulatedNetworkStatsGatheringMode::kDebug is enabled. + const SamplesStatsCounter& SentPacketsSizeCounter() const { + return overall_outgoing_stats.sent_packets_size; + } + + DataSize FirstSentPacketSize() const { + return overall_outgoing_stats.first_sent_packet_size; + } + + // Returns time of the first packet sent or infinite value if no packets were + // sent. + Timestamp FirstPacketSentTime() const { + return overall_outgoing_stats.first_packet_sent_time; + } + + // Returns time of the last packet sent or infinite value if no packets were + // sent. + Timestamp LastPacketSentTime() const { + return overall_outgoing_stats.last_packet_sent_time; + } + + DataRate AverageSendRate() const { + return overall_outgoing_stats.AverageSendRate(); + } + + // Total amount of packets received regardless of the destination address. + int64_t PacketsReceived() const { + return overall_incoming_stats.packets_received; + } + + // Total amount of bytes in received packets. + DataSize BytesReceived() const { + return overall_incoming_stats.bytes_received; + } + + // Returns the timestamped sizes of all received packets. + // Returned reference is valid until the next call to a non-const method. + // Collected iff EmulatedNetworkStatsGatheringMode::kDebug is enabled. + const SamplesStatsCounter& ReceivedPacketsSizeCounter() const { + return overall_incoming_stats.received_packets_size; + } + + // Total amount of packets that were received, but no destination was found. + int64_t PacketsDiscardedNoReceiver() const { + return overall_incoming_stats.packets_discarded_no_receiver; + } + + // Total amount of bytes in dropped packets. + DataSize BytesDiscardedNoReceiver() const { + return overall_incoming_stats.bytes_discarded_no_receiver; + } + + // Returns counter with timestamped sizes of all packets that were received, + // but no destination was found. + // Returned reference is valid until the next call to a non-const method. + // Collected iff EmulatedNetworkStatsGatheringMode::kDebug is enabled. + const SamplesStatsCounter& PacketsDiscardedNoReceiverSizeCounter() const { + return overall_incoming_stats.packets_discarded_no_receiver_size; + } + + DataSize FirstReceivedPacketSize() const { + return overall_incoming_stats.first_received_packet_size; + } + + // Returns time of the first packet received or infinite value if no packets + // were received. + Timestamp FirstPacketReceivedTime() const { + return overall_incoming_stats.first_packet_received_time; + } + + // Returns time of the last packet received or infinite value if no packets + // were received. + Timestamp LastPacketReceivedTime() const { + return overall_incoming_stats.last_packet_received_time; + } + + DataRate AverageReceiveRate() const { + return overall_incoming_stats.AverageReceiveRate(); + } + + // List of IP addresses that were used to send data considered in this stats + // object. + std::vector<rtc::IPAddress> local_addresses; + + // Overall outgoing stats for all IP addresses which were requested. + EmulatedNetworkOutgoingStats overall_outgoing_stats; + + // Overall incoming stats for all IP addresses from which data was received + // on requested interfaces. + EmulatedNetworkIncomingStats overall_incoming_stats; + + std::map<rtc::IPAddress, EmulatedNetworkOutgoingStats> + outgoing_stats_per_destination; + std::map<rtc::IPAddress, EmulatedNetworkIncomingStats> + incoming_stats_per_source; + + // Duration between packet was received on network interface and was + // dispatched to the network in microseconds. + // Collected iff EmulatedNetworkStatsGatheringMode::kDebug is enabled. + SamplesStatsCounter sent_packets_queue_wait_time_us; +}; + +struct EmulatedNetworkNodeStats { + // Amount of time each packet spent in the emulated network node for which + // stats were collected. + // + // Collected iff EmulatedNetworkStatsGatheringMode::kDebug is enabled. + SamplesStatsCounter packet_transport_time; + + // For each packet contains its size divided on the amount of time which it + // spent in the emulated network node for which stats were collected. + // + // Collected iff EmulatedNetworkStatsGatheringMode::kDebug is enabled. + SamplesStatsCounter size_to_packet_transport_time; +}; + +// EmulatedEndpoint is an abstraction for network interface on device. Instances +// of this are created by NetworkEmulationManager::CreateEndpoint and +// thread safe. +class EmulatedEndpoint : public EmulatedNetworkReceiverInterface { + public: + // Send packet into network. + // `from` will be used to set source address for the packet in destination + // socket. + // `to` will be used for routing verification and picking right socket by port + // on destination endpoint. + virtual void SendPacket(const rtc::SocketAddress& from, + const rtc::SocketAddress& to, + rtc::CopyOnWriteBuffer packet_data, + uint16_t application_overhead = 0) = 0; + + // Binds receiver to this endpoint to send and receive data. + // `desired_port` is a port that should be used. If it is equal to 0, + // endpoint will pick the first available port starting from + // `kFirstEphemeralPort`. + // + // Returns the port, that should be used (it will be equals to desired, if + // `desired_port` != 0 and is free or will be the one, selected by endpoint) + // or absl::nullopt if desired_port in used. Also fails if there are no more + // free ports to bind to. + // + // The Bind- and Unbind-methods must not be called from within a bound + // receiver's OnPacketReceived method. + virtual absl::optional<uint16_t> BindReceiver( + uint16_t desired_port, + EmulatedNetworkReceiverInterface* receiver) = 0; + // Unbinds receiver from the specified port. Do nothing if no receiver was + // bound before. After this method returns, no more packets can be delivered + // to the receiver, and it is safe to destroy it. + virtual void UnbindReceiver(uint16_t port) = 0; + // Binds receiver that will accept all packets which arrived on any port + // for which there are no bound receiver. + virtual void BindDefaultReceiver( + EmulatedNetworkReceiverInterface* receiver) = 0; + // Unbinds default receiver. Do nothing if no default receiver was bound + // before. + virtual void UnbindDefaultReceiver() = 0; + virtual rtc::IPAddress GetPeerLocalAddress() const = 0; + + private: + // Ensure that there can be no other subclass than EmulatedEndpointImpl. This + // means that it's always safe to downcast EmulatedEndpoint instances to + // EmulatedEndpointImpl. + friend class EmulatedEndpointImpl; + EmulatedEndpoint() = default; +}; + +// Simulates a TCP connection, this roughly implements the Reno algorithm. In +// difference from TCP this only support sending messages with a fixed length, +// no streaming. This is useful to simulate signaling and cross traffic using +// message based protocols such as HTTP. It differs from UDP messages in that +// they are guranteed to be delivered eventually, even on lossy networks. +class TcpMessageRoute { + public: + // Sends a TCP message of the given `size` over the route, `on_received` is + // called when the message has been delivered. Note that the connection + // parameters are reset iff there's no currently pending message on the route. + virtual void SendMessage(size_t size, std::function<void()> on_received) = 0; + + protected: + ~TcpMessageRoute() = default; +}; +} // namespace webrtc + +#endif // API_TEST_NETWORK_EMULATION_NETWORK_EMULATION_INTERFACES_H_ diff --git a/third_party/libwebrtc/api/test/network_emulation_manager.cc b/third_party/libwebrtc/api/test/network_emulation_manager.cc new file mode 100644 index 0000000000..236e2f0e17 --- /dev/null +++ b/third_party/libwebrtc/api/test/network_emulation_manager.cc @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/network_emulation_manager.h" + +#include <utility> + +#include "call/simulated_network.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +bool AbslParseFlag(absl::string_view text, TimeMode* mode, std::string* error) { + if (text == "realtime") { + *mode = TimeMode::kRealTime; + return true; + } + if (text == "simulated") { + *mode = TimeMode::kSimulated; + return true; + } + *error = + "Unknown value for TimeMode enum. Options are 'realtime' or 'simulated'"; + return false; +} + +std::string AbslUnparseFlag(TimeMode mode) { + switch (mode) { + case TimeMode::kRealTime: + return "realtime"; + case TimeMode::kSimulated: + return "simulated"; + } + RTC_CHECK_NOTREACHED(); + return "unknown"; +} + +NetworkEmulationManager::SimulatedNetworkNode::Builder& +NetworkEmulationManager::SimulatedNetworkNode::Builder::config( + BuiltInNetworkBehaviorConfig config) { + config_ = config; + return *this; +} + +NetworkEmulationManager::SimulatedNetworkNode::Builder& +NetworkEmulationManager::SimulatedNetworkNode::Builder::delay_ms( + int queue_delay_ms) { + config_.queue_delay_ms = queue_delay_ms; + return *this; +} + +NetworkEmulationManager::SimulatedNetworkNode::Builder& +NetworkEmulationManager::SimulatedNetworkNode::Builder::capacity_kbps( + int link_capacity_kbps) { + config_.link_capacity_kbps = link_capacity_kbps; + return *this; +} + +NetworkEmulationManager::SimulatedNetworkNode::Builder& +NetworkEmulationManager::SimulatedNetworkNode::Builder::capacity_Mbps( + int link_capacity_Mbps) { + config_.link_capacity_kbps = link_capacity_Mbps * 1000; + return *this; +} + +NetworkEmulationManager::SimulatedNetworkNode::Builder& +NetworkEmulationManager::SimulatedNetworkNode::Builder::loss(double loss_rate) { + config_.loss_percent = std::round(loss_rate * 100); + return *this; +} + +NetworkEmulationManager::SimulatedNetworkNode::Builder& +NetworkEmulationManager::SimulatedNetworkNode::Builder::packet_queue_length( + int max_queue_length_in_packets) { + config_.queue_length_packets = max_queue_length_in_packets; + return *this; +} + +NetworkEmulationManager::SimulatedNetworkNode +NetworkEmulationManager::SimulatedNetworkNode::Builder::Build( + uint64_t random_seed) const { + RTC_CHECK(net_); + return Build(net_, random_seed); +} + +NetworkEmulationManager::SimulatedNetworkNode +NetworkEmulationManager::SimulatedNetworkNode::Builder::Build( + NetworkEmulationManager* net, + uint64_t random_seed) const { + RTC_CHECK(net); + RTC_CHECK(net_ == nullptr || net_ == net); + SimulatedNetworkNode res; + auto behavior = std::make_unique<SimulatedNetwork>(config_, random_seed); + res.simulation = behavior.get(); + res.node = net->CreateEmulatedNode(std::move(behavior)); + return res; +} + +std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> +NetworkEmulationManager::CreateEndpointPairWithTwoWayRoutes( + const BuiltInNetworkBehaviorConfig& config) { + auto* alice_node = CreateEmulatedNode(config); + auto* bob_node = CreateEmulatedNode(config); + + auto* alice_endpoint = CreateEndpoint(EmulatedEndpointConfig()); + auto* bob_endpoint = CreateEndpoint(EmulatedEndpointConfig()); + + CreateRoute(alice_endpoint, {alice_node}, bob_endpoint); + CreateRoute(bob_endpoint, {bob_node}, alice_endpoint); + + return { + CreateEmulatedNetworkManagerInterface({alice_endpoint}), + CreateEmulatedNetworkManagerInterface({bob_endpoint}), + }; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/network_emulation_manager.h b/third_party/libwebrtc/api/test/network_emulation_manager.h new file mode 100644 index 0000000000..bc9279d306 --- /dev/null +++ b/third_party/libwebrtc/api/test/network_emulation_manager.h @@ -0,0 +1,357 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_NETWORK_EMULATION_MANAGER_H_ +#define API_TEST_NETWORK_EMULATION_MANAGER_H_ + +#include <functional> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "api/array_view.h" +#include "api/packet_socket_factory.h" +#include "api/test/network_emulation/cross_traffic.h" +#include "api/test/network_emulation/network_emulation_interfaces.h" +#include "api/test/peer_network_dependencies.h" +#include "api/test/simulated_network.h" +#include "api/test/time_controller.h" +#include "api/units/timestamp.h" +#include "rtc_base/network.h" +#include "rtc_base/network_constants.h" +#include "rtc_base/thread.h" + +namespace webrtc { + +// This API is still in development and can be changed without prior notice. + +// These classes are forward declared here, because they used as handles, to +// make it possible for client code to operate with these abstractions and build +// required network configuration. With forward declaration here implementation +// is more readable, than with interfaces approach and cause user needn't any +// API methods on these abstractions it is acceptable here. + +// EmulatedNetworkNode is an abstraction for some network in the real world, +// like 3G network between peers, or Wi-Fi for one peer and LTE for another. +// Multiple networks can be joined into chain emulating a network path from +// one peer to another. +class EmulatedNetworkNode; + +// EmulatedRoute is handle for single route from one network interface on one +// peer device to another network interface on another peer device. +class EmulatedRoute; + +enum class EmulatedNetworkStatsGatheringMode { + // Gather main network stats counters. See more details on which particular + // metrics are collected in the `EmulatedNetworkStats` and + // `EmulatedNetworkNodeStats` documentation. + kDefault, + // kDefault + also gather per packet statistics. In this mode more memory + // will be used. + kDebug +}; + +struct EmulatedEndpointConfig { + enum class IpAddressFamily { kIpv4, kIpv6 }; + + // If specified will be used to name endpoint for logging purposes. + absl::optional<std::string> name = absl::nullopt; + IpAddressFamily generated_ip_family = IpAddressFamily::kIpv4; + // If specified will be used as IP address for endpoint node. Must be unique + // among all created nodes. + absl::optional<rtc::IPAddress> ip; + // Should endpoint be enabled or not, when it will be created. + // Enabled endpoints will be available for webrtc to send packets. + bool start_as_enabled = true; + // Network type which will be used to represent endpoint to WebRTC. + rtc::AdapterType type = rtc::AdapterType::ADAPTER_TYPE_UNKNOWN; + // Allow endpoint to send packets specifying source IP address different to + // the current endpoint IP address. If false endpoint will crash if attempt + // to send such packet will be done. + bool allow_send_packet_with_different_source_ip = false; + // Allow endpoint to receive packet with destination IP address different to + // the current endpoint IP address. If false endpoint will crash if such + // packet will arrive. + bool allow_receive_packets_with_different_dest_ip = false; +}; + +struct EmulatedTURNServerConfig { + EmulatedEndpointConfig client_config; + EmulatedEndpointConfig peer_config; +}; + +// EmulatedTURNServer is an abstraction for a TURN server. +class EmulatedTURNServerInterface { + public: + struct IceServerConfig { + std::string username; + std::string password; + std::string url; + }; + + virtual ~EmulatedTURNServerInterface() {} + + // Get an IceServer configuration suitable to add to a PeerConnection. + virtual IceServerConfig GetIceServerConfig() const = 0; + + // Get non-null client endpoint, an endpoint that accepts TURN allocations. + // This shall typically be connected to one or more webrtc endpoint. + virtual EmulatedEndpoint* GetClientEndpoint() const = 0; + + // Returns socket address, which client should use to connect to TURN server + // and do TURN allocation. + virtual rtc::SocketAddress GetClientEndpointAddress() const = 0; + + // Get non-null peer endpoint, that is "connected to the internet". + // This shall typically be connected to another TURN server. + virtual EmulatedEndpoint* GetPeerEndpoint() const = 0; +}; + +// Provide interface to obtain all required objects to inject network emulation +// layer into PeerConnection. Also contains information about network interfaces +// accessible by PeerConnection. +class EmulatedNetworkManagerInterface { + public: + virtual ~EmulatedNetworkManagerInterface() = default; + + // Returns non-null pointer to thread that have to be used as network thread + // for WebRTC to properly setup network emulation. Returned thread is owned + // by EmulatedNetworkManagerInterface implementation. + virtual rtc::Thread* network_thread() = 0; + // Returns non-null pointer to network manager that have to be injected into + // WebRTC to properly setup network emulation. Returned manager is owned by + // EmulatedNetworkManagerInterface implementation. + virtual rtc::NetworkManager* network_manager() = 0; + // Returns non-null pointer to packet socket factory that have to be injected + // into WebRTC to properly setup network emulation. Returned factory is owned + // by EmulatedNetworkManagerInterface implementation. + virtual rtc::PacketSocketFactory* packet_socket_factory() = 0; + webrtc::webrtc_pc_e2e::PeerNetworkDependencies network_dependencies() { + return {network_thread(), network_manager(), packet_socket_factory()}; + } + // Returns list of endpoints that are associated with this instance. Pointers + // are guaranteed to be non-null and are owned by NetworkEmulationManager. + virtual std::vector<EmulatedEndpoint*> endpoints() const = 0; + + // Passes summarized network stats for endpoints for this manager into + // specified `stats_callback`. Callback will be executed on network emulation + // internal task queue. + virtual void GetStats( + std::function<void(EmulatedNetworkStats)> stats_callback) const = 0; +}; + +enum class TimeMode { kRealTime, kSimulated }; + +// Called implicitly when parsing an ABSL_FLAG of type TimeMode. +// from the command line flag value `text`. +// Returns `true` and sets `*mode` on success; +// returns `false` and sets `*error` on failure. +bool AbslParseFlag(absl::string_view text, TimeMode* mode, std::string* error); + +// AbslUnparseFlag returns a textual flag value corresponding to the TimeMode +// `mode`. +std::string AbslUnparseFlag(TimeMode mode); + +// Provides an API for creating and configuring emulated network layer. +// All objects returned by this API are owned by NetworkEmulationManager itself +// and will be deleted when manager will be deleted. +class NetworkEmulationManager { + public: + // Helper struct to simplify creation of simulated network behaviors. Contains + // non-owning pointers as the underlying instances are owned by the manager. + struct SimulatedNetworkNode { + SimulatedNetworkInterface* simulation; + EmulatedNetworkNode* node; + + class Builder { + public: + explicit Builder(NetworkEmulationManager* net) : net_(net) {} + Builder() : net_(nullptr) {} + Builder(const Builder&) = default; + // Sets the config state, note that this will replace any previously set + // values. + Builder& config(BuiltInNetworkBehaviorConfig config); + Builder& delay_ms(int queue_delay_ms); + Builder& capacity_kbps(int link_capacity_kbps); + Builder& capacity_Mbps(int link_capacity_Mbps); + Builder& loss(double loss_rate); + Builder& packet_queue_length(int max_queue_length_in_packets); + SimulatedNetworkNode Build(uint64_t random_seed = 1) const; + SimulatedNetworkNode Build(NetworkEmulationManager* net, + uint64_t random_seed = 1) const; + + private: + NetworkEmulationManager* const net_; + BuiltInNetworkBehaviorConfig config_; + }; + }; + virtual ~NetworkEmulationManager() = default; + + virtual TimeController* time_controller() = 0; + // Returns a mode in which underlying time controller operates. + virtual TimeMode time_mode() const = 0; + + // Creates an emulated network node, which represents ideal network with + // unlimited capacity, no delay and no packet loss. + EmulatedNetworkNode* CreateUnconstrainedEmulatedNode() { + return CreateEmulatedNode(BuiltInNetworkBehaviorConfig()); + } + // Creates an emulated network node, which represents single network in + // the emulated network layer. Uses default implementation on network behavior + // which can be configured with `config`. `random_seed` can be provided to + // alter randomization behavior. + virtual EmulatedNetworkNode* CreateEmulatedNode( + BuiltInNetworkBehaviorConfig config, + uint64_t random_seed = 1) = 0; + // Creates an emulated network node, which represents single network in + // the emulated network layer. `network_behavior` determines how created node + // will forward incoming packets to the next receiver. + virtual EmulatedNetworkNode* CreateEmulatedNode( + std::unique_ptr<NetworkBehaviorInterface> network_behavior) = 0; + + virtual SimulatedNetworkNode::Builder NodeBuilder() = 0; + + // Creates an emulated endpoint, which represents single network interface on + // the peer's device. + virtual EmulatedEndpoint* CreateEndpoint(EmulatedEndpointConfig config) = 0; + // Enable emulated endpoint to make it available for webrtc. + // Caller mustn't enable currently enabled endpoint. + virtual void EnableEndpoint(EmulatedEndpoint* endpoint) = 0; + // Disable emulated endpoint to make it unavailable for webrtc. + // Caller mustn't disable currently disabled endpoint. + virtual void DisableEndpoint(EmulatedEndpoint* endpoint) = 0; + + // Creates a route between endpoints going through specified network nodes. + // This route is single direction only and describe how traffic that was + // sent by network interface `from` have to be delivered to the network + // interface `to`. Return object can be used to remove created route. The + // route must contains at least one network node inside it. + // + // Assume that E{0-9} are endpoints and N{0-9} are network nodes, then + // creation of the route have to follow these rules: + // 1. A route consists of a source endpoint, an ordered list of one or + // more network nodes, and a destination endpoint. + // 2. If (E1, ..., E2) is a route, then E1 != E2. + // In other words, the source and the destination may not be the same. + // 3. Given two simultaneously existing routes (E1, ..., E2) and + // (E3, ..., E4), either E1 != E3 or E2 != E4. + // In other words, there may be at most one route from any given source + // endpoint to any given destination endpoint. + // 4. Given two simultaneously existing routes (E1, ..., N1, ..., E2) + // and (E3, ..., N2, ..., E4), either N1 != N2 or E2 != E4. + // In other words, a network node may not belong to two routes that lead + // to the same destination endpoint. + virtual EmulatedRoute* CreateRoute( + EmulatedEndpoint* from, + const std::vector<EmulatedNetworkNode*>& via_nodes, + EmulatedEndpoint* to) = 0; + + // Creates a route over the given `via_nodes` creating the required endpoints + // in the process. The returned EmulatedRoute pointer can be used in other + // calls as a transport route for message or cross traffic. + virtual EmulatedRoute* CreateRoute( + const std::vector<EmulatedNetworkNode*>& via_nodes) = 0; + + // Creates a default route between endpoints going through specified network + // nodes. Default route is used for packet when there is no known route for + // packet's destination IP. + // + // This route is single direction only and describe how traffic that was + // sent by network interface `from` have to be delivered in case if routing + // was unspecified. Return object can be used to remove created route. The + // route must contains at least one network node inside it. + // + // Assume that E{0-9} are endpoints and N{0-9} are network nodes, then + // creation of the route have to follow these rules: + // 1. A route consists of a source endpoint, an ordered list of one or + // more network nodes, and a destination endpoint. + // 2. If (E1, ..., E2) is a route, then E1 != E2. + // In other words, the source and the destination may not be the same. + // 3. Given two simultaneously existing routes (E1, ..., E2) and + // (E3, ..., E4), either E1 != E3 or E2 != E4. + // In other words, there may be at most one route from any given source + // endpoint to any given destination endpoint. + // 4. Given two simultaneously existing routes (E1, ..., N1, ..., E2) + // and (E3, ..., N2, ..., E4), either N1 != N2 or E2 != E4. + // In other words, a network node may not belong to two routes that lead + // to the same destination endpoint. + // 5. Any node N can belong to only one default route. + virtual EmulatedRoute* CreateDefaultRoute( + EmulatedEndpoint* from, + const std::vector<EmulatedNetworkNode*>& via_nodes, + EmulatedEndpoint* to) = 0; + + // Removes route previously created by CreateRoute(...). + // Caller mustn't call this function with route, that have been already + // removed earlier. Removing a route that is currently in use will lead to + // packets being dropped. + virtual void ClearRoute(EmulatedRoute* route) = 0; + + // Creates a simulated TCP connection using `send_route` for traffic and + // `ret_route` for feedback. This can be used to emulate HTTP cross traffic + // and to implement realistic reliable signaling over lossy networks. + // TODO(srte): Handle clearing of the routes involved. + virtual TcpMessageRoute* CreateTcpRoute(EmulatedRoute* send_route, + EmulatedRoute* ret_route) = 0; + + // Creates a route over the given `via_nodes`. Returns an object that can be + // used to emulate network load with cross traffic over the created route. + virtual CrossTrafficRoute* CreateCrossTrafficRoute( + const std::vector<EmulatedNetworkNode*>& via_nodes) = 0; + + // Starts generating cross traffic using given `generator`. Takes ownership + // over the generator. + virtual CrossTrafficGenerator* StartCrossTraffic( + std::unique_ptr<CrossTrafficGenerator> generator) = 0; + + // Stops generating cross traffic that was started using given `generator`. + // The `generator` shouldn't be used after and the reference may be invalid. + virtual void StopCrossTraffic(CrossTrafficGenerator* generator) = 0; + + // Creates EmulatedNetworkManagerInterface which can be used then to inject + // network emulation layer into PeerConnection. `endpoints` - are available + // network interfaces for PeerConnection. If endpoint is enabled, it will be + // immediately available for PeerConnection, otherwise user will be able to + // enable endpoint later to make it available for PeerConnection. + virtual EmulatedNetworkManagerInterface* + CreateEmulatedNetworkManagerInterface( + const std::vector<EmulatedEndpoint*>& endpoints) = 0; + + // Passes combined network stats for all specified `endpoints` into specified + // `stats_callback`. Callback will be executed on network emulation + // internal task queue. + virtual void GetStats( + rtc::ArrayView<EmulatedEndpoint* const> endpoints, + std::function<void(EmulatedNetworkStats)> stats_callback) = 0; + + // Passes combined network stats for all specified `nodes` into specified + // `stats_callback`. Callback will be executed on network emulation + // internal task queue. + virtual void GetStats( + rtc::ArrayView<EmulatedNetworkNode* const> nodes, + std::function<void(EmulatedNetworkNodeStats)> stats_callback) = 0; + + // Create a EmulatedTURNServer. + // The TURN server has 2 endpoints that need to be connected with routes, + // - GetClientEndpoint() - the endpoint that accepts TURN allocations. + // - GetPeerEndpoint() - the endpoint that is "connected to the internet". + virtual EmulatedTURNServerInterface* CreateTURNServer( + EmulatedTURNServerConfig config) = 0; + + // Create a pair of EmulatedNetworkManagerInterfaces connected to each other. + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + CreateEndpointPairWithTwoWayRoutes( + const BuiltInNetworkBehaviorConfig& config); +}; + +} // namespace webrtc + +#endif // API_TEST_NETWORK_EMULATION_MANAGER_H_ diff --git a/third_party/libwebrtc/api/test/pclf/BUILD.gn b/third_party/libwebrtc/api/test/pclf/BUILD.gn new file mode 100644 index 0000000000..0526478e8b --- /dev/null +++ b/third_party/libwebrtc/api/test/pclf/BUILD.gn @@ -0,0 +1,114 @@ +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_source_set("media_configuration") { + visibility = [ "*" ] + testonly = true + sources = [ + "media_configuration.cc", + "media_configuration.h", + ] + + deps = [ + "../..:array_view", + "../..:audio_options_api", + "../..:audio_quality_analyzer_api", + "../..:callfactory_api", + "../..:fec_controller_api", + "../..:frame_generator_api", + "../..:function_view", + "../..:libjingle_peerconnection_api", + "../..:media_stream_interface", + "../..:packet_socket_factory", + "../..:peer_network_dependencies", + "../..:rtp_parameters", + "../..:simulated_network_api", + "../..:stats_observer_interface", + "../..:track_id_stream_info_map", + "../..:video_quality_analyzer_api", + "../../../modules/audio_processing:api", + "../../../rtc_base:checks", + "../../../rtc_base:network", + "../../../rtc_base:rtc_certificate_generator", + "../../../rtc_base:ssl", + "../../../rtc_base:stringutils", + "../../../rtc_base:threading", + "../../../test:fileutils", + "../../../test:video_test_support", + "../../../test/pc/e2e/analyzer/video:video_dumping", + "../../audio:audio_mixer_api", + "../../rtc_event_log", + "../../task_queue", + "../../transport:network_control", + "../../units:time_delta", + "../../video_codecs:video_codecs_api", + "../video:video_frame_writer", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("media_quality_test_params") { + visibility = [ "*" ] + testonly = true + sources = [ "media_quality_test_params.h" ] + + deps = [ + ":media_configuration", + "../../../api:callfactory_api", + "../../../api:fec_controller_api", + "../../../api:field_trials_view", + "../../../api:libjingle_peerconnection_api", + "../../../api:packet_socket_factory", + "../../../api/audio:audio_mixer_api", + "../../../api/rtc_event_log", + "../../../api/task_queue", + "../../../api/transport:network_control", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/audio_processing:api", + "../../../p2p:rtc_p2p", + "../../../rtc_base:network", + "../../../rtc_base:rtc_certificate_generator", + "../../../rtc_base:ssl", + "../../../rtc_base:threading", + ] +} + +rtc_library("peer_configurer") { + visibility = [ "*" ] + testonly = true + sources = [ + "peer_configurer.cc", + "peer_configurer.h", + ] + deps = [ + ":media_configuration", + ":media_quality_test_params", + "../../../api:callfactory_api", + "../../../api:create_peer_connection_quality_test_frame_generator", + "../../../api:fec_controller_api", + "../../../api:packet_socket_factory", + "../../../api:peer_network_dependencies", + "../../../api/audio:audio_mixer_api", + "../../../api/rtc_event_log", + "../../../api/task_queue", + "../../../api/transport:network_control", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/audio_processing:api", + "../../../rtc_base:network", + "../../../rtc_base:rtc_certificate_generator", + "../../../rtc_base:ssl", + "../../../rtc_base:threading", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} diff --git a/third_party/libwebrtc/api/test/pclf/DEPS b/third_party/libwebrtc/api/test/pclf/DEPS new file mode 100644 index 0000000000..60cc0aeeb3 --- /dev/null +++ b/third_party/libwebrtc/api/test/pclf/DEPS @@ -0,0 +1,13 @@ +specific_include_rules = { + ".*": [ + "+modules/audio_processing/include/audio_processing.h", + "+rtc_base/checks.h", + "+rtc_base/network.h", + "+rtc_base/rtc_certificate_generator.h", + "+rtc_base/ssl_certificate.h", + "+rtc_base/thread.h", + ], + "media_quality_test_params\.h": [ + "+p2p/base/port_allocator.h", + ], +} diff --git a/third_party/libwebrtc/api/test/pclf/media_configuration.cc b/third_party/libwebrtc/api/test/pclf/media_configuration.cc new file mode 100644 index 0000000000..56b9e52e01 --- /dev/null +++ b/third_party/libwebrtc/api/test/pclf/media_configuration.cc @@ -0,0 +1,314 @@ +/* + * Copyright 2022 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/pclf/media_configuration.h" + +#include <string> +#include <utility> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/test/video/video_frame_writer.h" +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" +#include "test/pc/e2e/analyzer/video/video_dumping.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +std::string SpecToString(VideoResolution::Spec spec) { + switch (spec) { + case VideoResolution::Spec::kNone: + return "None"; + case VideoResolution::Spec::kMaxFromSender: + return "MaxFromSender"; + } +} + +void AppendResolution(const VideoResolution& resolution, + rtc::StringBuilder& builder) { + builder << "_" << resolution.width() << "x" << resolution.height() << "_" + << resolution.fps(); +} + +} // namespace + +ScreenShareConfig::ScreenShareConfig(TimeDelta slide_change_interval) + : slide_change_interval(slide_change_interval) { + RTC_CHECK_GT(slide_change_interval.ms(), 0); +} +VideoSimulcastConfig::VideoSimulcastConfig(int simulcast_streams_count) + : simulcast_streams_count(simulcast_streams_count) { + RTC_CHECK_GT(simulcast_streams_count, 1); +} +EmulatedSFUConfig::EmulatedSFUConfig(int target_layer_index) + : target_layer_index(target_layer_index) { + RTC_CHECK_GE(target_layer_index, 0); +} + +EmulatedSFUConfig::EmulatedSFUConfig(absl::optional<int> target_layer_index, + absl::optional<int> target_temporal_index) + : target_layer_index(target_layer_index), + target_temporal_index(target_temporal_index) { + RTC_CHECK_GE(target_temporal_index.value_or(0), 0); + if (target_temporal_index) + RTC_CHECK_GE(*target_temporal_index, 0); +} + +VideoResolution::VideoResolution(size_t width, size_t height, int32_t fps) + : width_(width), height_(height), fps_(fps), spec_(Spec::kNone) {} +VideoResolution::VideoResolution(Spec spec) + : width_(0), height_(0), fps_(0), spec_(spec) {} + +bool VideoResolution::operator==(const VideoResolution& other) const { + if (spec_ != Spec::kNone && spec_ == other.spec_) { + // If there is some particular spec set, then it doesn't matter what + // values we have in other fields. + return true; + } + return width_ == other.width_ && height_ == other.height_ && + fps_ == other.fps_ && spec_ == other.spec_; +} +bool VideoResolution::operator!=(const VideoResolution& other) const { + return !(*this == other); +} + +bool VideoResolution::IsRegular() const { + return spec_ == Spec::kNone; +} +std::string VideoResolution::ToString() const { + rtc::StringBuilder out; + out << "{ width=" << width_ << ", height=" << height_ << ", fps=" << fps_ + << ", spec=" << SpecToString(spec_) << " }"; + return out.Release(); +} + +VideoDumpOptions::VideoDumpOptions( + absl::string_view output_directory, + int sampling_modulo, + bool export_frame_ids, + std::function<std::unique_ptr<test::VideoFrameWriter>( + absl::string_view file_name_prefix, + const VideoResolution& resolution)> video_frame_writer_factory) + : output_directory_(output_directory), + sampling_modulo_(sampling_modulo), + export_frame_ids_(export_frame_ids), + video_frame_writer_factory_(video_frame_writer_factory) { + RTC_CHECK_GT(sampling_modulo, 0); +} + +VideoDumpOptions::VideoDumpOptions(absl::string_view output_directory, + bool export_frame_ids) + : VideoDumpOptions(output_directory, + kDefaultSamplingModulo, + export_frame_ids) {} + +std::unique_ptr<test::VideoFrameWriter> +VideoDumpOptions::CreateInputDumpVideoFrameWriter( + absl::string_view stream_label, + const VideoResolution& resolution) const { + std::unique_ptr<test::VideoFrameWriter> writer = video_frame_writer_factory_( + GetInputDumpFileName(stream_label, resolution), resolution); + absl::optional<std::string> frame_ids_file = + GetInputFrameIdsDumpFileName(stream_label, resolution); + if (frame_ids_file.has_value()) { + writer = CreateVideoFrameWithIdsWriter(std::move(writer), *frame_ids_file); + } + return writer; +} + +std::unique_ptr<test::VideoFrameWriter> +VideoDumpOptions::CreateOutputDumpVideoFrameWriter( + absl::string_view stream_label, + absl::string_view receiver, + const VideoResolution& resolution) const { + std::unique_ptr<test::VideoFrameWriter> writer = video_frame_writer_factory_( + GetOutputDumpFileName(stream_label, receiver, resolution), resolution); + absl::optional<std::string> frame_ids_file = + GetOutputFrameIdsDumpFileName(stream_label, receiver, resolution); + if (frame_ids_file.has_value()) { + writer = CreateVideoFrameWithIdsWriter(std::move(writer), *frame_ids_file); + } + return writer; +} + +std::unique_ptr<test::VideoFrameWriter> +VideoDumpOptions::Y4mVideoFrameWriterFactory( + absl::string_view file_name_prefix, + const VideoResolution& resolution) { + return std::make_unique<test::Y4mVideoFrameWriterImpl>( + std::string(file_name_prefix) + ".y4m", resolution.width(), + resolution.height(), resolution.fps()); +} + +std::string VideoDumpOptions::GetInputDumpFileName( + absl::string_view stream_label, + const VideoResolution& resolution) const { + rtc::StringBuilder file_name; + file_name << stream_label; + AppendResolution(resolution, file_name); + return test::JoinFilename(output_directory_, file_name.Release()); +} + +absl::optional<std::string> VideoDumpOptions::GetInputFrameIdsDumpFileName( + absl::string_view stream_label, + const VideoResolution& resolution) const { + if (!export_frame_ids_) { + return absl::nullopt; + } + return GetInputDumpFileName(stream_label, resolution) + ".frame_ids.txt"; +} + +std::string VideoDumpOptions::GetOutputDumpFileName( + absl::string_view stream_label, + absl::string_view receiver, + const VideoResolution& resolution) const { + rtc::StringBuilder file_name; + file_name << stream_label << "_" << receiver; + AppendResolution(resolution, file_name); + return test::JoinFilename(output_directory_, file_name.Release()); +} + +absl::optional<std::string> VideoDumpOptions::GetOutputFrameIdsDumpFileName( + absl::string_view stream_label, + absl::string_view receiver, + const VideoResolution& resolution) const { + if (!export_frame_ids_) { + return absl::nullopt; + } + return GetOutputDumpFileName(stream_label, receiver, resolution) + + ".frame_ids.txt"; +} + +std::string VideoDumpOptions::ToString() const { + rtc::StringBuilder out; + out << "{ output_directory_=" << output_directory_ + << ", sampling_modulo_=" << sampling_modulo_ + << ", export_frame_ids_=" << export_frame_ids_ << " }"; + return out.Release(); +} + +VideoConfig::VideoConfig(const VideoResolution& resolution) + : width(resolution.width()), + height(resolution.height()), + fps(resolution.fps()) { + RTC_CHECK(resolution.IsRegular()); +} +VideoConfig::VideoConfig(size_t width, size_t height, int32_t fps) + : width(width), height(height), fps(fps) {} +VideoConfig::VideoConfig(std::string stream_label, + size_t width, + size_t height, + int32_t fps) + : width(width), + height(height), + fps(fps), + stream_label(std::move(stream_label)) {} + +AudioConfig::AudioConfig(std::string stream_label) + : stream_label(std::move(stream_label)) {} + +VideoCodecConfig::VideoCodecConfig(std::string name) + : name(std::move(name)), required_params() {} +VideoCodecConfig::VideoCodecConfig( + std::string name, + std::map<std::string, std::string> required_params) + : name(std::move(name)), required_params(std::move(required_params)) {} + +absl::optional<VideoResolution> VideoSubscription::GetMaxResolution( + rtc::ArrayView<const VideoConfig> video_configs) { + std::vector<VideoResolution> resolutions; + for (const auto& video_config : video_configs) { + resolutions.push_back(video_config.GetResolution()); + } + return GetMaxResolution(resolutions); +} + +absl::optional<VideoResolution> VideoSubscription::GetMaxResolution( + rtc::ArrayView<const VideoResolution> resolutions) { + if (resolutions.empty()) { + return absl::nullopt; + } + + VideoResolution max_resolution; + for (const VideoResolution& resolution : resolutions) { + if (max_resolution.width() < resolution.width()) { + max_resolution.set_width(resolution.width()); + } + if (max_resolution.height() < resolution.height()) { + max_resolution.set_height(resolution.height()); + } + if (max_resolution.fps() < resolution.fps()) { + max_resolution.set_fps(resolution.fps()); + } + } + return max_resolution; +} + +bool VideoSubscription::operator==(const VideoSubscription& other) const { + return default_resolution_ == other.default_resolution_ && + peers_resolution_ == other.peers_resolution_; +} +bool VideoSubscription::operator!=(const VideoSubscription& other) const { + return !(*this == other); +} + +VideoSubscription& VideoSubscription::SubscribeToPeer( + absl::string_view peer_name, + VideoResolution resolution) { + peers_resolution_[std::string(peer_name)] = resolution; + return *this; +} + +VideoSubscription& VideoSubscription::SubscribeToAllPeers( + VideoResolution resolution) { + default_resolution_ = resolution; + return *this; +} + +absl::optional<VideoResolution> VideoSubscription::GetResolutionForPeer( + absl::string_view peer_name) const { + auto it = peers_resolution_.find(std::string(peer_name)); + if (it == peers_resolution_.end()) { + return default_resolution_; + } + return it->second; +} + +std::vector<std::string> VideoSubscription::GetSubscribedPeers() const { + std::vector<std::string> subscribed_streams; + subscribed_streams.reserve(peers_resolution_.size()); + for (const auto& entry : peers_resolution_) { + subscribed_streams.push_back(entry.first); + } + return subscribed_streams; +} + +std::string VideoSubscription::ToString() const { + rtc::StringBuilder out; + out << "{ default_resolution_=["; + if (default_resolution_.has_value()) { + out << default_resolution_->ToString(); + } else { + out << "undefined"; + } + out << "], {"; + for (const auto& [peer_name, resolution] : peers_resolution_) { + out << "[" << peer_name << ": " << resolution.ToString() << "], "; + } + out << "} }"; + return out.Release(); +} +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/pclf/media_configuration.h b/third_party/libwebrtc/api/test/pclf/media_configuration.h new file mode 100644 index 0000000000..8e841a265b --- /dev/null +++ b/third_party/libwebrtc/api/test/pclf/media_configuration.h @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_PCLF_MEDIA_CONFIGURATION_H_ +#define API_TEST_PCLF_MEDIA_CONFIGURATION_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <functional> +#include <map> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/async_resolver_factory.h" +#include "api/audio/audio_mixer.h" +#include "api/audio_options.h" +#include "api/call/call_factory_interface.h" +#include "api/fec_controller.h" +#include "api/function_view.h" +#include "api/media_stream_interface.h" +#include "api/peer_connection_interface.h" +#include "api/rtc_event_log/rtc_event_log_factory_interface.h" +#include "api/rtp_parameters.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/test/audio_quality_analyzer_interface.h" +#include "api/test/frame_generator_interface.h" +#include "api/test/peer_network_dependencies.h" +#include "api/test/simulated_network.h" +#include "api/test/stats_observer_interface.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/test/video/video_frame_writer.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/transport/network_control.h" +#include "api/units/time_delta.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/checks.h" +#include "rtc_base/network.h" +#include "rtc_base/rtc_certificate_generator.h" +#include "rtc_base/ssl_certificate.h" +#include "rtc_base/thread.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +constexpr size_t kDefaultSlidesWidth = 1850; +constexpr size_t kDefaultSlidesHeight = 1110; + +// The index of required capturing device in OS provided list of video +// devices. On Linux and Windows the list will be obtained via +// webrtc::VideoCaptureModule::DeviceInfo, on Mac OS via +// [RTCCameraVideoCapturer captureDevices]. +enum class CapturingDeviceIndex : size_t {}; + +// Contains parameters for screen share scrolling. +// +// If scrolling is enabled, then it will be done by putting sliding window +// on source video and moving this window from top left corner to the +// bottom right corner of the picture. +// +// In such case source dimensions must be greater or equal to the sliding +// window dimensions. So `source_width` and `source_height` are the dimensions +// of the source frame, while `VideoConfig::width` and `VideoConfig::height` +// are the dimensions of the sliding window. +// +// Because `source_width` and `source_height` are dimensions of the source +// frame, they have to be width and height of videos from +// `ScreenShareConfig::slides_yuv_file_names`. +// +// Because scrolling have to be done on single slide it also requires, that +// `duration` must be less or equal to +// `ScreenShareConfig::slide_change_interval`. +struct ScrollingParams { + // Duration of scrolling. + TimeDelta duration; + // Width of source slides video. + size_t source_width = kDefaultSlidesWidth; + // Height of source slides video. + size_t source_height = kDefaultSlidesHeight; +}; + +// Contains screen share video stream properties. +struct ScreenShareConfig { + explicit ScreenShareConfig(TimeDelta slide_change_interval); + + // Shows how long one slide should be presented on the screen during + // slide generation. + TimeDelta slide_change_interval; + // If true, slides will be generated programmatically. No scrolling params + // will be applied in such case. + bool generate_slides = false; + // If present scrolling will be applied. Please read extra requirement on + // `slides_yuv_file_names` for scrolling. + absl::optional<ScrollingParams> scrolling_params; + // Contains list of yuv files with slides. + // + // If empty, default set of slides will be used. In such case + // `VideoConfig::width` must be equal to `kDefaultSlidesWidth` and + // `VideoConfig::height` must be equal to `kDefaultSlidesHeight` or if + // `scrolling_params` are specified, then `ScrollingParams::source_width` + // must be equal to `kDefaultSlidesWidth` and + // `ScrollingParams::source_height` must be equal to `kDefaultSlidesHeight`. + std::vector<std::string> slides_yuv_file_names; +}; + +// Config for Vp8 simulcast or non-standard Vp9 SVC testing. +// +// To configure standard SVC setting, use `scalability_mode` in the +// `encoding_params` array. +// This configures Vp9 SVC by requesting simulcast layers, the request is +// internally converted to a request for SVC layers. +// +// SVC support is limited: +// During SVC testing there is no SFU, so framework will try to emulate SFU +// behavior in regular p2p call. Because of it there are such limitations: +// * if `target_spatial_index` is not equal to the highest spatial layer +// then no packet/frame drops are allowed. +// +// If there will be any drops, that will affect requested layer, then +// WebRTC SVC implementation will continue decoding only the highest +// available layer and won't restore lower layers, so analyzer won't +// receive required data which will cause wrong results or test failures. +struct VideoSimulcastConfig { + explicit VideoSimulcastConfig(int simulcast_streams_count); + + // Specified amount of simulcast streams/SVC layers, depending on which + // encoder is used. + int simulcast_streams_count; +}; + +// Configuration for the emulated Selective Forward Unit (SFU) +// +// The framework can optionally filter out frames that are decoded +// using an emulated SFU. +// When using simulcast or SVC, it's not always desirable to receive +// all frames. In a real world call, a SFU will only forward a subset +// of the frames. +// The emulated SFU is not able to change its configuration dynamically, +// if adaptation happens during the call, layers may be dropped and the +// analyzer won't receive the required data which will cause wrong results or +// test failures. +struct EmulatedSFUConfig { + EmulatedSFUConfig() = default; + explicit EmulatedSFUConfig(int target_layer_index); + EmulatedSFUConfig(absl::optional<int> target_layer_index, + absl::optional<int> target_temporal_index); + + // Specifies simulcast or spatial index of the video stream to analyze. + // There are 2 cases: + // 1. simulcast encoding is used: + // in such case `target_layer_index` will specify the index of + // simulcast stream, that should be analyzed. Other streams will be + // dropped. + // 2. SVC encoding is used: + // in such case `target_layer_index` will specify the top interesting + // spatial layer and all layers below, including target one will be + // processed. All layers above target one will be dropped. + // If not specified then all streams will be received and analyzed. + // When set, it instructs the framework to create an emulated Selective + // Forwarding Unit (SFU) that will propagate only the requested layers. + absl::optional<int> target_layer_index; + // Specifies the index of the maximum temporal unit to keep. + // If not specified then all temporal layers will be received and analyzed. + // When set, it instructs the framework to create an emulated Selective + // Forwarding Unit (SFU) that will propagate only up to the requested layer. + absl::optional<int> target_temporal_index; +}; + +class VideoResolution { + public: + // Determines special resolutions, which can't be expressed in terms of + // width, height and fps. + enum class Spec { + // No extra spec set. It describes a regular resolution described by + // width, height and fps. + kNone, + // Describes resolution which contains max value among all sender's + // video streams in each dimension (width, height, fps). + kMaxFromSender + }; + + VideoResolution(size_t width, size_t height, int32_t fps); + explicit VideoResolution(Spec spec = Spec::kNone); + + bool operator==(const VideoResolution& other) const; + bool operator!=(const VideoResolution& other) const; + + size_t width() const { return width_; } + void set_width(size_t width) { width_ = width; } + size_t height() const { return height_; } + void set_height(size_t height) { height_ = height; } + int32_t fps() const { return fps_; } + void set_fps(int32_t fps) { fps_ = fps; } + + // Returns if it is a regular resolution or not. The resolution is regular + // if it's spec is `Spec::kNone`. + bool IsRegular() const; + + std::string ToString() const; + + private: + size_t width_ = 0; + size_t height_ = 0; + int32_t fps_ = 0; + Spec spec_ = Spec::kNone; +}; + +class VideoDumpOptions { + public: + static constexpr int kDefaultSamplingModulo = 1; + + // output_directory - the output directory where stream will be dumped. The + // output files' names will be constructed as + // <stream_name>_<receiver_name>_<resolution>.<extension> for output dumps + // and <stream_name>_<resolution>.<extension> for input dumps. + // By default <extension> is "y4m". Resolution is in the format + // <width>x<height>_<fps>. + // sampling_modulo - the module for the video frames to be dumped. Modulo + // equals X means every Xth frame will be written to the dump file. The + // value must be greater than 0. (Default: 1) + // export_frame_ids - specifies if frame ids should be exported together + // with content of the stream. If true, an output file with the same name as + // video dump and suffix ".frame_ids.txt" will be created. It will contain + // the frame ids in the same order as original frames in the output + // file with stream content. File will contain one frame id per line. + // (Default: false) + // `video_frame_writer_factory` - factory function to create a video frame + // writer for input and output video files. (Default: Y4M video writer + // factory). + explicit VideoDumpOptions( + absl::string_view output_directory, + int sampling_modulo = kDefaultSamplingModulo, + bool export_frame_ids = false, + std::function<std::unique_ptr<test::VideoFrameWriter>( + absl::string_view file_name_prefix, + const VideoResolution& resolution)> video_frame_writer_factory = + Y4mVideoFrameWriterFactory); + VideoDumpOptions(absl::string_view output_directory, bool export_frame_ids); + + VideoDumpOptions(const VideoDumpOptions&) = default; + VideoDumpOptions& operator=(const VideoDumpOptions&) = default; + VideoDumpOptions(VideoDumpOptions&&) = default; + VideoDumpOptions& operator=(VideoDumpOptions&&) = default; + + std::string output_directory() const { return output_directory_; } + int sampling_modulo() const { return sampling_modulo_; } + bool export_frame_ids() const { return export_frame_ids_; } + + std::unique_ptr<test::VideoFrameWriter> CreateInputDumpVideoFrameWriter( + absl::string_view stream_label, + const VideoResolution& resolution) const; + + std::unique_ptr<test::VideoFrameWriter> CreateOutputDumpVideoFrameWriter( + absl::string_view stream_label, + absl::string_view receiver, + const VideoResolution& resolution) const; + + std::string ToString() const; + + private: + static std::unique_ptr<test::VideoFrameWriter> Y4mVideoFrameWriterFactory( + absl::string_view file_name_prefix, + const VideoResolution& resolution); + std::string GetInputDumpFileName(absl::string_view stream_label, + const VideoResolution& resolution) const; + // Returns file name for input frame ids dump if `export_frame_ids()` is + // true, absl::nullopt otherwise. + absl::optional<std::string> GetInputFrameIdsDumpFileName( + absl::string_view stream_label, + const VideoResolution& resolution) const; + std::string GetOutputDumpFileName(absl::string_view stream_label, + absl::string_view receiver, + const VideoResolution& resolution) const; + // Returns file name for output frame ids dump if `export_frame_ids()` is + // true, absl::nullopt otherwise. + absl::optional<std::string> GetOutputFrameIdsDumpFileName( + absl::string_view stream_label, + absl::string_view receiver, + const VideoResolution& resolution) const; + + std::string output_directory_; + int sampling_modulo_ = 1; + bool export_frame_ids_ = false; + std::function<std::unique_ptr<test::VideoFrameWriter>( + absl::string_view file_name_prefix, + const VideoResolution& resolution)> + video_frame_writer_factory_; +}; + +// Contains properties of single video stream. +struct VideoConfig { + explicit VideoConfig(const VideoResolution& resolution); + VideoConfig(size_t width, size_t height, int32_t fps); + VideoConfig(std::string stream_label, + size_t width, + size_t height, + int32_t fps); + + // Video stream width. + size_t width; + // Video stream height. + size_t height; + int32_t fps; + VideoResolution GetResolution() const { + return VideoResolution(width, height, fps); + } + + // Have to be unique among all specified configs for all peers in the call. + // Will be auto generated if omitted. + absl::optional<std::string> stream_label; + // Will be set for current video track. If equals to kText or kDetailed - + // screencast in on. + absl::optional<VideoTrackInterface::ContentHint> content_hint; + // If presented video will be transfered in simulcast/SVC mode depending on + // which encoder is used. + // + // Simulcast is supported only from 1st added peer. For VP8 simulcast only + // without RTX is supported so it will be automatically disabled for all + // simulcast tracks. For VP9 simulcast enables VP9 SVC mode and support RTX, + // but only on non-lossy networks. See more in documentation to + // VideoSimulcastConfig. + absl::optional<VideoSimulcastConfig> simulcast_config; + // Configuration for the emulated Selective Forward Unit (SFU). + absl::optional<EmulatedSFUConfig> emulated_sfu_config; + // Encoding parameters for both singlecast and per simulcast layer. + // If singlecast is used, if not empty, a single value can be provided. + // If simulcast is used, if not empty, `encoding_params` size have to be + // equal to `simulcast_config.simulcast_streams_count`. Will be used to set + // transceiver send encoding params for each layer. + // RtpEncodingParameters::rid may be changed by fixture implementation to + // ensure signaling correctness. + std::vector<RtpEncodingParameters> encoding_params; + // Count of temporal layers for video stream. This value will be set into + // each RtpEncodingParameters of RtpParameters of corresponding + // RtpSenderInterface for this video stream. + absl::optional<int> temporal_layers_count; + // If specified defines how input should be dumped. It is actually one of + // the test's output file, which contains copy of what was captured during + // the test for this video stream on sender side. It is useful when + // generator is used as input. + absl::optional<VideoDumpOptions> input_dump_options; + // If specified defines how output should be dumped on the receiver side for + // this stream. The produced files contain what was rendered for this video + // stream on receiver side per each receiver. + absl::optional<VideoDumpOptions> output_dump_options; + // If set to true uses fixed frame rate while dumping output video to the + // file. Requested `VideoSubscription::fps()` will be used as frame rate. + bool output_dump_use_fixed_framerate = false; + // If true will display input and output video on the user's screen. + bool show_on_screen = false; + // If specified, determines a sync group to which this video stream belongs. + // According to bugs.webrtc.org/4762 WebRTC supports synchronization only + // for pair of single audio and single video stream. + absl::optional<std::string> sync_group; + // If specified, it will be set into RtpParameters of corresponding + // RtpSenderInterface for this video stream. + // Note that this setting takes precedence over `content_hint`. + absl::optional<DegradationPreference> degradation_preference; +}; + +// Contains properties for audio in the call. +struct AudioConfig { + enum Mode { + kGenerated, + kFile, + }; + + AudioConfig() = default; + explicit AudioConfig(std::string stream_label); + + // Have to be unique among all specified configs for all peers in the call. + // Will be auto generated if omitted. + absl::optional<std::string> stream_label; + Mode mode = kGenerated; + // Have to be specified only if mode = kFile + absl::optional<std::string> input_file_name; + // If specified the input stream will be also copied to specified file. + absl::optional<std::string> input_dump_file_name; + // If specified the output stream will be copied to specified file. + absl::optional<std::string> output_dump_file_name; + + // Audio options to use. + cricket::AudioOptions audio_options; + // Sampling frequency of input audio data (from file or generated). + int sampling_frequency_in_hz = 48000; + // If specified, determines a sync group to which this audio stream belongs. + // According to bugs.webrtc.org/4762 WebRTC supports synchronization only + // for pair of single audio and single video stream. + absl::optional<std::string> sync_group; +}; + +struct VideoCodecConfig { + explicit VideoCodecConfig(std::string name); + VideoCodecConfig(std::string name, + std::map<std::string, std::string> required_params); + // Next two fields are used to specify concrete video codec, that should be + // used in the test. Video code will be negotiated in SDP during offer/ + // answer exchange. + // Video codec name. You can find valid names in + // media/base/media_constants.h + std::string name; + // Map of parameters, that have to be specified on SDP codec. Each parameter + // is described by key and value. Codec parameters will match the specified + // map if and only if for each key from `required_params` there will be + // a parameter with name equal to this key and parameter value will be equal + // to the value from `required_params` for this key. + // If empty then only name will be used to match the codec. + std::map<std::string, std::string> required_params; +}; + +// Subscription to the remote video streams. It declares which remote stream +// peer should receive and in which resolution (width x height x fps). +class VideoSubscription { + public: + // Returns the resolution constructed as maximum from all resolution + // dimensions: width, height and fps. + static absl::optional<VideoResolution> GetMaxResolution( + rtc::ArrayView<const VideoConfig> video_configs); + static absl::optional<VideoResolution> GetMaxResolution( + rtc::ArrayView<const VideoResolution> resolutions); + + bool operator==(const VideoSubscription& other) const; + bool operator!=(const VideoSubscription& other) const; + + // Subscribes receiver to all streams sent by the specified peer with + // specified resolution. It will override any resolution that was used in + // `SubscribeToAll` independently from methods call order. + VideoSubscription& SubscribeToPeer( + absl::string_view peer_name, + VideoResolution resolution = + VideoResolution(VideoResolution::Spec::kMaxFromSender)); + + // Subscribes receiver to the all sent streams with specified resolution. + // If any stream was subscribed to with `SubscribeTo` method that will + // override resolution passed to this function independently from methods + // call order. + VideoSubscription& SubscribeToAllPeers( + VideoResolution resolution = + VideoResolution(VideoResolution::Spec::kMaxFromSender)); + + // Returns resolution for specific sender. If no specific resolution was + // set for this sender, then will return resolution used for all streams. + // If subscription doesn't subscribe to all streams, `absl::nullopt` will be + // returned. + absl::optional<VideoResolution> GetResolutionForPeer( + absl::string_view peer_name) const; + + // Returns a maybe empty list of senders for which peer explicitly + // subscribed to with specific resolution. + std::vector<std::string> GetSubscribedPeers() const; + + std::string ToString() const; + + private: + absl::optional<VideoResolution> default_resolution_ = absl::nullopt; + std::map<std::string, VideoResolution> peers_resolution_; +}; + +// Contains configuration for echo emulator. +struct EchoEmulationConfig { + // Delay which represents the echo path delay, i.e. how soon rendered signal + // should reach capturer. + TimeDelta echo_delay = TimeDelta::Millis(50); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_PCLF_MEDIA_CONFIGURATION_H_ diff --git a/third_party/libwebrtc/api/test/pclf/media_quality_test_params.h b/third_party/libwebrtc/api/test/pclf/media_quality_test_params.h new file mode 100644 index 0000000000..3377d31f68 --- /dev/null +++ b/third_party/libwebrtc/api/test/pclf/media_quality_test_params.h @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_PCLF_MEDIA_QUALITY_TEST_PARAMS_H_ +#define API_TEST_PCLF_MEDIA_QUALITY_TEST_PARAMS_H_ + +#include <cstddef> +#include <memory> +#include <string> +#include <vector> + +#include "api/async_resolver_factory.h" +#include "api/audio/audio_mixer.h" +#include "api/call/call_factory_interface.h" +#include "api/fec_controller.h" +#include "api/field_trials_view.h" +#include "api/rtc_event_log/rtc_event_log_factory_interface.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/test/pclf/media_configuration.h" +#include "api/transport/network_control.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "p2p/base/port_allocator.h" +#include "rtc_base/network.h" +#include "rtc_base/rtc_certificate_generator.h" +#include "rtc_base/ssl_certificate.h" +#include "rtc_base/thread.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Contains most part from PeerConnectionFactoryDependencies. Also all fields +// are optional and defaults will be provided by fixture implementation if +// any will be omitted. +// +// Separate class was introduced to clarify which components can be +// overridden. For example worker and signaling threads will be provided by +// fixture implementation. The same is applicable to the media engine. So user +// can override only some parts of media engine like video encoder/decoder +// factories. +struct PeerConnectionFactoryComponents { + std::unique_ptr<TaskQueueFactory> task_queue_factory; + std::unique_ptr<CallFactoryInterface> call_factory; + std::unique_ptr<RtcEventLogFactoryInterface> event_log_factory; + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory; + std::unique_ptr<NetworkControllerFactoryInterface> network_controller_factory; + std::unique_ptr<NetEqFactory> neteq_factory; + + // Will be passed to MediaEngineInterface, that will be used in + // PeerConnectionFactory. + std::unique_ptr<VideoEncoderFactory> video_encoder_factory; + std::unique_ptr<VideoDecoderFactory> video_decoder_factory; + + std::unique_ptr<FieldTrialsView> trials; + + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing; + rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer; +}; + +// Contains most parts from PeerConnectionDependencies. Also all fields are +// optional and defaults will be provided by fixture implementation if any +// will be omitted. +// +// Separate class was introduced to clarify which components can be +// overridden. For example observer, which is required to +// PeerConnectionDependencies, will be provided by fixture implementation, +// so client can't inject its own. Also only network manager can be overridden +// inside port allocator. +struct PeerConnectionComponents { + PeerConnectionComponents(rtc::NetworkManager* network_manager, + rtc::PacketSocketFactory* packet_socket_factory) + : network_manager(network_manager), + packet_socket_factory(packet_socket_factory) { + RTC_CHECK(network_manager); + } + + rtc::NetworkManager* const network_manager; + rtc::PacketSocketFactory* const packet_socket_factory; + std::unique_ptr<webrtc::AsyncResolverFactory> async_resolver_factory; + std::unique_ptr<rtc::RTCCertificateGeneratorInterface> cert_generator; + std::unique_ptr<rtc::SSLCertificateVerifier> tls_cert_verifier; + std::unique_ptr<IceTransportFactory> ice_transport_factory; +}; + +// Contains all components, that can be overridden in peer connection. Also +// has a network thread, that will be used to communicate with another peers. +struct InjectableComponents { + InjectableComponents(rtc::Thread* network_thread, + rtc::NetworkManager* network_manager, + rtc::PacketSocketFactory* packet_socket_factory) + : network_thread(network_thread), + worker_thread(nullptr), + pcf_dependencies(std::make_unique<PeerConnectionFactoryComponents>()), + pc_dependencies( + std::make_unique<PeerConnectionComponents>(network_manager, + packet_socket_factory)) { + RTC_CHECK(network_thread); + } + + rtc::Thread* const network_thread; + rtc::Thread* worker_thread; + + std::unique_ptr<PeerConnectionFactoryComponents> pcf_dependencies; + std::unique_ptr<PeerConnectionComponents> pc_dependencies; +}; + +// Contains information about call media streams (up to 1 audio stream and +// unlimited amount of video streams) and rtc configuration, that will be used +// to set up peer connection. +struct Params { + // Peer name. If empty - default one will be set by the fixture. + absl::optional<std::string> name; + // If `audio_config` is set audio stream will be configured + absl::optional<AudioConfig> audio_config; + // Flags to set on `cricket::PortAllocator`. These flags will be added + // to the default ones that are presented on the port allocator. + uint32_t port_allocator_extra_flags = cricket::kDefaultPortAllocatorFlags; + // If `rtc_event_log_path` is set, an RTCEventLog will be saved in that + // location and it will be available for further analysis. + absl::optional<std::string> rtc_event_log_path; + // If `aec_dump_path` is set, an AEC dump will be saved in that location and + // it will be available for further analysis. + absl::optional<std::string> aec_dump_path; + + bool use_ulp_fec = false; + bool use_flex_fec = false; + // Specifies how much video encoder target bitrate should be different than + // target bitrate, provided by WebRTC stack. Must be greater then 0. Can be + // used to emulate overshooting of video encoders. This multiplier will + // be applied for all video encoder on both sides for all layers. Bitrate + // estimated by WebRTC stack will be multiplied by this multiplier and then + // provided into VideoEncoder::SetRates(...). + double video_encoder_bitrate_multiplier = 1.0; + + PeerConnectionInterface::RTCConfiguration rtc_configuration; + PeerConnectionInterface::RTCOfferAnswerOptions rtc_offer_answer_options; + BitrateSettings bitrate_settings; + std::vector<VideoCodecConfig> video_codecs; + + // A list of RTP header extensions which will be enforced on all video streams + // added to this peer. + std::vector<std::string> extra_video_rtp_header_extensions; + // A list of RTP header extensions which will be enforced on all audio streams + // added to this peer. + std::vector<std::string> extra_audio_rtp_header_extensions; +}; + +// Contains parameters that maybe changed by test writer during the test call. +struct ConfigurableParams { + // If `video_configs` is empty - no video should be added to the test call. + std::vector<VideoConfig> video_configs; + + VideoSubscription video_subscription = + VideoSubscription().SubscribeToAllPeers(); +}; + +// Contains parameters, that describe how long framework should run quality +// test. +struct RunParams { + explicit RunParams(TimeDelta run_duration) : run_duration(run_duration) {} + + // Specifies how long the test should be run. This time shows how long + // the media should flow after connection was established and before + // it will be shut downed. + TimeDelta run_duration; + + // If set to true peers will be able to use Flex FEC, otherwise they won't + // be able to negotiate it even if it's enabled on per peer level. + bool enable_flex_fec_support = false; + // If true will set conference mode in SDP media section for all video + // tracks for all peers. + bool use_conference_mode = false; + // If specified echo emulation will be done, by mixing the render audio into + // the capture signal. In such case input signal will be reduced by half to + // avoid saturation or compression in the echo path simulation. + absl::optional<EchoEmulationConfig> echo_emulation_config; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_PCLF_MEDIA_QUALITY_TEST_PARAMS_H_ diff --git a/third_party/libwebrtc/api/test/pclf/peer_configurer.cc b/third_party/libwebrtc/api/test/pclf/peer_configurer.cc new file mode 100644 index 0000000000..2203a4c4f0 --- /dev/null +++ b/third_party/libwebrtc/api/test/pclf/peer_configurer.cc @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/pclf/peer_configurer.h" + +#include <set> + +#include "absl/strings/string_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/peer_network_dependencies.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +PeerConfigurer::PeerConfigurer( + const PeerNetworkDependencies& network_dependencies) + : components_(std::make_unique<InjectableComponents>( + network_dependencies.network_thread, + network_dependencies.network_manager, + network_dependencies.packet_socket_factory)), + params_(std::make_unique<Params>()), + configurable_params_(std::make_unique<ConfigurableParams>()) {} + +PeerConfigurer* PeerConfigurer::SetName(absl::string_view name) { + params_->name = std::string(name); + return this; +} + +PeerConfigurer* PeerConfigurer::SetTaskQueueFactory( + std::unique_ptr<TaskQueueFactory> task_queue_factory) { + components_->pcf_dependencies->task_queue_factory = + std::move(task_queue_factory); + return this; +} +PeerConfigurer* PeerConfigurer::SetCallFactory( + std::unique_ptr<CallFactoryInterface> call_factory) { + components_->pcf_dependencies->call_factory = std::move(call_factory); + return this; +} +PeerConfigurer* PeerConfigurer::SetEventLogFactory( + std::unique_ptr<RtcEventLogFactoryInterface> event_log_factory) { + components_->pcf_dependencies->event_log_factory = + std::move(event_log_factory); + return this; +} +PeerConfigurer* PeerConfigurer::SetFecControllerFactory( + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory) { + components_->pcf_dependencies->fec_controller_factory = + std::move(fec_controller_factory); + return this; +} +PeerConfigurer* PeerConfigurer::SetNetworkControllerFactory( + std::unique_ptr<NetworkControllerFactoryInterface> + network_controller_factory) { + components_->pcf_dependencies->network_controller_factory = + std::move(network_controller_factory); + return this; +} +PeerConfigurer* PeerConfigurer::SetVideoEncoderFactory( + std::unique_ptr<VideoEncoderFactory> video_encoder_factory) { + components_->pcf_dependencies->video_encoder_factory = + std::move(video_encoder_factory); + return this; +} +PeerConfigurer* PeerConfigurer::SetVideoDecoderFactory( + std::unique_ptr<VideoDecoderFactory> video_decoder_factory) { + components_->pcf_dependencies->video_decoder_factory = + std::move(video_decoder_factory); + return this; +} + +PeerConfigurer* PeerConfigurer::SetAsyncResolverFactory( + std::unique_ptr<webrtc::AsyncResolverFactory> async_resolver_factory) { + components_->pc_dependencies->async_resolver_factory = + std::move(async_resolver_factory); + return this; +} +PeerConfigurer* PeerConfigurer::SetRTCCertificateGenerator( + std::unique_ptr<rtc::RTCCertificateGeneratorInterface> cert_generator) { + components_->pc_dependencies->cert_generator = std::move(cert_generator); + return this; +} +PeerConfigurer* PeerConfigurer::SetSSLCertificateVerifier( + std::unique_ptr<rtc::SSLCertificateVerifier> tls_cert_verifier) { + components_->pc_dependencies->tls_cert_verifier = + std::move(tls_cert_verifier); + return this; +} + +PeerConfigurer* PeerConfigurer::AddVideoConfig(VideoConfig config) { + video_sources_.push_back( + CreateSquareFrameGenerator(config, /*type=*/absl::nullopt)); + configurable_params_->video_configs.push_back(std::move(config)); + return this; +} +PeerConfigurer* PeerConfigurer::AddVideoConfig( + VideoConfig config, + std::unique_ptr<test::FrameGeneratorInterface> generator) { + configurable_params_->video_configs.push_back(std::move(config)); + video_sources_.push_back(std::move(generator)); + return this; +} +PeerConfigurer* PeerConfigurer::AddVideoConfig(VideoConfig config, + CapturingDeviceIndex index) { + configurable_params_->video_configs.push_back(std::move(config)); + video_sources_.push_back(index); + return this; +} +PeerConfigurer* PeerConfigurer::SetVideoSubscription( + VideoSubscription subscription) { + configurable_params_->video_subscription = std::move(subscription); + return this; +} +PeerConfigurer* PeerConfigurer::SetVideoCodecs( + std::vector<VideoCodecConfig> video_codecs) { + params_->video_codecs = std::move(video_codecs); + return this; +} +PeerConfigurer* PeerConfigurer::SetExtraVideoRtpHeaderExtensions( + std::vector<std::string> extensions) { + params_->extra_video_rtp_header_extensions = std::move(extensions); + return this; +} +PeerConfigurer* PeerConfigurer::SetAudioConfig(AudioConfig config) { + params_->audio_config = std::move(config); + return this; +} +PeerConfigurer* PeerConfigurer::SetExtraAudioRtpHeaderExtensions( + std::vector<std::string> extensions) { + params_->extra_audio_rtp_header_extensions = std::move(extensions); + return this; +} +PeerConfigurer* PeerConfigurer::SetUseUlpFEC(bool value) { + params_->use_ulp_fec = value; + return this; +} +PeerConfigurer* PeerConfigurer::SetUseFlexFEC(bool value) { + params_->use_flex_fec = value; + return this; +} +PeerConfigurer* PeerConfigurer::SetVideoEncoderBitrateMultiplier( + double multiplier) { + params_->video_encoder_bitrate_multiplier = multiplier; + return this; +} +PeerConfigurer* PeerConfigurer::SetNetEqFactory( + std::unique_ptr<NetEqFactory> neteq_factory) { + components_->pcf_dependencies->neteq_factory = std::move(neteq_factory); + return this; +} +PeerConfigurer* PeerConfigurer::SetAudioProcessing( + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing) { + components_->pcf_dependencies->audio_processing = audio_processing; + return this; +} +PeerConfigurer* PeerConfigurer::SetAudioMixer( + rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer) { + components_->pcf_dependencies->audio_mixer = audio_mixer; + return this; +} + +PeerConfigurer* PeerConfigurer::SetUseNetworkThreadAsWorkerThread() { + components_->worker_thread = components_->network_thread; + return this; +} + +PeerConfigurer* PeerConfigurer::SetRtcEventLogPath(std::string path) { + params_->rtc_event_log_path = std::move(path); + return this; +} +PeerConfigurer* PeerConfigurer::SetAecDumpPath(std::string path) { + params_->aec_dump_path = std::move(path); + return this; +} +PeerConfigurer* PeerConfigurer::SetRTCConfiguration( + PeerConnectionInterface::RTCConfiguration configuration) { + params_->rtc_configuration = std::move(configuration); + return this; +} +PeerConfigurer* PeerConfigurer::SetRTCOfferAnswerOptions( + PeerConnectionInterface::RTCOfferAnswerOptions options) { + params_->rtc_offer_answer_options = std::move(options); + return this; +} +PeerConfigurer* PeerConfigurer::SetBitrateSettings( + BitrateSettings bitrate_settings) { + params_->bitrate_settings = bitrate_settings; + return this; +} + +PeerConfigurer* PeerConfigurer::SetIceTransportFactory( + std::unique_ptr<IceTransportFactory> factory) { + components_->pc_dependencies->ice_transport_factory = std::move(factory); + return this; +} + +PeerConfigurer* PeerConfigurer::SetPortAllocatorExtraFlags( + uint32_t extra_flags) { + params_->port_allocator_extra_flags = extra_flags; + return this; +} +std::unique_ptr<InjectableComponents> PeerConfigurer::ReleaseComponents() { + RTC_CHECK(components_); + auto components = std::move(components_); + components_ = nullptr; + return components; +} + +// Returns Params and transfer ownership to the caller. +// Can be called once. +std::unique_ptr<Params> PeerConfigurer::ReleaseParams() { + RTC_CHECK(params_); + auto params = std::move(params_); + params_ = nullptr; + return params; +} + +// Returns ConfigurableParams and transfer ownership to the caller. +// Can be called once. +std::unique_ptr<ConfigurableParams> +PeerConfigurer::ReleaseConfigurableParams() { + RTC_CHECK(configurable_params_); + auto configurable_params = std::move(configurable_params_); + configurable_params_ = nullptr; + return configurable_params; +} + +// Returns video sources and transfer frame generators ownership to the +// caller. Can be called once. +std::vector<PeerConfigurer::VideoSource> PeerConfigurer::ReleaseVideoSources() { + auto video_sources = std::move(video_sources_); + video_sources_.clear(); + return video_sources; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/pclf/peer_configurer.h b/third_party/libwebrtc/api/test/pclf/peer_configurer.h new file mode 100644 index 0000000000..996dc5cd89 --- /dev/null +++ b/third_party/libwebrtc/api/test/pclf/peer_configurer.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_PCLF_PEER_CONFIGURER_H_ +#define API_TEST_PCLF_PEER_CONFIGURER_H_ + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/async_resolver_factory.h" +#include "api/audio/audio_mixer.h" +#include "api/call/call_factory_interface.h" +#include "api/fec_controller.h" +#include "api/rtc_event_log/rtc_event_log_factory_interface.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/test/create_peer_connection_quality_test_frame_generator.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/peer_network_dependencies.h" +#include "api/transport/network_control.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/network.h" +#include "rtc_base/rtc_certificate_generator.h" +#include "rtc_base/ssl_certificate.h" +#include "rtc_base/thread.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// This class is used to fully configure one peer inside a call. +class PeerConfigurer { + public: + using VideoSource = + absl::variant<std::unique_ptr<test::FrameGeneratorInterface>, + CapturingDeviceIndex>; + + explicit PeerConfigurer(const PeerNetworkDependencies& network_dependencies); + + // Sets peer name that will be used to report metrics related to this peer. + // If not set, some default name will be assigned. All names have to be + // unique. + PeerConfigurer* SetName(absl::string_view name); + + // The parameters of the following 9 methods will be passed to the + // PeerConnectionFactoryInterface implementation that will be created for + // this peer. + PeerConfigurer* SetTaskQueueFactory( + std::unique_ptr<TaskQueueFactory> task_queue_factory); + PeerConfigurer* SetCallFactory( + std::unique_ptr<CallFactoryInterface> call_factory); + PeerConfigurer* SetEventLogFactory( + std::unique_ptr<RtcEventLogFactoryInterface> event_log_factory); + PeerConfigurer* SetFecControllerFactory( + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory); + PeerConfigurer* SetNetworkControllerFactory( + std::unique_ptr<NetworkControllerFactoryInterface> + network_controller_factory); + PeerConfigurer* SetVideoEncoderFactory( + std::unique_ptr<VideoEncoderFactory> video_encoder_factory); + PeerConfigurer* SetVideoDecoderFactory( + std::unique_ptr<VideoDecoderFactory> video_decoder_factory); + // Set a custom NetEqFactory to be used in the call. + PeerConfigurer* SetNetEqFactory(std::unique_ptr<NetEqFactory> neteq_factory); + PeerConfigurer* SetAudioProcessing( + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing); + PeerConfigurer* SetAudioMixer( + rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer); + + // Forces the Peerconnection to use the network thread as the worker thread. + // Ie, worker thread and the network thread is the same thread. + PeerConfigurer* SetUseNetworkThreadAsWorkerThread(); + + // The parameters of the following 4 methods will be passed to the + // PeerConnectionInterface implementation that will be created for this + // peer. + PeerConfigurer* SetAsyncResolverFactory( + std::unique_ptr<webrtc::AsyncResolverFactory> async_resolver_factory); + PeerConfigurer* SetRTCCertificateGenerator( + std::unique_ptr<rtc::RTCCertificateGeneratorInterface> cert_generator); + PeerConfigurer* SetSSLCertificateVerifier( + std::unique_ptr<rtc::SSLCertificateVerifier> tls_cert_verifier); + PeerConfigurer* SetIceTransportFactory( + std::unique_ptr<IceTransportFactory> factory); + // Flags to set on `cricket::PortAllocator`. These flags will be added + // to the default ones that are presented on the port allocator. + // For possible values check p2p/base/port_allocator.h. + PeerConfigurer* SetPortAllocatorExtraFlags(uint32_t extra_flags); + + // Add new video stream to the call that will be sent from this peer. + // Default implementation of video frames generator will be used. + PeerConfigurer* AddVideoConfig(VideoConfig config); + // Add new video stream to the call that will be sent from this peer with + // provided own implementation of video frames generator. + PeerConfigurer* AddVideoConfig( + VideoConfig config, + std::unique_ptr<test::FrameGeneratorInterface> generator); + // Add new video stream to the call that will be sent from this peer. + // Capturing device with specified index will be used to get input video. + PeerConfigurer* AddVideoConfig(VideoConfig config, + CapturingDeviceIndex capturing_device_index); + // Sets video subscription for the peer. By default subscription will + // include all streams with `VideoSubscription::kSameAsSendStream` + // resolution. To this behavior use this method. + PeerConfigurer* SetVideoSubscription(VideoSubscription subscription); + // Sets the list of video codecs used by the peer during the test. These + // codecs will be negotiated in SDP during offer/answer exchange. The order + // of these codecs during negotiation will be the same as in `video_codecs`. + // Codecs have to be available in codecs list provided by peer connection to + // be negotiated. If some of specified codecs won't be found, the test will + // crash. + PeerConfigurer* SetVideoCodecs(std::vector<VideoCodecConfig> video_codecs); + // Sets a list of RTP header extensions which will be enforced on all video + // streams added to this peer. + PeerConfigurer* SetExtraVideoRtpHeaderExtensions( + std::vector<std::string> extensions); + // Sets the audio stream for the call from this peer. If this method won't + // be invoked, this peer will send no audio. + PeerConfigurer* SetAudioConfig(AudioConfig config); + // Sets a list of RTP header extensions which will be enforced on all audio + // streams added to this peer. + PeerConfigurer* SetExtraAudioRtpHeaderExtensions( + std::vector<std::string> extensions); + + // Set if ULP FEC should be used or not. False by default. + PeerConfigurer* SetUseUlpFEC(bool value); + // Set if Flex FEC should be used or not. False by default. + // Client also must enable `enable_flex_fec_support` in the `RunParams` to + // be able to use this feature. + PeerConfigurer* SetUseFlexFEC(bool value); + // Specifies how much video encoder target bitrate should be different than + // target bitrate, provided by WebRTC stack. Must be greater than 0. Can be + // used to emulate overshooting of video encoders. This multiplier will + // be applied for all video encoder on both sides for all layers. Bitrate + // estimated by WebRTC stack will be multiplied by this multiplier and then + // provided into VideoEncoder::SetRates(...). 1.0 by default. + PeerConfigurer* SetVideoEncoderBitrateMultiplier(double multiplier); + + // If is set, an RTCEventLog will be saved in that location and it will be + // available for further analysis. + PeerConfigurer* SetRtcEventLogPath(std::string path); + // If is set, an AEC dump will be saved in that location and it will be + // available for further analysis. + PeerConfigurer* SetAecDumpPath(std::string path); + PeerConfigurer* SetRTCConfiguration( + PeerConnectionInterface::RTCConfiguration configuration); + PeerConfigurer* SetRTCOfferAnswerOptions( + PeerConnectionInterface::RTCOfferAnswerOptions options); + // Set bitrate parameters on PeerConnection. This constraints will be + // applied to all summed RTP streams for this peer. + PeerConfigurer* SetBitrateSettings(BitrateSettings bitrate_settings); + + // Returns InjectableComponents and transfer ownership to the caller. + // Can be called once. + std::unique_ptr<InjectableComponents> ReleaseComponents(); + + // Returns Params and transfer ownership to the caller. + // Can be called once. + std::unique_ptr<Params> ReleaseParams(); + + // Returns ConfigurableParams and transfer ownership to the caller. + // Can be called once. + std::unique_ptr<ConfigurableParams> ReleaseConfigurableParams(); + + // Returns video sources and transfer frame generators ownership to the + // caller. Can be called once. + std::vector<VideoSource> ReleaseVideoSources(); + + InjectableComponents* components() { return components_.get(); } + Params* params() { return params_.get(); } + ConfigurableParams* configurable_params() { + return configurable_params_.get(); + } + const Params& params() const { return *params_; } + const ConfigurableParams& configurable_params() const { + return *configurable_params_; + } + std::vector<VideoSource>* video_sources() { return &video_sources_; } + + private: + std::unique_ptr<InjectableComponents> components_; + std::unique_ptr<Params> params_; + std::unique_ptr<ConfigurableParams> configurable_params_; + std::vector<VideoSource> video_sources_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_PCLF_PEER_CONFIGURER_H_ diff --git a/third_party/libwebrtc/api/test/peer_network_dependencies.h b/third_party/libwebrtc/api/test/peer_network_dependencies.h new file mode 100644 index 0000000000..6f85ad0a4d --- /dev/null +++ b/third_party/libwebrtc/api/test/peer_network_dependencies.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_PEER_NETWORK_DEPENDENCIES_H_ +#define API_TEST_PEER_NETWORK_DEPENDENCIES_H_ + +#include "api/packet_socket_factory.h" +#include "rtc_base/network.h" +#include "rtc_base/thread.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// The network dependencies needed when adding a peer to tests using +// PeerConnectionE2EQualityTestFixture. +struct PeerNetworkDependencies { + rtc::Thread* network_thread; + rtc::NetworkManager* network_manager; + rtc::PacketSocketFactory* packet_socket_factory; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_PEER_NETWORK_DEPENDENCIES_H_ diff --git a/third_party/libwebrtc/api/test/peerconnection_quality_test_fixture.h b/third_party/libwebrtc/api/test/peerconnection_quality_test_fixture.h new file mode 100644 index 0000000000..74470cdf86 --- /dev/null +++ b/third_party/libwebrtc/api/test/peerconnection_quality_test_fixture.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ +#define API_TEST_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <functional> +#include <map> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/async_resolver_factory.h" +#include "api/audio/audio_mixer.h" +#include "api/call/call_factory_interface.h" +#include "api/fec_controller.h" +#include "api/function_view.h" +#include "api/media_stream_interface.h" +#include "api/peer_connection_interface.h" +#include "api/rtc_event_log/rtc_event_log_factory_interface.h" +#include "api/rtp_parameters.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/test/audio_quality_analyzer_interface.h" +#include "api/test/frame_generator_interface.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/peer_network_dependencies.h" +#include "api/test/simulated_network.h" +#include "api/test/stats_observer_interface.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/test/video/video_frame_writer.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/transport/network_control.h" +#include "api/units/time_delta.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "media/base/media_constants.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/checks.h" +#include "rtc_base/network.h" +#include "rtc_base/rtc_certificate_generator.h" +#include "rtc_base/ssl_certificate.h" +#include "rtc_base/thread.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// API is in development. Can be changed/removed without notice. +class PeerConnectionE2EQualityTestFixture { + public: + // Represent an entity that will report quality metrics after test. + class QualityMetricsReporter : public StatsObserverInterface { + public: + virtual ~QualityMetricsReporter() = default; + + // Invoked by framework after peer connection factory and peer connection + // itself will be created but before offer/answer exchange will be started. + // `test_case_name` is name of test case, that should be used to report all + // metrics. + // `reporter_helper` is a pointer to a class that will allow track_id to + // stream_id matching. The caller is responsible for ensuring the + // TrackIdStreamInfoMap will be valid from Start() to + // StopAndReportResults(). + virtual void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) = 0; + + // Invoked by framework after call is ended and peer connection factory and + // peer connection are destroyed. + virtual void StopAndReportResults() = 0; + }; + + // Represents single participant in call and can be used to perform different + // in-call actions. Might be extended in future. + class PeerHandle { + public: + virtual ~PeerHandle() = default; + }; + + virtual ~PeerConnectionE2EQualityTestFixture() = default; + + // Add activity that will be executed on the best effort at least after + // `target_time_since_start` after call will be set up (after offer/answer + // exchange, ICE gathering will be done and ICE candidates will passed to + // remote side). `func` param is amount of time spent from the call set up. + virtual void ExecuteAt(TimeDelta target_time_since_start, + std::function<void(TimeDelta)> func) = 0; + // Add activity that will be executed every `interval` with first execution + // on the best effort at least after `initial_delay_since_start` after call + // will be set up (after all participants will be connected). `func` param is + // amount of time spent from the call set up. + virtual void ExecuteEvery(TimeDelta initial_delay_since_start, + TimeDelta interval, + std::function<void(TimeDelta)> func) = 0; + + // Add stats reporter entity to observe the test. + virtual void AddQualityMetricsReporter( + std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) = 0; + + // Add a new peer to the call and return an object through which caller + // can configure peer's behavior. + // `network_dependencies` are used to provide networking for peer's peer + // connection. Members must be non-null. + // `configurer` function will be used to configure peer in the call. + virtual PeerHandle* AddPeer(std::unique_ptr<PeerConfigurer> configurer) = 0; + + // Runs the media quality test, which includes setting up the call with + // configured participants, running it according to provided `run_params` and + // terminating it properly at the end. During call duration media quality + // metrics are gathered, which are then reported to stdout and (if configured) + // to the json/protobuf output file through the WebRTC perf test results + // reporting system. + virtual void Run(RunParams run_params) = 0; + + // Returns real test duration - the time of test execution measured during + // test. Client must call this method only after test is finished (after + // Run(...) method returned). Test execution time is time from end of call + // setup (offer/answer, ICE candidates exchange done and ICE connected) to + // start of call tear down (PeerConnection closed). + virtual TimeDelta GetRealTestDuration() const = 0; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/peerconnection_quality_test_fixture_unittest.cc b/third_party/libwebrtc/api/test/peerconnection_quality_test_fixture_unittest.cc new file mode 100644 index 0000000000..26ae8cf98f --- /dev/null +++ b/third_party/libwebrtc/api/test/peerconnection_quality_test_fixture_unittest.cc @@ -0,0 +1,142 @@ +/* + * Copyright 2022 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/peerconnection_quality_test_fixture.h" + +#include <vector> + +#include "absl/types/optional.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video/video_frame_writer.h" +#include "rtc_base/gunit.h" +#include "test/gmock.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::Eq; + +TEST(PclfVideoSubscriptionTest, + MaxFromSenderSpecEqualIndependentOfOtherFields) { + VideoResolution r1(VideoResolution::Spec::kMaxFromSender); + r1.set_width(1); + r1.set_height(2); + r1.set_fps(3); + VideoResolution r2(VideoResolution::Spec::kMaxFromSender); + r1.set_width(4); + r1.set_height(5); + r1.set_fps(6); + EXPECT_EQ(r1, r2); +} + +TEST(PclfVideoSubscriptionTest, WhenSpecIsNotSetFieldsAreCompared) { + VideoResolution test_resolution(/*width=*/1, /*height=*/2, + /*fps=*/3); + VideoResolution equal_resolution(/*width=*/1, /*height=*/2, + /*fps=*/3); + VideoResolution different_width(/*width=*/10, /*height=*/2, + /*fps=*/3); + VideoResolution different_height(/*width=*/1, /*height=*/20, + /*fps=*/3); + VideoResolution different_fps(/*width=*/1, /*height=*/20, + /*fps=*/30); + + EXPECT_EQ(test_resolution, equal_resolution); + EXPECT_NE(test_resolution, different_width); + EXPECT_NE(test_resolution, different_height); + EXPECT_NE(test_resolution, different_fps); +} + +TEST(PclfVideoSubscriptionTest, GetMaxResolutionForEmptyReturnsNullopt) { + absl::optional<VideoResolution> resolution = + VideoSubscription::GetMaxResolution(std::vector<VideoConfig>{}); + ASSERT_FALSE(resolution.has_value()); +} + +TEST(PclfVideoSubscriptionTest, GetMaxResolutionSelectMaxForEachDimention) { + VideoConfig max_width(/*width=*/1000, /*height=*/1, /*fps=*/1); + VideoConfig max_height(/*width=*/1, /*height=*/100, /*fps=*/1); + VideoConfig max_fps(/*width=*/1, /*height=*/1, /*fps=*/10); + + absl::optional<VideoResolution> resolution = + VideoSubscription::GetMaxResolution( + std::vector<VideoConfig>{max_width, max_height, max_fps}); + ASSERT_TRUE(resolution.has_value()); + EXPECT_EQ(resolution->width(), static_cast<size_t>(1000)); + EXPECT_EQ(resolution->height(), static_cast<size_t>(100)); + EXPECT_EQ(resolution->fps(), 10); +} + +struct TestVideoFrameWriter : public test::VideoFrameWriter { + public: + TestVideoFrameWriter(absl::string_view file_name_prefix, + const VideoResolution& resolution) + : file_name_prefix(file_name_prefix), resolution(resolution) {} + + bool WriteFrame(const VideoFrame& frame) override { return true; } + + void Close() override {} + + std::string file_name_prefix; + VideoResolution resolution; +}; + +TEST(VideoDumpOptionsTest, InputVideoWriterHasCorrectFileName) { + VideoResolution resolution(/*width=*/1280, /*height=*/720, /*fps=*/30); + + TestVideoFrameWriter* writer = nullptr; + VideoDumpOptions options("foo", /*sampling_modulo=*/1, + /*export_frame_ids=*/false, + /*video_frame_writer_factory=*/ + [&](absl::string_view file_name_prefix, + const VideoResolution& resolution) { + auto out = std::make_unique<TestVideoFrameWriter>( + file_name_prefix, resolution); + writer = out.get(); + return out; + }); + std::unique_ptr<test::VideoFrameWriter> created_writer = + options.CreateInputDumpVideoFrameWriter("alice-video", resolution); + + ASSERT_TRUE(writer != nullptr); + ASSERT_THAT(writer->file_name_prefix, + Eq(test::JoinFilename("foo", "alice-video_1280x720_30"))); + ASSERT_THAT(writer->resolution, Eq(resolution)); +} + +TEST(VideoDumpOptionsTest, OutputVideoWriterHasCorrectFileName) { + VideoResolution resolution(/*width=*/1280, /*height=*/720, /*fps=*/30); + + TestVideoFrameWriter* writer = nullptr; + VideoDumpOptions options("foo", /*sampling_modulo=*/1, + /*export_frame_ids=*/false, + /*video_frame_writer_factory=*/ + [&](absl::string_view file_name_prefix, + const VideoResolution& resolution) { + auto out = std::make_unique<TestVideoFrameWriter>( + file_name_prefix, resolution); + writer = out.get(); + return out; + }); + std::unique_ptr<test::VideoFrameWriter> created_writer = + options.CreateOutputDumpVideoFrameWriter("alice-video", "bob", + resolution); + + ASSERT_TRUE(writer != nullptr); + ASSERT_THAT(writer->file_name_prefix, + Eq(test::JoinFilename("foo", "alice-video_bob_1280x720_30"))); + ASSERT_THAT(writer->resolution, Eq(resolution)); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/simulated_network.h b/third_party/libwebrtc/api/test/simulated_network.h new file mode 100644 index 0000000000..04c5517c8d --- /dev/null +++ b/third_party/libwebrtc/api/test/simulated_network.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_SIMULATED_NETWORK_H_ +#define API_TEST_SIMULATED_NETWORK_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <deque> +#include <queue> +#include <vector> + +#include "absl/types/optional.h" +#include "rtc_base/random.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +struct PacketInFlightInfo { + PacketInFlightInfo(size_t size, int64_t send_time_us, uint64_t packet_id) + : size(size), send_time_us(send_time_us), packet_id(packet_id) {} + + size_t size; + int64_t send_time_us; + // Unique identifier for the packet in relation to other packets in flight. + uint64_t packet_id; +}; + +struct PacketDeliveryInfo { + static constexpr int kNotReceived = -1; + PacketDeliveryInfo(PacketInFlightInfo source, int64_t receive_time_us) + : receive_time_us(receive_time_us), packet_id(source.packet_id) {} + + bool operator==(const PacketDeliveryInfo& other) const { + return receive_time_us == other.receive_time_us && + packet_id == other.packet_id; + } + + int64_t receive_time_us; + uint64_t packet_id; +}; + +// BuiltInNetworkBehaviorConfig is a built-in network behavior configuration +// for built-in network behavior that will be used by WebRTC if no custom +// NetworkBehaviorInterface is provided. +struct BuiltInNetworkBehaviorConfig { + // Queue length in number of packets. + size_t queue_length_packets = 0; + // Delay in addition to capacity induced delay. + int queue_delay_ms = 0; + // Standard deviation of the extra delay. + int delay_standard_deviation_ms = 0; + // Link capacity in kbps. + int link_capacity_kbps = 0; + // Random packet loss. + int loss_percent = 0; + // If packets are allowed to be reordered. + bool allow_reordering = false; + // The average length of a burst of lost packets. + int avg_burst_loss_length = -1; + // Additional bytes to add to packet size. + int packet_overhead = 0; +}; + +// Interface that represents a Network behaviour. +// +// It is clients of this interface responsibility to enqueue and dequeue +// packets (based on the estimated delivery time expressed by +// NextDeliveryTimeUs). +// +// To enqueue packets, call EnqueuePacket: +// EXPECT_TRUE(network.EnqueuePacket( +// PacketInFlightInfo(/*size=*/1, /*send_time_us=*/0, /*packet_id=*/1))); +// +// To know when to call DequeueDeliverablePackets to pull packets out of the +// network, call NextDeliveryTimeUs and schedule a task to invoke +// DequeueDeliverablePackets (if not already scheduled). +// +// DequeueDeliverablePackets will return a vector of delivered packets, but this +// vector can be empty in case of extra delay. In such case, make sure to invoke +// NextDeliveryTimeUs and schedule a task to call DequeueDeliverablePackets for +// the next estimated delivery of packets. +// +// std::vector<PacketDeliveryInfo> delivered_packets = +// network.DequeueDeliverablePackets(/*receive_time_us=*/1000000); +class NetworkBehaviorInterface { + public: + // Enqueues a packet in the network and returns true if the action was + // successful, false otherwise (for example, because the network capacity has + // been saturated). If the return value is false, the packet should be + // considered as dropped and it will not be returned by future calls + // to DequeueDeliverablePackets. + // Packets enqueued will exit the network when DequeueDeliverablePackets is + // called and enough time has passed (see NextDeliveryTimeUs). + virtual bool EnqueuePacket(PacketInFlightInfo packet_info) = 0; + // Retrieves all packets that should be delivered by the given receive time. + // Not all the packets in the returned std::vector are actually delivered. + // In order to know the state of each packet it is necessary to check the + // `receive_time_us` field of each packet. If that is set to + // PacketDeliveryInfo::kNotReceived then the packet is considered lost in the + // network. + virtual std::vector<PacketDeliveryInfo> DequeueDeliverablePackets( + int64_t receive_time_us) = 0; + // Returns time in microseconds when caller should call + // DequeueDeliverablePackets to get the next set of delivered packets. It is + // possible that no packet will be delivered by that time (e.g. in case of + // random extra delay), in such case this method should be called again to get + // the updated estimated delivery time. + virtual absl::optional<int64_t> NextDeliveryTimeUs() const = 0; + virtual ~NetworkBehaviorInterface() = default; +}; + +// Class simulating a network link. This is a simple and naive solution just +// faking capacity and adding an extra transport delay in addition to the +// capacity introduced delay. +class SimulatedNetworkInterface : public NetworkBehaviorInterface { + public: + // Sets a new configuration. + virtual void SetConfig(const BuiltInNetworkBehaviorConfig& config) = 0; + virtual void UpdateConfig( + std::function<void(BuiltInNetworkBehaviorConfig*)> config_modifier) = 0; + // Pauses the network until `until_us`. This affects both delivery (calling + // DequeueDeliverablePackets before `until_us` results in an empty std::vector + // of packets) and capacity (the network is paused, so packets are not + // flowing and they will restart flowing at `until_us`). + virtual void PauseTransmissionUntil(int64_t until_us) = 0; +}; + +} // namespace webrtc + +#endif // API_TEST_SIMULATED_NETWORK_H_ diff --git a/third_party/libwebrtc/api/test/simulcast_test_fixture.h b/third_party/libwebrtc/api/test/simulcast_test_fixture.h new file mode 100644 index 0000000000..c7130d2909 --- /dev/null +++ b/third_party/libwebrtc/api/test/simulcast_test_fixture.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_SIMULCAST_TEST_FIXTURE_H_ +#define API_TEST_SIMULCAST_TEST_FIXTURE_H_ + +namespace webrtc { +namespace test { + +class SimulcastTestFixture { + public: + virtual ~SimulcastTestFixture() = default; + + virtual void TestKeyFrameRequestsOnAllStreams() = 0; + virtual void TestKeyFrameRequestsOnSpecificStreams() = 0; + virtual void TestPaddingAllStreams() = 0; + virtual void TestPaddingTwoStreams() = 0; + virtual void TestPaddingTwoStreamsOneMaxedOut() = 0; + virtual void TestPaddingOneStream() = 0; + virtual void TestPaddingOneStreamTwoMaxedOut() = 0; + virtual void TestSendAllStreams() = 0; + virtual void TestDisablingStreams() = 0; + virtual void TestActiveStreams() = 0; + virtual void TestSwitchingToOneStream() = 0; + virtual void TestSwitchingToOneOddStream() = 0; + virtual void TestSwitchingToOneSmallStream() = 0; + virtual void TestSpatioTemporalLayers333PatternEncoder() = 0; + virtual void TestSpatioTemporalLayers321PatternEncoder() = 0; + virtual void TestStrideEncodeDecode() = 0; + virtual void TestDecodeWidthHeightSet() = 0; + virtual void + TestEncoderInfoForDefaultTemporalLayerProfileHasFpsAllocation() = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_SIMULCAST_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/stats_observer_interface.h b/third_party/libwebrtc/api/test/stats_observer_interface.h new file mode 100644 index 0000000000..58d8f52d77 --- /dev/null +++ b/third_party/libwebrtc/api/test/stats_observer_interface.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_STATS_OBSERVER_INTERFACE_H_ +#define API_TEST_STATS_OBSERVER_INTERFACE_H_ + +#include "absl/strings/string_view.h" +#include "api/stats/rtc_stats_report.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// API is in development and can be changed without notice. +class StatsObserverInterface { + public: + virtual ~StatsObserverInterface() = default; + + // Method called when stats reports are available for the PeerConnection + // identified by `pc_label`. + virtual void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) = 0; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_STATS_OBSERVER_INTERFACE_H_ diff --git a/third_party/libwebrtc/api/test/test_dependency_factory.cc b/third_party/libwebrtc/api/test/test_dependency_factory.cc new file mode 100644 index 0000000000..41ad70cc3f --- /dev/null +++ b/third_party/libwebrtc/api/test/test_dependency_factory.cc @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/test_dependency_factory.h" + +#include <memory> +#include <utility> + +#include "rtc_base/checks.h" +#include "rtc_base/platform_thread_types.h" + +namespace webrtc { + +namespace { +// This checks everything in this file gets called on the same thread. It's +// static because it needs to look at the static methods too. +bool IsValidTestDependencyFactoryThread() { + const rtc::PlatformThreadRef main_thread = rtc::CurrentThreadRef(); + return rtc::IsThreadRefEqual(main_thread, rtc::CurrentThreadRef()); +} +} // namespace + +std::unique_ptr<TestDependencyFactory> TestDependencyFactory::instance_ = + nullptr; + +const TestDependencyFactory& TestDependencyFactory::GetInstance() { + RTC_DCHECK(IsValidTestDependencyFactoryThread()); + if (instance_ == nullptr) { + instance_ = std::make_unique<TestDependencyFactory>(); + } + return *instance_; +} + +void TestDependencyFactory::SetInstance( + std::unique_ptr<TestDependencyFactory> instance) { + RTC_DCHECK(IsValidTestDependencyFactoryThread()); + RTC_CHECK(instance_ == nullptr); + instance_ = std::move(instance); +} + +std::unique_ptr<VideoQualityTestFixtureInterface::InjectionComponents> +TestDependencyFactory::CreateComponents() const { + RTC_DCHECK(IsValidTestDependencyFactoryThread()); + return nullptr; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/test_dependency_factory.h b/third_party/libwebrtc/api/test/test_dependency_factory.h new file mode 100644 index 0000000000..29f00b8070 --- /dev/null +++ b/third_party/libwebrtc/api/test/test_dependency_factory.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_TEST_DEPENDENCY_FACTORY_H_ +#define API_TEST_TEST_DEPENDENCY_FACTORY_H_ + +#include <memory> + +#include "api/test/video_quality_test_fixture.h" + +namespace webrtc { + +// Override this class if to inject custom components into WebRTC tests. +// Not all WebRTC tests get their components from here, so you need to make +// sure the tests you want actually use this class. +// +// This class is not thread safe and you need to make call calls from the same +// (test main) thread. +class TestDependencyFactory { + public: + virtual ~TestDependencyFactory() = default; + + // The singleton MUST be stateless since tests execute in any order. It must + // be set before tests start executing. + static const TestDependencyFactory& GetInstance(); + static void SetInstance(std::unique_ptr<TestDependencyFactory> instance); + + // Returns the component a test should use. Returning nullptr means that the + // test is free to use whatever defaults it wants. The injection components + // themselves can be mutable, but we need to make new ones for every test that + // executes so state doesn't spread between tests. + virtual std::unique_ptr<VideoQualityTestFixtureInterface::InjectionComponents> + CreateComponents() const; + + private: + static std::unique_ptr<TestDependencyFactory> instance_; +}; + +} // namespace webrtc + +#endif // API_TEST_TEST_DEPENDENCY_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/time_controller.cc b/third_party/libwebrtc/api/test/time_controller.cc new file mode 100644 index 0000000000..364dbc235d --- /dev/null +++ b/third_party/libwebrtc/api/test/time_controller.cc @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/time_controller.h" + +namespace webrtc { +std::unique_ptr<TaskQueueFactory> TimeController::CreateTaskQueueFactory() { + class FactoryWrapper final : public TaskQueueFactory { + public: + explicit FactoryWrapper(TaskQueueFactory* inner_factory) + : inner_(inner_factory) {} + std::unique_ptr<TaskQueueBase, TaskQueueDeleter> CreateTaskQueue( + absl::string_view name, + Priority priority) const override { + return inner_->CreateTaskQueue(name, priority); + } + + private: + TaskQueueFactory* const inner_; + }; + return std::make_unique<FactoryWrapper>(GetTaskQueueFactory()); +} +bool TimeController::Wait(const std::function<bool()>& condition, + TimeDelta max_duration) { + // Step size is chosen to be short enough to not significantly affect latency + // in real time tests while being long enough to avoid adding too much load to + // the system. + const auto kStep = TimeDelta::Millis(5); + for (auto elapsed = TimeDelta::Zero(); elapsed < max_duration; + elapsed += kStep) { + if (condition()) + return true; + AdvanceTime(kStep); + } + return condition(); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/time_controller.h b/third_party/libwebrtc/api/test/time_controller.h new file mode 100644 index 0000000000..121f65cea9 --- /dev/null +++ b/third_party/libwebrtc/api/test/time_controller.h @@ -0,0 +1,89 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_TIME_CONTROLLER_H_ +#define API_TEST_TIME_CONTROLLER_H_ + +#include <functional> +#include <memory> +#include <string> + +#include "api/task_queue/task_queue_factory.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/yield_policy.h" +#include "rtc_base/thread.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +// Interface for controlling time progress. This allows us to execute test code +// in either real time or simulated time by using different implementation of +// this interface. +class TimeController { + public: + virtual ~TimeController() = default; + // Provides a clock instance that follows implementation defined time + // progress. + virtual Clock* GetClock() = 0; + // The returned factory will created task queues that runs in implementation + // defined time domain. + virtual TaskQueueFactory* GetTaskQueueFactory() = 0; + // Simple helper to create an owned factory that can be used as a parameter + // for PeerConnectionFactory. Note that this might depend on the underlying + // time controller and therfore must be destroyed before the time controller + // is destroyed. + std::unique_ptr<TaskQueueFactory> CreateTaskQueueFactory(); + + // Creates an rtc::Thread instance. If `socket_server` is nullptr, a default + // noop socket server is created. + // Returned thread is not null and started. + virtual std::unique_ptr<rtc::Thread> CreateThread( + const std::string& name, + std::unique_ptr<rtc::SocketServer> socket_server = nullptr) = 0; + + // Creates an rtc::Thread instance that ensure that it's set as the current + // thread. + virtual rtc::Thread* GetMainThread() = 0; + // Allow task queues and process threads created by this instance to execute + // for the given `duration`. + virtual void AdvanceTime(TimeDelta duration) = 0; + + // Waits until condition() == true, polling condition() in small time + // intervals. + // Returns true if condition() was evaluated to true before `max_duration` + // elapsed and false otherwise. + bool Wait(const std::function<bool()>& condition, + TimeDelta max_duration = TimeDelta::Seconds(5)); +}; + +// Interface for telling time, scheduling an event to fire at a particular time, +// and waiting for time to pass. +class ControlledAlarmClock { + public: + virtual ~ControlledAlarmClock() = default; + + // Gets a clock that tells the alarm clock's notion of time. + virtual Clock* GetClock() = 0; + + // Schedules the alarm to fire at `deadline`. + // An alarm clock only supports one deadline. Calls to `ScheduleAlarmAt` with + // an earlier deadline will reset the alarm to fire earlier.Calls to + // `ScheduleAlarmAt` with a later deadline are ignored. Returns true if the + // deadline changed, false otherwise. + virtual bool ScheduleAlarmAt(Timestamp deadline) = 0; + + // Sets the callback that should be run when the alarm fires. + virtual void SetCallback(std::function<void()> callback) = 0; + + // Waits for `duration` to pass, according to the alarm clock. + virtual void Sleep(TimeDelta duration) = 0; +}; + +} // namespace webrtc +#endif // API_TEST_TIME_CONTROLLER_H_ diff --git a/third_party/libwebrtc/api/test/track_id_stream_info_map.h b/third_party/libwebrtc/api/test/track_id_stream_info_map.h new file mode 100644 index 0000000000..b016de57a9 --- /dev/null +++ b/third_party/libwebrtc/api/test/track_id_stream_info_map.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_TRACK_ID_STREAM_INFO_MAP_H_ +#define API_TEST_TRACK_ID_STREAM_INFO_MAP_H_ + +#include <string> + +#include "absl/strings/string_view.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Instances of `TrackIdStreamInfoMap` provide bookkeeping capabilities that +// are useful to associate stats reports track_ids to the remote stream info. +class TrackIdStreamInfoMap { + public: + struct StreamInfo { + std::string receiver_peer; + std::string stream_label; + std::string sync_group; + }; + + virtual ~TrackIdStreamInfoMap() = default; + + // These methods must be called on the same thread where + // StatsObserverInterface::OnStatsReports is invoked. + + // Precondition: `track_id` must be already mapped to stream info. + virtual StreamInfo GetStreamInfoFromTrackId( + absl::string_view track_id) const = 0; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // API_TEST_TRACK_ID_STREAM_INFO_MAP_H_ diff --git a/third_party/libwebrtc/api/test/video/BUILD.gn b/third_party/libwebrtc/api/test/video/BUILD.gn new file mode 100644 index 0000000000..d24ffa5fa6 --- /dev/null +++ b/third_party/libwebrtc/api/test/video/BUILD.gn @@ -0,0 +1,31 @@ +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_library("function_video_factory") { + visibility = [ "*" ] + testonly = true + public = [ + "function_video_decoder_factory.h", + "function_video_encoder_factory.h", + ] + + deps = [ + "../../../rtc_base:checks", + "../../video_codecs:video_codecs_api", + ] +} + +rtc_library("video_frame_writer") { + visibility = [ "*" ] + testonly = true + public = [ "video_frame_writer.h" ] + + deps = [ "../../video:video_frame" ] +} diff --git a/third_party/libwebrtc/api/test/video/function_video_decoder_factory.h b/third_party/libwebrtc/api/test/video/function_video_decoder_factory.h new file mode 100644 index 0000000000..2145c71bff --- /dev/null +++ b/third_party/libwebrtc/api/test/video/function_video_decoder_factory.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_FUNCTION_VIDEO_DECODER_FACTORY_H_ +#define API_TEST_VIDEO_FUNCTION_VIDEO_DECODER_FACTORY_H_ + +#include <functional> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +// A decoder factory producing decoders by calling a supplied create function. +class FunctionVideoDecoderFactory final : public VideoDecoderFactory { + public: + explicit FunctionVideoDecoderFactory( + std::function<std::unique_ptr<VideoDecoder>()> create) + : create_([create = std::move(create)](const SdpVideoFormat&) { + return create(); + }) {} + explicit FunctionVideoDecoderFactory( + std::function<std::unique_ptr<VideoDecoder>(const SdpVideoFormat&)> + create) + : create_(std::move(create)) {} + FunctionVideoDecoderFactory( + std::function<std::unique_ptr<VideoDecoder>()> create, + std::vector<SdpVideoFormat> sdp_video_formats) + : create_([create = std::move(create)](const SdpVideoFormat&) { + return create(); + }), + sdp_video_formats_(std::move(sdp_video_formats)) {} + + std::vector<SdpVideoFormat> GetSupportedFormats() const override { + return sdp_video_formats_; + } + + std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) override { + return create_(format); + } + + private: + const std::function<std::unique_ptr<VideoDecoder>(const SdpVideoFormat&)> + create_; + const std::vector<SdpVideoFormat> sdp_video_formats_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_FUNCTION_VIDEO_DECODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/video/function_video_encoder_factory.h b/third_party/libwebrtc/api/test/video/function_video_encoder_factory.h new file mode 100644 index 0000000000..98ece2bc94 --- /dev/null +++ b/third_party/libwebrtc/api/test/video/function_video_encoder_factory.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_FUNCTION_VIDEO_ENCODER_FACTORY_H_ +#define API_TEST_VIDEO_FUNCTION_VIDEO_ENCODER_FACTORY_H_ + +#include <functional> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace test { + +// An encoder factory producing encoders by calling a supplied create +// function. +class FunctionVideoEncoderFactory final : public VideoEncoderFactory { + public: + explicit FunctionVideoEncoderFactory( + std::function<std::unique_ptr<VideoEncoder>()> create) + : create_([create = std::move(create)](const SdpVideoFormat&) { + return create(); + }) {} + explicit FunctionVideoEncoderFactory( + std::function<std::unique_ptr<VideoEncoder>(const SdpVideoFormat&)> + create) + : create_(std::move(create)) {} + + // Unused by tests. + std::vector<SdpVideoFormat> GetSupportedFormats() const override { + RTC_DCHECK_NOTREACHED(); + return {}; + } + + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override { + return create_(format); + } + + private: + const std::function<std::unique_ptr<VideoEncoder>(const SdpVideoFormat&)> + create_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_FUNCTION_VIDEO_ENCODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/test/video/video_frame_writer.h b/third_party/libwebrtc/api/test/video/video_frame_writer.h new file mode 100644 index 0000000000..ac72534890 --- /dev/null +++ b/third_party/libwebrtc/api/test/video/video_frame_writer.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_VIDEO_FRAME_WRITER_H_ +#define API_TEST_VIDEO_VIDEO_FRAME_WRITER_H_ + +#include "api/video/video_frame.h" + +namespace webrtc { +namespace test { + +class VideoFrameWriter { + public: + virtual ~VideoFrameWriter() = default; + + // Writes `VideoFrame` and returns true if operation was successful, false + // otherwise. + // + // Calling `WriteFrame` after `Close` is not allowed. + virtual bool WriteFrame(const VideoFrame& frame) = 0; + + // Closes writer and cleans up all resources. No invocations to `WriteFrame` + // are allowed after `Close` was invoked. + virtual void Close() = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_VIDEO_FRAME_WRITER_H_ diff --git a/third_party/libwebrtc/api/test/video_codec_tester.h b/third_party/libwebrtc/api/test/video_codec_tester.h new file mode 100644 index 0000000000..0eaaa1b895 --- /dev/null +++ b/third_party/libwebrtc/api/test/video_codec_tester.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_CODEC_TESTER_H_ +#define API_TEST_VIDEO_CODEC_TESTER_H_ + +#include <memory> + +#include "absl/functional/any_invocable.h" +#include "api/test/videocodec_test_stats.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video/video_frame.h" + +namespace webrtc { +namespace test { + +// Interface for a video codec tester. The interface provides minimalistic set +// of data structures that enables implementation of decode-only, encode-only +// and encode-decode tests. +class VideoCodecTester { + public: + // Pacing settings for codec input. + struct PacingSettings { + enum PacingMode { + // Pacing is not used. Frames are sent to codec back-to-back. + kNoPacing, + // Pace with the rate equal to the target video frame rate. Pacing time is + // derived from RTP timestamp. + kRealTime, + // Pace with the explicitly provided rate. + kConstantRate, + }; + PacingMode mode = PacingMode::kNoPacing; + // Pacing rate for `kConstantRate` mode. + Frequency constant_rate = Frequency::Zero(); + }; + + struct DecoderSettings { + PacingSettings pacing; + }; + + struct EncoderSettings { + PacingSettings pacing; + }; + + virtual ~VideoCodecTester() = default; + + // Interface for a raw video frames source. + class RawVideoSource { + public: + virtual ~RawVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional<VideoFrame> PullFrame() = 0; + + // Returns early pulled frame with RTP timestamp equal to `timestamp_rtp`. + virtual VideoFrame GetFrame(uint32_t timestamp_rtp, + Resolution resolution) = 0; + }; + + // Interface for a coded video frames source. + class CodedVideoSource { + public: + virtual ~CodedVideoSource() = default; + + // Returns next frame. If no more frames to pull, returns `absl::nullopt`. + // For analysis and pacing purposes, frame must have RTP timestamp set. The + // timestamp must represent the target video frame rate and be unique. + virtual absl::optional<EncodedImage> PullFrame() = 0; + }; + + // Interface for a video encoder. + class Encoder { + public: + using EncodeCallback = + absl::AnyInvocable<void(const EncodedImage& encoded_frame)>; + + virtual ~Encoder() = default; + + virtual void Encode(const VideoFrame& frame, EncodeCallback callback) = 0; + }; + + // Interface for a video decoder. + class Decoder { + public: + using DecodeCallback = + absl::AnyInvocable<void(const VideoFrame& decoded_frame)>; + + virtual ~Decoder() = default; + + virtual void Decode(const EncodedImage& frame, DecodeCallback callback) = 0; + }; + + // Pulls coded video frames from `video_source` and passes them to `decoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr<VideoCodecTestStats> RunDecodeTest( + std::unique_ptr<CodedVideoSource> video_source, + std::unique_ptr<Decoder> decoder, + const DecoderSettings& decoder_settings) = 0; + + // Pulls raw video frames from `video_source` and passes them to `encoder`. + // Returns `VideoCodecTestStats` object that contains collected per-frame + // metrics. + virtual std::unique_ptr<VideoCodecTestStats> RunEncodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + const EncoderSettings& encoder_settings) = 0; + + // Pulls raw video frames from `video_source`, passes them to `encoder` and + // then passes encoded frames to `decoder`. Returns `VideoCodecTestStats` + // object that contains collected per-frame metrics. + virtual std::unique_ptr<VideoCodecTestStats> RunEncodeDecodeTest( + std::unique_ptr<RawVideoSource> video_source, + std::unique_ptr<Encoder> encoder, + std::unique_ptr<Decoder> decoder, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEO_CODEC_TESTER_H_ diff --git a/third_party/libwebrtc/api/test/video_quality_analyzer_interface.h b/third_party/libwebrtc/api/test/video_quality_analyzer_interface.h new file mode 100644 index 0000000000..d35be8ca1a --- /dev/null +++ b/third_party/libwebrtc/api/test/video_quality_analyzer_interface.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_QUALITY_ANALYZER_INTERFACE_H_ +#define API_TEST_VIDEO_QUALITY_ANALYZER_INTERFACE_H_ + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/test/stats_observer_interface.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_encoder.h" + +namespace webrtc { + +// API is in development and can be changed without notice. + +// Base interface for video quality analyzer for peer connection level end-2-end +// tests. Interface has only one abstract method, which have to return frame id. +// Other methods have empty implementation by default, so user can override only +// required parts. +// +// VideoQualityAnalyzerInterface will be injected into WebRTC pipeline on both +// sides of the call. Here is video data flow in WebRTC pipeline +// +// Alice: +// ___________ ________ _________ +// | | | | | | +// | Frame |-(A)→| WebRTC |-(B)→| Video |-(C)┐ +// | Generator | | Stack | | Decoder | | +// ¯¯¯¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯¯ | +// __↓________ +// | Transport | +// | & | +// | Network | +// ¯¯|¯¯¯¯¯¯¯¯ +// Bob: | +// _______ ________ _________ | +// | | | | | | | +// | Video |←(F)-| WebRTC |←(E)-| Video |←(D)----┘ +// | Sink | | Stack | | Decoder | +// ¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯¯ +// The analyzer will be injected in all points from A to F. +class VideoQualityAnalyzerInterface + : public webrtc_pc_e2e::StatsObserverInterface { + public: + // Contains extra statistic provided by video encoder. + struct EncoderStats { + std::string encoder_name = "unknown"; + // TODO(hbos) https://crbug.com/webrtc/9547, + // https://crbug.com/webrtc/11443: improve stats API to make available + // there. + uint32_t target_encode_bitrate = 0; + // Encoder quantizer value. + int qp = -1; + }; + // Contains extra statistic provided by video decoder. + struct DecoderStats { + std::string decoder_name = "unknown"; + // Decode time provided by decoder itself. If decoder doesn’t produce such + // information can be omitted. + absl::optional<int32_t> decode_time_ms = absl::nullopt; + }; + + ~VideoQualityAnalyzerInterface() override = default; + + // Will be called by framework before test. + // `test_case_name` is name of test case, that should be used to report all + // video metrics. + // `threads_count` is number of threads that analyzer can use for heavy + // calculations. Analyzer can perform simple calculations on the calling + // thread in each method, but should remember, that it is the same thread, + // that is used in video pipeline. + virtual void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) {} + + // Will be called when frame was generated from the input stream. + // `peer_name` is name of the peer on which side frame was captured. + // Returns frame id, that will be set by framework to the frame. + virtual uint16_t OnFrameCaptured(absl::string_view peer_name, + const std::string& stream_label, + const VideoFrame& frame) = 0; + // Will be called before calling the encoder. + // `peer_name` is name of the peer on which side frame came to encoder. + virtual void OnFramePreEncode(absl::string_view peer_name, + const VideoFrame& frame) {} + // Will be called for each EncodedImage received from encoder. Single + // VideoFrame can produce multiple EncodedImages. Each encoded image will + // have id from VideoFrame. + // `peer_name` is name of the peer on which side frame was encoded. + virtual void OnFrameEncoded(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) {} + // Will be called for each frame dropped by encoder. + // `peer_name` is name of the peer on which side frame drop was detected. + virtual void OnFrameDropped(absl::string_view peer_name, + EncodedImageCallback::DropReason reason) {} + // Will be called before calling the decoder. + // `peer_name` is name of the peer on which side frame was received. + virtual void OnFramePreDecode(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image) {} + // Will be called after decoding the frame. + // `peer_name` is name of the peer on which side frame was decoded. + virtual void OnFrameDecoded(absl::string_view peer_name, + const VideoFrame& frame, + const DecoderStats& stats) {} + // Will be called when frame will be obtained from PeerConnection stack. + // `peer_name` is name of the peer on which side frame was rendered. + virtual void OnFrameRendered(absl::string_view peer_name, + const VideoFrame& frame) {} + // Will be called if encoder return not WEBRTC_VIDEO_CODEC_OK. + // All available codes are listed in + // modules/video_coding/include/video_error_codes.h + // `peer_name` is name of the peer on which side error acquired. + virtual void OnEncoderError(absl::string_view peer_name, + const VideoFrame& frame, + int32_t error_code) {} + // Will be called if decoder return not WEBRTC_VIDEO_CODEC_OK. + // All available codes are listed in + // modules/video_coding/include/video_error_codes.h + // `peer_name` is name of the peer on which side error acquired. + virtual void OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) {} + // Will be called every time new stats reports are available for the + // Peer Connection identified by `pc_label`. + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override {} + + // Will be called before test adds new participant in the middle of a call. + virtual void RegisterParticipantInCall(absl::string_view peer_name) {} + // Will be called after test removed existing participant in the middle of the + // call. + virtual void UnregisterParticipantInCall(absl::string_view peer_name) {} + + // Tells analyzer that analysis complete and it should calculate final + // statistics. + virtual void Stop() {} + + // Returns the last stream where this frame was captured. It means that if + // frame ids space wraps around, then stream label for frame id may change. + // It will crash, if the specified `frame_id` wasn't captured. + virtual std::string GetStreamLabel(uint16_t frame_id) = 0; +}; + +} // namespace webrtc + +#endif // API_TEST_VIDEO_QUALITY_ANALYZER_INTERFACE_H_ diff --git a/third_party/libwebrtc/api/test/video_quality_test_fixture.h b/third_party/libwebrtc/api/test/video_quality_test_fixture.h new file mode 100644 index 0000000000..b45faef286 --- /dev/null +++ b/third_party/libwebrtc/api/test/video_quality_test_fixture.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_ +#define API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "api/fec_controller.h" +#include "api/media_types.h" +#include "api/network_state_predictor.h" +#include "api/test/simulated_network.h" +#include "api/transport/bitrate_settings.h" +#include "api/transport/network_control.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "video/config/video_encoder_config.h" + +namespace webrtc { + +class VideoQualityTestFixtureInterface { + public: + // Parameters are grouped into smaller structs to make it easier to set + // the desired elements and skip unused. + struct Params { + struct CallConfig { + bool send_side_bwe = false; + bool generic_descriptor = false; + bool dependency_descriptor = false; + BitrateConstraints call_bitrate_config; + int num_thumbnails = 0; + // Indicates if secondary_(video|ss|screenshare) structures are used. + bool dual_video = false; + } call; + struct Video { + bool enabled = false; + size_t width = 640; + size_t height = 480; + int32_t fps = 30; + int min_bitrate_bps = 50; + int target_bitrate_bps = 800; + int max_bitrate_bps = 800; + bool suspend_below_min_bitrate = false; + std::string codec = "VP8"; + int num_temporal_layers = 1; + int selected_tl = -1; + int min_transmit_bps = 0; + bool ulpfec = false; + bool flexfec = false; + bool automatic_scaling = false; + std::string clip_path; // "Generator" to generate frames instead. + size_t capture_device_index = 0; + SdpVideoFormat::Parameters sdp_params; + double encoder_overshoot_factor = 0.0; + } video[2]; + struct Audio { + bool enabled = false; + bool sync_video = false; + bool dtx = false; + bool use_real_adm = false; + absl::optional<std::string> ana_config; + } audio; + struct Screenshare { + bool enabled = false; + bool generate_slides = false; + int32_t slide_change_interval = 10; + int32_t scroll_duration = 0; + std::vector<std::string> slides; + } screenshare[2]; + struct Analyzer { + std::string test_label; + double avg_psnr_threshold = 0.0; // (*) + double avg_ssim_threshold = 0.0; // (*) + int test_durations_secs = 0; + std::string graph_data_output_filename; + std::string graph_title; + } analyzer; + // Config for default simulation implementation. Must be nullopt if + // `sender_network` and `receiver_network` in InjectionComponents are + // non-null. May be nullopt even if `sender_network` and `receiver_network` + // are null; in that case, a default config will be used. + absl::optional<BuiltInNetworkBehaviorConfig> config; + struct SS { // Spatial scalability. + std::vector<VideoStream> streams; // If empty, one stream is assumed. + size_t selected_stream = 0; + int num_spatial_layers = 0; + int selected_sl = -1; + InterLayerPredMode inter_layer_pred = InterLayerPredMode::kOn; + // If empty, bitrates are generated in VP9Impl automatically. + std::vector<SpatialLayer> spatial_layers; + // If set, default parameters will be used instead of `streams`. + bool infer_streams = false; + } ss[2]; + struct Logging { + std::string rtc_event_log_name; + std::string rtp_dump_name; + std::string encoded_frame_base_path; + } logging; + }; + + // Contains objects, that will be injected on different layers of test + // framework to override the behavior of system parts. + struct InjectionComponents { + InjectionComponents(); + ~InjectionComponents(); + + // Simulations of sender and receiver networks. They must either both be + // null (in which case `config` from Params is used), or both be non-null + // (in which case `config` from Params must be nullopt). + std::unique_ptr<NetworkBehaviorInterface> sender_network; + std::unique_ptr<NetworkBehaviorInterface> receiver_network; + + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory; + std::unique_ptr<VideoEncoderFactory> video_encoder_factory; + std::unique_ptr<VideoDecoderFactory> video_decoder_factory; + std::unique_ptr<NetworkStatePredictorFactoryInterface> + network_state_predictor_factory; + std::unique_ptr<NetworkControllerFactoryInterface> + network_controller_factory; + }; + + virtual ~VideoQualityTestFixtureInterface() = default; + + virtual void RunWithAnalyzer(const Params& params) = 0; + virtual void RunWithRenderers(const Params& params) = 0; + + virtual const std::map<uint8_t, webrtc::MediaType>& payload_type_map() = 0; +}; + +} // namespace webrtc + +#endif // API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/videocodec_test_fixture.h b/third_party/libwebrtc/api/test/videocodec_test_fixture.h new file mode 100644 index 0000000000..dbf20993e2 --- /dev/null +++ b/third_party/libwebrtc/api/test/videocodec_test_fixture.h @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ +#define API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ + +#include <string> +#include <vector> + +#include "api/test/videocodec_test_stats.h" +#include "api/video_codecs/h264_profile_level_id.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/include/video_codec_interface.h" + +namespace webrtc { +namespace test { + +// Rates for the encoder and the frame number when to apply profile. +struct RateProfile { + size_t target_kbps; + double input_fps; + size_t frame_num; +}; + +struct RateControlThresholds { + double max_avg_bitrate_mismatch_percent; + double max_time_to_reach_target_bitrate_sec; + // TODO(ssilkin): Use absolute threshold for framerate. + double max_avg_framerate_mismatch_percent; + double max_avg_buffer_level_sec; + double max_max_key_frame_delay_sec; + double max_max_delta_frame_delay_sec; + size_t max_num_spatial_resizes; + size_t max_num_key_frames; +}; + +struct QualityThresholds { + double min_avg_psnr; + double min_min_psnr; + double min_avg_ssim; + double min_min_ssim; +}; + +struct BitstreamThresholds { + size_t max_max_nalu_size_bytes; +}; + +// NOTE: This class is still under development and may change without notice. +class VideoCodecTestFixture { + public: + class EncodedFrameChecker { + public: + virtual ~EncodedFrameChecker() = default; + virtual void CheckEncodedFrame(VideoCodecType codec, + const EncodedImage& encoded_frame) const = 0; + }; + + struct Config { + Config(); + void SetCodecSettings(std::string codec_name, + size_t num_simulcast_streams, + size_t num_spatial_layers, + size_t num_temporal_layers, + bool denoising_on, + bool frame_dropper_on, + bool spatial_resize_on, + size_t width, + size_t height); + + size_t NumberOfCores() const; + size_t NumberOfTemporalLayers() const; + size_t NumberOfSpatialLayers() const; + size_t NumberOfSimulcastStreams() const; + + std::string ToString() const; + std::string CodecName() const; + + // Name of this config, to be used for accounting by the test runner. + std::string test_name; + + // Plain name of YUV file to process without file extension. + std::string filename; + // Dimensions of test clip. Falls back to (codec_settings.width/height) if + // not set. + absl::optional<int> clip_width; + absl::optional<int> clip_height; + // Framerate of input clip. Defaults to 30fps if not set. + absl::optional<int> clip_fps; + + // The resolution at which psnr/ssim comparisons should be made. Frames + // will be scaled to this size if different. + absl::optional<int> reference_width; + absl::optional<int> reference_height; + + // File to process. This must be a video file in the YUV format. + std::string filepath; + + // Number of frames to process. + size_t num_frames = 0; + + // Bitstream constraints. + size_t max_payload_size_bytes = 1440; + + // Should we decode the encoded frames? + bool decode = true; + + // Force the encoder and decoder to use a single core for processing. + bool use_single_core = false; + + // Should cpu usage be measured? + // If set to true, the encoding will run in real-time. + bool measure_cpu = false; + + // Simulate frames arriving in real-time by adding delays between frames. + bool encode_in_real_time = false; + + // Codec settings to use. + VideoCodec codec_settings; + + // Name of the codec being tested. + std::string codec_name; + + // Encoder and decoder format and parameters. If provided, format is used to + // instantiate the codec. If not provided, the test creates and uses the + // default `SdpVideoFormat` based on `codec_name`. + // Encoder and decoder name (`SdpVideoFormat::name`) should be the same as + // `codec_name`. + absl::optional<SdpVideoFormat> encoder_format; + absl::optional<SdpVideoFormat> decoder_format; + + // H.264 specific settings. + struct H264CodecSettings { + H264Profile profile = H264Profile::kProfileConstrainedBaseline; + H264PacketizationMode packetization_mode = + H264PacketizationMode::NonInterleaved; + } h264_codec_settings; + + // Custom checker that will be called for each frame. + const EncodedFrameChecker* encoded_frame_checker = nullptr; + + // Print out frame level stats. + bool print_frame_level_stats = false; + + // Path to a directory where encoded or/and decoded video should be saved. + std::string output_path; + + // Should video be saved persistently to disk for post-run visualization? + struct VisualizationParams { + bool save_encoded_ivf = false; + bool save_decoded_y4m = false; + } visualization_params; + + // Enables quality analysis for dropped frames. + bool analyze_quality_of_dropped_frames = false; + }; + + virtual ~VideoCodecTestFixture() = default; + + virtual void RunTest(const std::vector<RateProfile>& rate_profiles, + const std::vector<RateControlThresholds>* rc_thresholds, + const std::vector<QualityThresholds>* quality_thresholds, + const BitstreamThresholds* bs_thresholds) = 0; + virtual VideoCodecTestStats& GetStats() = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEOCODEC_TEST_FIXTURE_H_ diff --git a/third_party/libwebrtc/api/test/videocodec_test_stats.cc b/third_party/libwebrtc/api/test/videocodec_test_stats.cc new file mode 100644 index 0000000000..f082b1e935 --- /dev/null +++ b/third_party/libwebrtc/api/test/videocodec_test_stats.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/videocodec_test_stats.h" + +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace test { + +VideoCodecTestStats::FrameStatistics::FrameStatistics(size_t frame_number, + size_t rtp_timestamp, + size_t spatial_idx) + : frame_number(frame_number), + rtp_timestamp(rtp_timestamp), + spatial_idx(spatial_idx) {} + +std::string VideoCodecTestStats::FrameStatistics::ToString() const { + rtc::StringBuilder ss; + for (const auto& entry : ToMap()) { + if (ss.size() > 0) { + ss << " "; + } + ss << entry.first << " " << entry.second; + } + return ss.Release(); +} + +std::map<std::string, std::string> VideoCodecTestStats::FrameStatistics::ToMap() + const { + std::map<std::string, std::string> map; + map["frame_number"] = std::to_string(frame_number); + map["decoded_width"] = std::to_string(decoded_width); + map["decoded_height"] = std::to_string(decoded_height); + map["spatial_idx"] = std::to_string(spatial_idx); + map["temporal_idx"] = std::to_string(temporal_idx); + map["inter_layer_predicted"] = std::to_string(inter_layer_predicted); + map["non_ref_for_inter_layer_pred"] = + std::to_string(non_ref_for_inter_layer_pred); + map["frame_type"] = std::to_string(static_cast<int>(frame_type)); + map["length_bytes"] = std::to_string(length_bytes); + map["qp"] = std::to_string(qp); + map["psnr"] = std::to_string(psnr); + map["psnr_y"] = std::to_string(psnr_y); + map["psnr_u"] = std::to_string(psnr_u); + map["psnr_v"] = std::to_string(psnr_v); + map["ssim"] = std::to_string(ssim); + map["encode_time_us"] = std::to_string(encode_time_us); + map["decode_time_us"] = std::to_string(decode_time_us); + map["rtp_timestamp"] = std::to_string(rtp_timestamp); + map["target_bitrate_kbps"] = std::to_string(target_bitrate_kbps); + map["target_framerate_fps"] = std::to_string(target_framerate_fps); + return map; +} + +std::string VideoCodecTestStats::VideoStatistics::ToString( + std::string prefix) const { + rtc::StringBuilder ss; + for (const auto& entry : ToMap()) { + if (ss.size() > 0) { + ss << "\n"; + } + ss << prefix << entry.first << ": " << entry.second; + } + return ss.Release(); +} + +std::map<std::string, std::string> VideoCodecTestStats::VideoStatistics::ToMap() + const { + std::map<std::string, std::string> map; + map["target_bitrate_kbps"] = std::to_string(target_bitrate_kbps); + map["input_framerate_fps"] = std::to_string(input_framerate_fps); + map["spatial_idx"] = std::to_string(spatial_idx); + map["temporal_idx"] = std::to_string(temporal_idx); + map["width"] = std::to_string(width); + map["height"] = std::to_string(height); + map["length_bytes"] = std::to_string(length_bytes); + map["bitrate_kbps"] = std::to_string(bitrate_kbps); + map["framerate_fps"] = std::to_string(framerate_fps); + map["enc_speed_fps"] = std::to_string(enc_speed_fps); + map["dec_speed_fps"] = std::to_string(dec_speed_fps); + map["avg_encode_latency_sec"] = std::to_string(avg_encode_latency_sec); + map["max_encode_latency_sec"] = std::to_string(max_encode_latency_sec); + map["avg_decode_latency_sec"] = std::to_string(avg_decode_latency_sec); + map["max_decode_latency_sec"] = std::to_string(max_decode_latency_sec); + map["avg_delay_sec"] = std::to_string(avg_delay_sec); + map["max_key_frame_delay_sec"] = std::to_string(max_key_frame_delay_sec); + map["max_delta_frame_delay_sec"] = std::to_string(max_delta_frame_delay_sec); + map["time_to_reach_target_bitrate_sec"] = + std::to_string(time_to_reach_target_bitrate_sec); + map["avg_bitrate_mismatch_pct"] = std::to_string(avg_bitrate_mismatch_pct); + map["avg_framerate_mismatch_pct"] = + std::to_string(avg_framerate_mismatch_pct); + map["avg_key_frame_size_bytes"] = std::to_string(avg_key_frame_size_bytes); + map["avg_delta_frame_size_bytes"] = + std::to_string(avg_delta_frame_size_bytes); + map["avg_qp"] = std::to_string(avg_qp); + map["avg_psnr"] = std::to_string(avg_psnr); + map["min_psnr"] = std::to_string(min_psnr); + map["avg_ssim"] = std::to_string(avg_ssim); + map["min_ssim"] = std::to_string(min_ssim); + map["num_input_frames"] = std::to_string(num_input_frames); + map["num_encoded_frames"] = std::to_string(num_encoded_frames); + map["num_decoded_frames"] = std::to_string(num_decoded_frames); + map["num_dropped_frames"] = + std::to_string(num_input_frames - num_encoded_frames); + map["num_key_frames"] = std::to_string(num_key_frames); + map["num_spatial_resizes"] = std::to_string(num_spatial_resizes); + map["max_nalu_size_bytes"] = std::to_string(max_nalu_size_bytes); + return map; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/api/test/videocodec_test_stats.h b/third_party/libwebrtc/api/test/videocodec_test_stats.h new file mode 100644 index 0000000000..12c60638db --- /dev/null +++ b/third_party/libwebrtc/api/test/videocodec_test_stats.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEOCODEC_TEST_STATS_H_ +#define API_TEST_VIDEOCODEC_TEST_STATS_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <map> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/units/data_rate.h" +#include "api/units/frequency.h" +#include "api/video/video_frame_type.h" + +namespace webrtc { +namespace test { + +// Statistics for a sequence of processed frames. This class is not thread safe. +class VideoCodecTestStats { + public: + // Statistics for one processed frame. + struct FrameStatistics { + FrameStatistics(size_t frame_number, + size_t rtp_timestamp, + size_t spatial_idx); + + std::string ToString() const; + + // Returns name -> value text map of frame statistics. + std::map<std::string, std::string> ToMap() const; + + size_t frame_number = 0; + size_t rtp_timestamp = 0; + + // Encoding. + int64_t encode_start_ns = 0; + int encode_return_code = 0; + bool encoding_successful = false; + size_t encode_time_us = 0; + size_t target_bitrate_kbps = 0; + double target_framerate_fps = 0.0; + size_t length_bytes = 0; + VideoFrameType frame_type = VideoFrameType::kVideoFrameDelta; + + // Layering. + size_t spatial_idx = 0; + size_t temporal_idx = 0; + bool inter_layer_predicted = false; + bool non_ref_for_inter_layer_pred = true; + + // H264 specific. + size_t max_nalu_size_bytes = 0; + + // Decoding. + int64_t decode_start_ns = 0; + int decode_return_code = 0; + bool decoding_successful = false; + size_t decode_time_us = 0; + size_t decoded_width = 0; + size_t decoded_height = 0; + + // Quantization. + int qp = -1; + + // Quality. + bool quality_analysis_successful = false; + float psnr_y = 0.0f; + float psnr_u = 0.0f; + float psnr_v = 0.0f; + float psnr = 0.0f; // 10 * log10(255^2 / (mse_y + mse_u + mse_v)). + float ssim = 0.0f; // 0.8 * ssim_y + 0.1 * (ssim_u + ssim_v). + }; + + struct VideoStatistics { + std::string ToString(std::string prefix) const; + + // Returns name -> value text map of video statistics. + std::map<std::string, std::string> ToMap() const; + + size_t target_bitrate_kbps = 0; + float input_framerate_fps = 0.0f; + + size_t spatial_idx = 0; + size_t temporal_idx = 0; + + size_t width = 0; + size_t height = 0; + + size_t length_bytes = 0; + size_t bitrate_kbps = 0; + float framerate_fps = 0; + + float enc_speed_fps = 0.0f; + float dec_speed_fps = 0.0f; + + float avg_encode_latency_sec = 0.0f; + float max_encode_latency_sec = 0.0f; + float avg_decode_latency_sec = 0.0f; + float max_decode_latency_sec = 0.0f; + + float avg_delay_sec = 0.0f; + float max_key_frame_delay_sec = 0.0f; + float max_delta_frame_delay_sec = 0.0f; + float time_to_reach_target_bitrate_sec = 0.0f; + float avg_bitrate_mismatch_pct = 0.0f; + float avg_framerate_mismatch_pct = 0.0f; + + float avg_key_frame_size_bytes = 0.0f; + float avg_delta_frame_size_bytes = 0.0f; + float avg_qp = 0.0f; + + float avg_psnr_y = 0.0f; + float avg_psnr_u = 0.0f; + float avg_psnr_v = 0.0f; + float avg_psnr = 0.0f; + float min_psnr = 0.0f; + float avg_ssim = 0.0f; + float min_ssim = 0.0f; + + size_t num_input_frames = 0; + size_t num_encoded_frames = 0; + size_t num_decoded_frames = 0; + size_t num_key_frames = 0; + size_t num_spatial_resizes = 0; + size_t max_nalu_size_bytes = 0; + }; + + virtual ~VideoCodecTestStats() = default; + + virtual std::vector<FrameStatistics> GetFrameStatistics() const = 0; + + virtual std::vector<VideoStatistics> SliceAndCalcLayerVideoStatistic( + size_t first_frame_num, + size_t last_frame_num) = 0; + + virtual VideoStatistics CalcVideoStatistic(size_t first_frame, + size_t last_frame, + DataRate target_bitrate, + Frequency target_framerate) = 0; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_VIDEOCODEC_TEST_STATS_H_ |