diff options
Diffstat (limited to 'third_party/libwebrtc/rtc_tools/frame_analyzer')
20 files changed, 2621 insertions, 0 deletions
diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/frame_analyzer.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/frame_analyzer.cc new file mode 100644 index 0000000000..501a6142a8 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/frame_analyzer.cc @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <stdio.h> +#include <stdlib.h> + +#include <cstddef> +#include <string> +#include <vector> + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/strings/match.h" +#include "api/scoped_refptr.h" +#include "api/test/metrics/chrome_perf_dashboard_metrics_exporter.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/metrics/metrics_exporter.h" +#include "api/test/metrics/stdout_metrics_exporter.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_tools/frame_analyzer/video_color_aligner.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" +#include "rtc_tools/frame_analyzer/video_temporal_aligner.h" +#include "rtc_tools/video_file_reader.h" +#include "rtc_tools/video_file_writer.h" + +ABSL_FLAG(int32_t, width, -1, "The width of the reference and test files"); +ABSL_FLAG(int32_t, height, -1, "The height of the reference and test files"); +ABSL_FLAG(std::string, + label, + "MY_TEST", + "The label to use for the perf output"); +ABSL_FLAG(std::string, + reference_file, + "ref.yuv", + "The reference YUV file to run the analysis against"); +ABSL_FLAG(std::string, + test_file, + "test.yuv", + "The test YUV file to run the analysis for"); +ABSL_FLAG(std::string, + aligned_output_file, + "", + "Where to write aligned YUV/Y4M output file, f not present, no files " + "will be written"); +ABSL_FLAG(std::string, + yuv_directory, + "", + "Where to write aligned YUV ref+test output files, if not present, " + "no files will be written"); +ABSL_FLAG(std::string, + chartjson_result_file, + "", + "Where to store perf result in chartjson format, if not present, no " + "perf result will be stored"); + +namespace { + +#ifdef WIN32 +const char* const kPathDelimiter = "\\"; +#else +const char* const kPathDelimiter = "/"; +#endif + +std::string JoinFilename(std::string directory, std::string filename) { + return directory + kPathDelimiter + filename; +} + +} // namespace + +/* + * A command line tool running PSNR and SSIM on a reference video and a test + * video. The test video is a record of the reference video which can start at + * an arbitrary point. It is possible that there will be repeated frames or + * skipped frames as well. The video files should be I420 .y4m or .yuv videos. + * If both files are .y4m, it's not needed to specify width/height. The tool + * prints the result to standard output in the Chromium perf format: + * RESULT <metric>:<label>= <values> + * + * The max value for PSNR is 48.0 (between equal frames), as for SSIM it is 1.0. + * + * Usage: + * frame_analyzer --label=<test_label> --reference_file=<name_of_file> + * --test_file_ref=<name_of_file> --width=<frame_width> --height=<frame_height> + */ +int main(int argc, char* argv[]) { + absl::ParseCommandLine(argc, argv); + + int width = absl::GetFlag(FLAGS_width); + int height = absl::GetFlag(FLAGS_height); + const std::string reference_file_name = absl::GetFlag(FLAGS_reference_file); + const std::string test_file_name = absl::GetFlag(FLAGS_test_file); + + // .yuv files require explicit resolution. + if ((absl::EndsWith(reference_file_name, ".yuv") || + absl::EndsWith(test_file_name, ".yuv")) && + (width <= 0 || height <= 0)) { + fprintf(stderr, + "Error: You need to specify width and height when using .yuv " + "files\n"); + return -1; + } + + webrtc::test::ResultsContainer results; + + rtc::scoped_refptr<webrtc::test::Video> reference_video = + webrtc::test::OpenYuvOrY4mFile(reference_file_name, width, height); + rtc::scoped_refptr<webrtc::test::Video> test_video = + webrtc::test::OpenYuvOrY4mFile(test_file_name, width, height); + + if (!reference_video || !test_video) { + fprintf(stderr, "Error opening video files\n"); + return 1; + } + + const std::vector<size_t> matching_indices = + webrtc::test::FindMatchingFrameIndices(reference_video, test_video); + + // Align the reference video both temporally and geometrically. I.e. align the + // frames to match up in order to the test video, and align a crop region of + // the reference video to match up to the test video. + const rtc::scoped_refptr<webrtc::test::Video> aligned_reference_video = + AdjustCropping(ReorderVideo(reference_video, matching_indices), + test_video); + + // Calculate if there is any systematic color difference between the reference + // and test video. + const webrtc::test::ColorTransformationMatrix color_transformation = + CalculateColorTransformationMatrix(aligned_reference_video, test_video); + + char buf[256]; + rtc::SimpleStringBuilder string_builder(buf); + for (int i = 0; i < 3; ++i) { + string_builder << "\n"; + for (int j = 0; j < 4; ++j) + string_builder.AppendFormat("%6.2f ", color_transformation[i][j]); + } + printf("Adjusting test video with color transformation: %s\n", + string_builder.str()); + + // Adjust all frames in the test video with the calculated color + // transformation. + const rtc::scoped_refptr<webrtc::test::Video> color_adjusted_test_video = + AdjustColors(color_transformation, test_video); + + results.frames = webrtc::test::RunAnalysis( + aligned_reference_video, color_adjusted_test_video, matching_indices); + + const std::vector<webrtc::test::Cluster> clusters = + webrtc::test::CalculateFrameClusters(matching_indices); + results.max_repeated_frames = webrtc::test::GetMaxRepeatedFrames(clusters); + results.max_skipped_frames = webrtc::test::GetMaxSkippedFrames(clusters); + results.total_skipped_frames = + webrtc::test::GetTotalNumberOfSkippedFrames(clusters); + results.decode_errors_ref = 0; + results.decode_errors_test = 0; + + webrtc::test::PrintAnalysisResults(absl::GetFlag(FLAGS_label), results, + *webrtc::test::GetGlobalMetricsLogger()); + + std::vector<std::unique_ptr<webrtc::test::MetricsExporter>> exporters; + exporters.push_back(std::make_unique<webrtc::test::StdoutMetricsExporter>()); + std::string chartjson_result_file = + absl::GetFlag(FLAGS_chartjson_result_file); + if (!chartjson_result_file.empty()) { + exporters.push_back( + std::make_unique<webrtc::test::ChromePerfDashboardMetricsExporter>( + chartjson_result_file)); + } + if (!webrtc::test::ExportPerfMetric(*webrtc::test::GetGlobalMetricsLogger(), + std::move(exporters))) { + return 1; + } + std::string aligned_output_file = absl::GetFlag(FLAGS_aligned_output_file); + if (!aligned_output_file.empty()) { + webrtc::test::WriteVideoToFile(aligned_reference_video, aligned_output_file, + /*fps=*/30); + } + std::string yuv_directory = absl::GetFlag(FLAGS_yuv_directory); + if (!yuv_directory.empty()) { + webrtc::test::WriteVideoToFile(aligned_reference_video, + JoinFilename(yuv_directory, "ref.yuv"), + /*fps=*/30); + webrtc::test::WriteVideoToFile(color_adjusted_test_video, + JoinFilename(yuv_directory, "test.yuv"), + /*fps=*/30); + } + + return 0; +} diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/linear_least_squares.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/linear_least_squares.cc new file mode 100644 index 0000000000..93a6f90e69 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/linear_least_squares.cc @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/linear_least_squares.h" + +#include <math.h> + +#include <cstdint> +#include <cstdlib> +#include <functional> +#include <numeric> +#include <type_traits> +#include <utility> + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace test { + +template <class T> +using Matrix = std::valarray<std::valarray<T>>; + +namespace { + +template <typename R, typename T> +R DotProduct(const std::valarray<T>& a, const std::valarray<T>& b) { + RTC_CHECK_EQ(a.size(), b.size()); + return std::inner_product(std::begin(a), std::end(a), std::begin(b), R(0)); +} + +// Calculates a^T * b. +template <typename R, typename T> +Matrix<R> MatrixMultiply(const Matrix<T>& a, const Matrix<T>& b) { + Matrix<R> result(std::valarray<R>(a.size()), b.size()); + for (size_t i = 0; i < a.size(); ++i) { + for (size_t j = 0; j < b.size(); ++j) + result[j][i] = DotProduct<R>(a[i], b[j]); + } + + return result; +} + +template <typename T> +Matrix<T> Transpose(const Matrix<T>& matrix) { + if (matrix.size() == 0) + return Matrix<T>(); + const size_t rows = matrix.size(); + const size_t columns = matrix[0].size(); + Matrix<T> result(std::valarray<T>(rows), columns); + + for (size_t i = 0; i < rows; ++i) { + for (size_t j = 0; j < columns; ++j) + result[j][i] = matrix[i][j]; + } + + return result; +} + +// Convert valarray from type T to type R. +template <typename R, typename T> +std::valarray<R> ConvertTo(const std::valarray<T>& v) { + std::valarray<R> result(v.size()); + for (size_t i = 0; i < v.size(); ++i) + result[i] = static_cast<R>(v[i]); + return result; +} + +// Convert valarray Matrix from type T to type R. +template <typename R, typename T> +Matrix<R> ConvertTo(const Matrix<T>& mat) { + Matrix<R> result(mat.size()); + for (size_t i = 0; i < mat.size(); ++i) + result[i] = ConvertTo<R>(mat[i]); + return result; +} + +// Convert from valarray Matrix back to the more conventional std::vector. +template <typename T> +std::vector<std::vector<T>> ToVectorMatrix(const Matrix<T>& m) { + std::vector<std::vector<T>> result; + for (const std::valarray<T>& v : m) + result.emplace_back(std::begin(v), std::end(v)); + return result; +} + +// Create a valarray Matrix from a conventional std::vector. +template <typename T> +Matrix<T> FromVectorMatrix(const std::vector<std::vector<T>>& mat) { + Matrix<T> result(mat.size()); + for (size_t i = 0; i < mat.size(); ++i) + result[i] = std::valarray<T>(mat[i].data(), mat[i].size()); + return result; +} + +// Returns `matrix_to_invert`^-1 * `right_hand_matrix`. `matrix_to_invert` must +// have square size. +Matrix<double> GaussianElimination(Matrix<double> matrix_to_invert, + Matrix<double> right_hand_matrix) { + // `n` is the width/height of `matrix_to_invert`. + const size_t n = matrix_to_invert.size(); + // Make sure `matrix_to_invert` has square size. + for (const std::valarray<double>& column : matrix_to_invert) + RTC_CHECK_EQ(n, column.size()); + // Make sure `right_hand_matrix` has correct size. + for (const std::valarray<double>& column : right_hand_matrix) + RTC_CHECK_EQ(n, column.size()); + + // Transpose the matrices before and after so that we can perform Gaussian + // elimination on the columns instead of the rows, since that is easier with + // our representation. + matrix_to_invert = Transpose(matrix_to_invert); + right_hand_matrix = Transpose(right_hand_matrix); + + // Loop over the diagonal of `matrix_to_invert` and perform column reduction. + // Column reduction is a sequence of elementary column operations that is + // performed on both `matrix_to_invert` and `right_hand_matrix` until + // `matrix_to_invert` has been transformed to the identity matrix. + for (size_t diagonal_index = 0; diagonal_index < n; ++diagonal_index) { + // Make sure the diagonal element has the highest absolute value by + // swapping columns if necessary. + for (size_t column = diagonal_index + 1; column < n; ++column) { + if (std::abs(matrix_to_invert[column][diagonal_index]) > + std::abs(matrix_to_invert[diagonal_index][diagonal_index])) { + std::swap(matrix_to_invert[column], matrix_to_invert[diagonal_index]); + std::swap(right_hand_matrix[column], right_hand_matrix[diagonal_index]); + } + } + + // Reduce the diagonal element to be 1, by dividing the column with that + // value. If the diagonal element is 0, it means the system of equations has + // many solutions, and in that case we will return an arbitrary solution. + if (matrix_to_invert[diagonal_index][diagonal_index] == 0.0) { + RTC_LOG(LS_WARNING) << "Matrix is not invertible, ignoring."; + continue; + } + const double diagonal_element = + matrix_to_invert[diagonal_index][diagonal_index]; + matrix_to_invert[diagonal_index] /= diagonal_element; + right_hand_matrix[diagonal_index] /= diagonal_element; + + // Eliminate the other entries in row `diagonal_index` by making them zero. + for (size_t column = 0; column < n; ++column) { + if (column == diagonal_index) + continue; + const double row_element = matrix_to_invert[column][diagonal_index]; + matrix_to_invert[column] -= + row_element * matrix_to_invert[diagonal_index]; + right_hand_matrix[column] -= + row_element * right_hand_matrix[diagonal_index]; + } + } + + // Transpose the result before returning it, explained in comment above. + return Transpose(right_hand_matrix); +} + +} // namespace + +IncrementalLinearLeastSquares::IncrementalLinearLeastSquares() = default; +IncrementalLinearLeastSquares::~IncrementalLinearLeastSquares() = default; + +void IncrementalLinearLeastSquares::AddObservations( + const std::vector<std::vector<uint8_t>>& x, + const std::vector<std::vector<uint8_t>>& y) { + if (x.empty() || y.empty()) + return; + // Make sure all columns are the same size. + const size_t n = x[0].size(); + for (const std::vector<uint8_t>& column : x) + RTC_CHECK_EQ(n, column.size()); + for (const std::vector<uint8_t>& column : y) + RTC_CHECK_EQ(n, column.size()); + + // We will multiply the uint8_t values together, so we need to expand to a + // type that can safely store those values, i.e. uint16_t. + const Matrix<uint16_t> unpacked_x = ConvertTo<uint16_t>(FromVectorMatrix(x)); + const Matrix<uint16_t> unpacked_y = ConvertTo<uint16_t>(FromVectorMatrix(y)); + + const Matrix<uint64_t> xx = MatrixMultiply<uint64_t>(unpacked_x, unpacked_x); + const Matrix<uint64_t> xy = MatrixMultiply<uint64_t>(unpacked_x, unpacked_y); + if (sum_xx && sum_xy) { + *sum_xx += xx; + *sum_xy += xy; + } else { + sum_xx = xx; + sum_xy = xy; + } +} + +std::vector<std::vector<double>> +IncrementalLinearLeastSquares::GetBestSolution() const { + RTC_CHECK(sum_xx && sum_xy) << "No observations have been added"; + return ToVectorMatrix(GaussianElimination(ConvertTo<double>(*sum_xx), + ConvertTo<double>(*sum_xy))); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/linear_least_squares.h b/third_party/libwebrtc/rtc_tools/frame_analyzer/linear_least_squares.h new file mode 100644 index 0000000000..7006db1d65 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/linear_least_squares.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_TOOLS_FRAME_ANALYZER_LINEAR_LEAST_SQUARES_H_ +#define RTC_TOOLS_FRAME_ANALYZER_LINEAR_LEAST_SQUARES_H_ + +#include <stdint.h> + +#include <valarray> +#include <vector> + +#include "absl/types/optional.h" + +namespace webrtc { +namespace test { + +// This class is used for finding a matrix b that roughly solves the equation: +// y = x * b. This is generally impossible to do exactly, so the problem is +// rephrased as finding the matrix b that minimizes the difference: +// |y - x * b|^2. Calling multiple AddObservations() is equivalent to +// concatenating the observation vectors and calling AddObservations() once. The +// reason for doing it incrementally is that we can't store the raw YUV values +// for a whole video file in memory at once. This class has a constant memory +// footprint, regardless how may times AddObservations() is called. +class IncrementalLinearLeastSquares { + public: + IncrementalLinearLeastSquares(); + ~IncrementalLinearLeastSquares(); + + // Add a number of observations. The subvectors of x and y must have the same + // length. + void AddObservations(const std::vector<std::vector<uint8_t>>& x, + const std::vector<std::vector<uint8_t>>& y); + + // Calculate and return the best linear solution, given the observations so + // far. + std::vector<std::vector<double>> GetBestSolution() const; + + private: + // Running sum of x^T * x. + absl::optional<std::valarray<std::valarray<uint64_t>>> sum_xx; + // Running sum of x^T * y. + absl::optional<std::valarray<std::valarray<uint64_t>>> sum_xy; +}; + +} // namespace test +} // namespace webrtc + +#endif // RTC_TOOLS_FRAME_ANALYZER_LINEAR_LEAST_SQUARES_H_ diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/linear_least_squares_unittest.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/linear_least_squares_unittest.cc new file mode 100644 index 0000000000..d4a23e87a6 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/linear_least_squares_unittest.cc @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/linear_least_squares.h" + +#include <cstdint> + +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +TEST(LinearLeastSquares, ScalarIdentityOneObservation) { + IncrementalLinearLeastSquares lls; + lls.AddObservations({{1}}, {{1}}); + EXPECT_EQ(std::vector<std::vector<double>>({{1.0}}), lls.GetBestSolution()); +} + +TEST(LinearLeastSquares, ScalarIdentityTwoObservationsOneCall) { + IncrementalLinearLeastSquares lls; + lls.AddObservations({{1, 2}}, {{1, 2}}); + EXPECT_EQ(std::vector<std::vector<double>>({{1.0}}), lls.GetBestSolution()); +} + +TEST(LinearLeastSquares, ScalarIdentityTwoObservationsTwoCalls) { + IncrementalLinearLeastSquares lls; + lls.AddObservations({{1}}, {{1}}); + lls.AddObservations({{2}}, {{2}}); + EXPECT_EQ(std::vector<std::vector<double>>({{1.0}}), lls.GetBestSolution()); +} + +TEST(LinearLeastSquares, MatrixIdentityOneObservation) { + IncrementalLinearLeastSquares lls; + lls.AddObservations({{1, 2}, {3, 4}}, {{1, 2}, {3, 4}}); + EXPECT_EQ(std::vector<std::vector<double>>({{1.0, 0.0}, {0.0, 1.0}}), + lls.GetBestSolution()); +} + +TEST(LinearLeastSquares, MatrixManyObservations) { + IncrementalLinearLeastSquares lls; + // Test that we can find the solution of the overspecified equation system: + // [1, 2] [1, 3] = [5, 11] + // [3, 4] [2, 4] [11, 25] + // [5, 6] [17, 39] + lls.AddObservations({{1}, {2}}, {{5}, {11}}); + lls.AddObservations({{3}, {4}}, {{11}, {25}}); + lls.AddObservations({{5}, {6}}, {{17}, {39}}); + + const std::vector<std::vector<double>> result = lls.GetBestSolution(); + // We allow some numerical flexibility here. + EXPECT_DOUBLE_EQ(1.0, result[0][0]); + EXPECT_DOUBLE_EQ(2.0, result[0][1]); + EXPECT_DOUBLE_EQ(3.0, result[1][0]); + EXPECT_DOUBLE_EQ(4.0, result[1][1]); +} + +TEST(LinearLeastSquares, MatrixVectorOneObservation) { + IncrementalLinearLeastSquares lls; + // Test that we can find the solution of the overspecified equation system: + // [1, 2] [1] = [5] + // [3, 4] [2] [11] + // [5, 6] [17] + lls.AddObservations({{1, 3, 5}, {2, 4, 6}}, {{5, 11, 17}}); + + const std::vector<std::vector<double>> result = lls.GetBestSolution(); + // We allow some numerical flexibility here. + EXPECT_DOUBLE_EQ(1.0, result[0][0]); + EXPECT_DOUBLE_EQ(2.0, result[0][1]); +} + +TEST(LinearLeastSquares, LinearLeastSquaresNonPerfectSolution) { + IncrementalLinearLeastSquares lls; + // Test that we can find the non-perfect solution of the overspecified + // equation system: + // [1] [20] = [21] + // [2] [39] + // [3] [60] + // [2] [41] + // [1] [19] + lls.AddObservations({{1, 2, 3, 2, 1}}, {{21, 39, 60, 41, 19}}); + + EXPECT_DOUBLE_EQ(20.0, lls.GetBestSolution()[0][0]); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis.cc new file mode 100644 index 0000000000..fca89a8f2f --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis.cc @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include <stdio.h> +#include <stdlib.h> + +#include <string> + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/flags/usage.h" +#include "rtc_tools/frame_analyzer/reference_less_video_analysis_lib.h" + +ABSL_FLAG(std::string, + video_file, + "", + "Path of the video file to be analyzed, only y4m file format is " + "supported"); + +int main(int argc, char* argv[]) { + absl::SetProgramUsageMessage( + "Outputs the freezing score by comparing " + "current frame with the previous frame.\n" + "Example usage:\n" + "./reference_less_video_analysis " + "--video_file=video_file.y4m\n"); + absl::ParseCommandLine(argc, argv); + + std::string video_file = absl::GetFlag(FLAGS_video_file); + if (video_file.empty()) { + exit(EXIT_FAILURE); + } + + return run_analysis(video_file); +} diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis_lib.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis_lib.cc new file mode 100644 index 0000000000..ebfc6650b2 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis_lib.cc @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "rtc_tools/frame_analyzer/reference_less_video_analysis_lib.h" + +#include <stdio.h> + +#include <numeric> +#include <vector> + +#include "api/video/video_frame_buffer.h" +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" + +#define STATS_LINE_LENGTH 28 +#define PSNR_FREEZE_THRESHOLD 47 +#define SSIM_FREEZE_THRESHOLD .999 + +#if defined(_WIN32) || defined(_WIN64) +#define strtok_r strtok_s +#endif + +bool frozen_frame(std::vector<double> psnr_per_frame, + std::vector<double> ssim_per_frame, + size_t frame) { + if (psnr_per_frame[frame] >= PSNR_FREEZE_THRESHOLD || + ssim_per_frame[frame] >= SSIM_FREEZE_THRESHOLD) + return true; + return false; +} + +std::vector<int> find_frame_clusters( + const std::vector<double>& psnr_per_frame, + const std::vector<double>& ssim_per_frame) { + std::vector<int> identical_frame_clusters; + int num_frozen = 0; + size_t total_no_of_frames = psnr_per_frame.size(); + + for (size_t each_frame = 0; each_frame < total_no_of_frames; each_frame++) { + if (frozen_frame(psnr_per_frame, ssim_per_frame, each_frame)) { + num_frozen++; + } else if (num_frozen > 0) { + // Not frozen anymore. + identical_frame_clusters.push_back(num_frozen); + num_frozen = 0; + } + } + return identical_frame_clusters; +} + +void print_freezing_metrics(const std::vector<double>& psnr_per_frame, + const std::vector<double>& ssim_per_frame) { + /* + * Prints the different metrics mainly: + * 1) Identical frame number, PSNR and SSIM values. + * 2) Length of continuous frozen frames. + * 3) Max length of continuous freezed frames. + * 4) No of unique frames found. + * 5) Total different identical frames found. + * + * Sample output: + * Printing metrics for file: /src/rtc_tools/test_3.y4m + ============================= + Total number of frames received: 74 + Total identical frames: 5 + Number of unique frames: 69 + Printing Identical Frames: + Frame Number: 29 PSNR: 48.000000 SSIM: 0.999618 + Frame Number: 30 PSNR: 48.000000 SSIM: 0.999898 + Frame Number: 60 PSNR: 48.000000 SSIM: 0.999564 + Frame Number: 64 PSNR: 48.000000 SSIM: 0.999651 + Frame Number: 69 PSNR: 48.000000 SSIM: 0.999684 + Print identical frame which appears in clusters : + 2 1 1 1 + * + */ + size_t total_no_of_frames = psnr_per_frame.size(); + std::vector<int> identical_frame_clusters = + find_frame_clusters(psnr_per_frame, ssim_per_frame); + int total_identical_frames = std::accumulate( + identical_frame_clusters.begin(), identical_frame_clusters.end(), 0); + size_t unique_frames = total_no_of_frames - total_identical_frames; + + printf("Total number of frames received: %zu\n", total_no_of_frames); + printf("Total identical frames: %d\n", total_identical_frames); + printf("Number of unique frames: %zu\n", unique_frames); + + printf("Printing Identical Frames: \n"); + for (size_t frame = 0; frame < total_no_of_frames; frame++) { + if (frozen_frame(psnr_per_frame, ssim_per_frame, frame)) { + printf(" Frame Number: %zu PSNR: %f SSIM: %f \n", frame, + psnr_per_frame[frame], ssim_per_frame[frame]); + } + } + + printf("Print identical frame which appears in clusters : \n"); + for (int cluster = 0; + cluster < static_cast<int>(identical_frame_clusters.size()); cluster++) + printf("%d ", identical_frame_clusters[cluster]); + printf("\n"); +} + +void compute_metrics(const rtc::scoped_refptr<webrtc::test::Video>& video, + std::vector<double>* psnr_per_frame, + std::vector<double>* ssim_per_frame) { + for (size_t i = 0; i < video->number_of_frames() - 1; ++i) { + const rtc::scoped_refptr<webrtc::I420BufferInterface> current_frame = + video->GetFrame(i); + const rtc::scoped_refptr<webrtc::I420BufferInterface> next_frame = + video->GetFrame(i + 1); + double result_psnr = webrtc::test::Psnr(current_frame, next_frame); + double result_ssim = webrtc::test::Ssim(current_frame, next_frame); + + psnr_per_frame->push_back(result_psnr); + ssim_per_frame->push_back(result_ssim); + } +} + +int run_analysis(const std::string& video_file) { + std::vector<double> psnr_per_frame; + std::vector<double> ssim_per_frame; + rtc::scoped_refptr<webrtc::test::Video> video = + webrtc::test::OpenY4mFile(video_file); + if (video) { + compute_metrics(video, &psnr_per_frame, &ssim_per_frame); + } else { + return -1; + } + printf("=============================\n"); + printf("Printing metrics for file: %s\n", video_file.c_str()); + printf("=============================\n"); + print_freezing_metrics(psnr_per_frame, ssim_per_frame); + return 0; +} diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis_lib.h b/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis_lib.h new file mode 100644 index 0000000000..3c93119905 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis_lib.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_TOOLS_FRAME_ANALYZER_REFERENCE_LESS_VIDEO_ANALYSIS_LIB_H_ +#define RTC_TOOLS_FRAME_ANALYZER_REFERENCE_LESS_VIDEO_ANALYSIS_LIB_H_ + +#include <stddef.h> + +#include <string> +#include <vector> + +#include "api/scoped_refptr.h" +#include "rtc_tools/video_file_reader.h" + +// Returns true if the frame is frozen based on psnr and ssim freezing +// threshold values. +bool frozen_frame(std::vector<double> psnr_per_frame, + std::vector<double> ssim_per_frame, + size_t frame); + +// Returns the vector of identical cluster of frames that are frozen +// and appears continuously. +std::vector<int> find_frame_clusters(const std::vector<double>& psnr_per_frame, + const std::vector<double>& ssim_per_frame); + +// Prints various freezing metrics like identical frames, +// total unique frames etc. +void print_freezing_metrics(const std::vector<double>& psnr_per_frame, + const std::vector<double>& ssim_per_frame); + +// Compute the metrics like freezing score based on PSNR and SSIM values for a +// given video file. +void compute_metrics(const rtc::scoped_refptr<webrtc::test::Video>& video, + std::vector<double>* psnr_per_frame, + std::vector<double>* ssim_per_frame); + +// Compute freezing score metrics and prints the metrics +// for a list of video files. +int run_analysis(const std::string& video_file); + +#endif // RTC_TOOLS_FRAME_ANALYZER_REFERENCE_LESS_VIDEO_ANALYSIS_LIB_H_ diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis_unittest.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis_unittest.cc new file mode 100644 index 0000000000..b98a014989 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/reference_less_video_analysis_unittest.cc @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include <vector> + +#include "api/scoped_refptr.h" +#include "rtc_tools/frame_analyzer/reference_less_video_analysis_lib.h" +#include "rtc_tools/video_file_reader.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +class ReferenceLessVideoAnalysisTest : public ::testing::Test { + public: + void SetUp() override { + video = webrtc::test::OpenY4mFile( + webrtc::test::ResourcePath("reference_less_video_test_file", "y4m")); + ASSERT_TRUE(video); + } + + rtc::scoped_refptr<webrtc::test::Video> video; + std::vector<double> psnr_per_frame; + std::vector<double> ssim_per_frame; +}; + +TEST_F(ReferenceLessVideoAnalysisTest, MatchComputedMetrics) { + compute_metrics(video, &psnr_per_frame, &ssim_per_frame); + EXPECT_EQ(74, (int)psnr_per_frame.size()); + + ASSERT_NEAR(27.2f, psnr_per_frame[1], 0.1f); + ASSERT_NEAR(24.9f, psnr_per_frame[5], 0.1f); + + ASSERT_NEAR(0.9f, ssim_per_frame[1], 0.1f); + ASSERT_NEAR(0.9f, ssim_per_frame[5], 0.1f); +} + +TEST_F(ReferenceLessVideoAnalysisTest, MatchIdenticalFrameClusters) { + compute_metrics(video, &psnr_per_frame, &ssim_per_frame); + std::vector<int> identical_frame_clusters = + find_frame_clusters(psnr_per_frame, ssim_per_frame); + EXPECT_EQ(5, (int)identical_frame_clusters.size()); + EXPECT_EQ(1, identical_frame_clusters[0]); + EXPECT_EQ(1, identical_frame_clusters[4]); +} diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_color_aligner.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_color_aligner.cc new file mode 100644 index 0000000000..5983e47f69 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_color_aligner.cc @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/video_color_aligner.h" + +#include <stddef.h> + +#include <algorithm> +#include <cmath> +#include <cstdint> +#include <vector> + +#include "api/array_view.h" +#include "api/make_ref_counted.h" +#include "api/video/i420_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_tools/frame_analyzer/linear_least_squares.h" +#include "third_party/libyuv/include/libyuv/planar_functions.h" +#include "third_party/libyuv/include/libyuv/scale.h" + +namespace webrtc { +namespace test { + +namespace { + +// Helper function for AdjustColors(). This functions calculates a single output +// row for y with the given color coefficients. The u/v channels are assumed to +// be subsampled by a factor of 2, which is the case of I420. +void CalculateYChannel(rtc::ArrayView<const uint8_t> y_data, + rtc::ArrayView<const uint8_t> u_data, + rtc::ArrayView<const uint8_t> v_data, + const std::array<float, 4>& coeff, + rtc::ArrayView<uint8_t> output) { + RTC_CHECK_EQ(y_data.size(), output.size()); + // Each u/v element represents two y elements. Make sure we have enough to + // cover the Y values. + RTC_CHECK_GE(u_data.size() * 2, y_data.size()); + RTC_CHECK_GE(v_data.size() * 2, y_data.size()); + + // Do two pixels at a time since u/v are subsampled. + for (size_t i = 0; i * 2 < y_data.size() - 1; ++i) { + const float uv_contribution = + coeff[1] * u_data[i] + coeff[2] * v_data[i] + coeff[3]; + + const float val0 = coeff[0] * y_data[i * 2 + 0] + uv_contribution; + const float val1 = coeff[0] * y_data[i * 2 + 1] + uv_contribution; + + // Clamp result to a byte. + output[i * 2 + 0] = static_cast<uint8_t>( + std::round(std::max(0.0f, std::min(val0, 255.0f)))); + output[i * 2 + 1] = static_cast<uint8_t>( + std::round(std::max(0.0f, std::min(val1, 255.0f)))); + } + + // Handle the last pixel for odd widths. + if (y_data.size() % 2 == 1) { + const float val = coeff[0] * y_data[y_data.size() - 1] + + coeff[1] * u_data[(y_data.size() - 1) / 2] + + coeff[2] * v_data[(y_data.size() - 1) / 2] + coeff[3]; + output[y_data.size() - 1] = + static_cast<uint8_t>(std::round(std::max(0.0f, std::min(val, 255.0f)))); + } +} + +// Helper function for AdjustColors(). This functions calculates a single output +// row for either u or v, with the given color coefficients. Y, U, and V are +// assumed to be the same size, i.e. no subsampling. +void CalculateUVChannel(rtc::ArrayView<const uint8_t> y_data, + rtc::ArrayView<const uint8_t> u_data, + rtc::ArrayView<const uint8_t> v_data, + const std::array<float, 4>& coeff, + rtc::ArrayView<uint8_t> output) { + RTC_CHECK_EQ(y_data.size(), u_data.size()); + RTC_CHECK_EQ(y_data.size(), v_data.size()); + RTC_CHECK_EQ(y_data.size(), output.size()); + + for (size_t x = 0; x < y_data.size(); ++x) { + const float val = coeff[0] * y_data[x] + coeff[1] * u_data[x] + + coeff[2] * v_data[x] + coeff[3]; + // Clamp result to a byte. + output[x] = + static_cast<uint8_t>(std::round(std::max(0.0f, std::min(val, 255.0f)))); + } +} + +// Convert a frame to four vectors consisting of [y, u, v, 1]. +std::vector<std::vector<uint8_t>> FlattenYuvData( + const rtc::scoped_refptr<I420BufferInterface>& frame) { + std::vector<std::vector<uint8_t>> result( + 4, std::vector<uint8_t>(frame->ChromaWidth() * frame->ChromaHeight())); + + // Downscale the Y plane so that all YUV planes are the same size. + libyuv::ScalePlane(frame->DataY(), frame->StrideY(), frame->width(), + frame->height(), result[0].data(), frame->ChromaWidth(), + frame->ChromaWidth(), frame->ChromaHeight(), + libyuv::kFilterBox); + + libyuv::CopyPlane(frame->DataU(), frame->StrideU(), result[1].data(), + frame->ChromaWidth(), frame->ChromaWidth(), + frame->ChromaHeight()); + + libyuv::CopyPlane(frame->DataV(), frame->StrideV(), result[2].data(), + frame->ChromaWidth(), frame->ChromaWidth(), + frame->ChromaHeight()); + + std::fill(result[3].begin(), result[3].end(), 1u); + + return result; +} + +ColorTransformationMatrix VectorToColorMatrix( + const std::vector<std::vector<double>>& v) { + ColorTransformationMatrix color_transformation; + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 4; ++j) + color_transformation[i][j] = v[i][j]; + } + return color_transformation; +} + +} // namespace + +ColorTransformationMatrix CalculateColorTransformationMatrix( + const rtc::scoped_refptr<I420BufferInterface>& reference_frame, + const rtc::scoped_refptr<I420BufferInterface>& test_frame) { + IncrementalLinearLeastSquares incremental_lls; + incremental_lls.AddObservations(FlattenYuvData(test_frame), + FlattenYuvData(reference_frame)); + return VectorToColorMatrix(incremental_lls.GetBestSolution()); +} + +ColorTransformationMatrix CalculateColorTransformationMatrix( + const rtc::scoped_refptr<Video>& reference_video, + const rtc::scoped_refptr<Video>& test_video) { + RTC_CHECK_GE(reference_video->number_of_frames(), + test_video->number_of_frames()); + + IncrementalLinearLeastSquares incremental_lls; + for (size_t i = 0; i < test_video->number_of_frames(); ++i) { + incremental_lls.AddObservations( + FlattenYuvData(test_video->GetFrame(i)), + FlattenYuvData(reference_video->GetFrame(i))); + } + + return VectorToColorMatrix(incremental_lls.GetBestSolution()); +} + +rtc::scoped_refptr<Video> AdjustColors( + const ColorTransformationMatrix& color_transformation, + const rtc::scoped_refptr<Video>& video) { + class ColorAdjustedVideo : public Video { + public: + ColorAdjustedVideo(const ColorTransformationMatrix& color_transformation, + const rtc::scoped_refptr<Video>& video) + : color_transformation_(color_transformation), video_(video) {} + + int width() const override { return video_->width(); } + int height() const override { return video_->height(); } + size_t number_of_frames() const override { + return video_->number_of_frames(); + } + + rtc::scoped_refptr<I420BufferInterface> GetFrame( + size_t index) const override { + return AdjustColors(color_transformation_, video_->GetFrame(index)); + } + + private: + const ColorTransformationMatrix color_transformation_; + const rtc::scoped_refptr<Video> video_; + }; + + return rtc::make_ref_counted<ColorAdjustedVideo>(color_transformation, video); +} + +rtc::scoped_refptr<I420BufferInterface> AdjustColors( + const ColorTransformationMatrix& color_matrix, + const rtc::scoped_refptr<I420BufferInterface>& frame) { + // Allocate I420 buffer that will hold the color adjusted frame. + rtc::scoped_refptr<I420Buffer> adjusted_frame = + I420Buffer::Create(frame->width(), frame->height()); + + // Create a downscaled Y plane with the same size as the U/V planes to + // simplify converting the U/V planes. + std::vector<uint8_t> downscaled_y_plane(frame->ChromaWidth() * + frame->ChromaHeight()); + libyuv::ScalePlane(frame->DataY(), frame->StrideY(), frame->width(), + frame->height(), downscaled_y_plane.data(), + frame->ChromaWidth(), frame->ChromaWidth(), + frame->ChromaHeight(), libyuv::kFilterBox); + + // Fill in the adjusted data row by row. + for (int y = 0; y < frame->height(); ++y) { + const int half_y = y / 2; + rtc::ArrayView<const uint8_t> y_row(frame->DataY() + frame->StrideY() * y, + frame->width()); + rtc::ArrayView<const uint8_t> u_row( + frame->DataU() + frame->StrideU() * half_y, frame->ChromaWidth()); + rtc::ArrayView<const uint8_t> v_row( + frame->DataV() + frame->StrideV() * half_y, frame->ChromaWidth()); + rtc::ArrayView<uint8_t> output_y_row( + adjusted_frame->MutableDataY() + adjusted_frame->StrideY() * y, + frame->width()); + + CalculateYChannel(y_row, u_row, v_row, color_matrix[0], output_y_row); + + // Chroma channels only exist every second row for I420. + if (y % 2 == 0) { + rtc::ArrayView<const uint8_t> downscaled_y_row( + downscaled_y_plane.data() + frame->ChromaWidth() * half_y, + frame->ChromaWidth()); + rtc::ArrayView<uint8_t> output_u_row( + adjusted_frame->MutableDataU() + adjusted_frame->StrideU() * half_y, + frame->ChromaWidth()); + rtc::ArrayView<uint8_t> output_v_row( + adjusted_frame->MutableDataV() + adjusted_frame->StrideV() * half_y, + frame->ChromaWidth()); + + CalculateUVChannel(downscaled_y_row, u_row, v_row, color_matrix[1], + output_u_row); + CalculateUVChannel(downscaled_y_row, u_row, v_row, color_matrix[2], + output_v_row); + } + } + + return adjusted_frame; +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_color_aligner.h b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_color_aligner.h new file mode 100644 index 0000000000..b51e06060b --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_color_aligner.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_TOOLS_FRAME_ANALYZER_VIDEO_COLOR_ALIGNER_H_ +#define RTC_TOOLS_FRAME_ANALYZER_VIDEO_COLOR_ALIGNER_H_ + +#include <array> + +#include "api/scoped_refptr.h" +#include "api/video/video_frame_buffer.h" +#include "rtc_tools/video_file_reader.h" + +namespace webrtc { +namespace test { + +// Represents a linear color transformation from [y, u, v] to [y', u', v'] +// through the equation: [y', u', v'] = [y, u, v, 1] * matrix. +using ColorTransformationMatrix = std::array<std::array<float, 4>, 3>; + +// Calculate the optimal color transformation that should be applied to the test +// video to match as closely as possible to the reference video. +ColorTransformationMatrix CalculateColorTransformationMatrix( + const rtc::scoped_refptr<Video>& reference_video, + const rtc::scoped_refptr<Video>& test_video); + +// Calculate color transformation for a single I420 frame. +ColorTransformationMatrix CalculateColorTransformationMatrix( + const rtc::scoped_refptr<I420BufferInterface>& reference_frame, + const rtc::scoped_refptr<I420BufferInterface>& test_frame); + +// Apply a color transformation to a video. +rtc::scoped_refptr<Video> AdjustColors( + const ColorTransformationMatrix& color_matrix, + const rtc::scoped_refptr<Video>& video); + +// Apply a color transformation to a single I420 frame. +rtc::scoped_refptr<I420BufferInterface> AdjustColors( + const ColorTransformationMatrix& color_matrix, + const rtc::scoped_refptr<I420BufferInterface>& frame); + +} // namespace test +} // namespace webrtc + +#endif // RTC_TOOLS_FRAME_ANALYZER_VIDEO_COLOR_ALIGNER_H_ diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_color_aligner_unittest.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_color_aligner_unittest.cc new file mode 100644 index 0000000000..980898b6bd --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_color_aligner_unittest.cc @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/video_color_aligner.h" + +#include <stdint.h> + +#include "api/video/i420_buffer.h" +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" +#include "rtc_tools/video_file_reader.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { + +namespace { + +const ColorTransformationMatrix kIdentityColorMatrix = { + {{1, 0, 0, 0}, {0, 1, 0, 0}, {0, 0, 1, 0}}}; + +void ExpectNear(const ColorTransformationMatrix& expected, + const ColorTransformationMatrix& actual) { + // The scaling factor on y/u/v should be pretty precise. + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 3; ++j) + EXPECT_NEAR(expected[i][j], actual[i][j], /* abs_error= */ 1.0e-3) + << "at element i: " << i << ", j: " << j; + } + // The offset can be less precise since the range is [0, 255]. + for (int i = 0; i < 3; ++i) + EXPECT_NEAR(expected[i][3], actual[i][3], /* abs_error= */ 0.1) + << "at element i: " << i; +} + +} // namespace + +class VideoColorAlignerTest : public ::testing::Test { + protected: + void SetUp() { + reference_video_ = + OpenYuvFile(ResourcePath("foreman_128x96", "yuv"), 128, 96); + ASSERT_TRUE(reference_video_); + } + + rtc::scoped_refptr<Video> reference_video_; +}; + +TEST_F(VideoColorAlignerTest, AdjustColorsFrameIdentity) { + const rtc::scoped_refptr<I420BufferInterface> test_frame = + reference_video_->GetFrame(0); + + // Assume perfect match, i.e. ssim == 1. + EXPECT_EQ(1.0, + Ssim(test_frame, AdjustColors(kIdentityColorMatrix, test_frame))); +} + +TEST_F(VideoColorAlignerTest, AdjustColorsFrame1x1) { + const ColorTransformationMatrix color_matrix = { + {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}}}; + + const uint8_t data_y[] = {2}; + const uint8_t data_u[] = {6}; + const uint8_t data_v[] = {7}; + const rtc::scoped_refptr<I420BufferInterface> i420_buffer = I420Buffer::Copy( + /* width= */ 1, /* height= */ 1, data_y, /* stride_y= */ 1, data_u, + /* stride_u= */ 1, data_v, /* stride_v= */ 1); + + const rtc::scoped_refptr<I420BufferInterface> adjusted_buffer = + AdjustColors(color_matrix, i420_buffer); + + EXPECT_EQ(2 * 1 + 6 * 2 + 7 * 3 + 4, adjusted_buffer->DataY()[0]); + EXPECT_EQ(2 * 5 + 6 * 6 + 7 * 7 + 8, adjusted_buffer->DataU()[0]); + EXPECT_EQ(2 * 9 + 6 * 10 + 7 * 11 + 12, adjusted_buffer->DataV()[0]); +} + +TEST_F(VideoColorAlignerTest, AdjustColorsFrame1x1Negative) { + const ColorTransformationMatrix color_matrix = { + {{-1, 0, 0, 255}, {0, -1, 0, 255}, {0, 0, -1, 255}}}; + + const uint8_t data_y[] = {2}; + const uint8_t data_u[] = {6}; + const uint8_t data_v[] = {7}; + const rtc::scoped_refptr<I420BufferInterface> i420_buffer = I420Buffer::Copy( + /* width= */ 1, /* height= */ 1, data_y, /* stride_y= */ 1, data_u, + /* stride_u= */ 1, data_v, /* stride_v= */ 1); + + const rtc::scoped_refptr<I420BufferInterface> adjusted_buffer = + AdjustColors(color_matrix, i420_buffer); + + EXPECT_EQ(255 - 2, adjusted_buffer->DataY()[0]); + EXPECT_EQ(255 - 6, adjusted_buffer->DataU()[0]); + EXPECT_EQ(255 - 7, adjusted_buffer->DataV()[0]); +} + +TEST_F(VideoColorAlignerTest, AdjustColorsFrame2x2) { + const ColorTransformationMatrix color_matrix = { + {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}}}; + + const uint8_t data_y[] = {0, 1, 3, 4}; + const uint8_t data_u[] = {6}; + const uint8_t data_v[] = {7}; + const rtc::scoped_refptr<I420BufferInterface> i420_buffer = I420Buffer::Copy( + /* width= */ 2, /* height= */ 2, data_y, /* stride_y= */ 2, data_u, + /* stride_u= */ 1, data_v, /* stride_v= */ 1); + + const rtc::scoped_refptr<I420BufferInterface> adjusted_buffer = + AdjustColors(color_matrix, i420_buffer); + + EXPECT_EQ(0 * 1 + 6 * 2 + 7 * 3 + 4, adjusted_buffer->DataY()[0]); + EXPECT_EQ(1 * 1 + 6 * 2 + 7 * 3 + 4, adjusted_buffer->DataY()[1]); + EXPECT_EQ(3 * 1 + 6 * 2 + 7 * 3 + 4, adjusted_buffer->DataY()[2]); + EXPECT_EQ(4 * 1 + 6 * 2 + 7 * 3 + 4, adjusted_buffer->DataY()[3]); + + EXPECT_EQ(2 * 5 + 6 * 6 + 7 * 7 + 8, adjusted_buffer->DataU()[0]); + EXPECT_EQ(2 * 9 + 6 * 10 + 7 * 11 + 12, adjusted_buffer->DataV()[0]); +} + +TEST_F(VideoColorAlignerTest, CalculateColorTransformationMatrixIdentity) { + EXPECT_EQ(kIdentityColorMatrix, CalculateColorTransformationMatrix( + reference_video_, reference_video_)); +} + +TEST_F(VideoColorAlignerTest, CalculateColorTransformationMatrixOffset) { + const uint8_t small_data_y[] = {0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15}; + const uint8_t small_data_u[] = {15, 13, 17, 29}; + const uint8_t small_data_v[] = {3, 200, 170, 29}; + const rtc::scoped_refptr<I420BufferInterface> small_i420_buffer = + I420Buffer::Copy( + /* width= */ 4, /* height= */ 4, small_data_y, /* stride_y= */ 4, + small_data_u, /* stride_u= */ 2, small_data_v, /* stride_v= */ 2); + + uint8_t big_data_y[16]; + uint8_t big_data_u[4]; + uint8_t big_data_v[4]; + // Create another I420 frame where all values are 10 bigger. + for (int i = 0; i < 16; ++i) + big_data_y[i] = small_data_y[i] + 10; + for (int i = 0; i < 4; ++i) + big_data_u[i] = small_data_u[i] + 10; + for (int i = 0; i < 4; ++i) + big_data_v[i] = small_data_v[i] + 10; + + const rtc::scoped_refptr<I420BufferInterface> big_i420_buffer = + I420Buffer::Copy( + /* width= */ 4, /* height= */ 4, big_data_y, /* stride_y= */ 4, + big_data_u, /* stride_u= */ 2, big_data_v, /* stride_v= */ 2); + + const ColorTransformationMatrix color_matrix = + CalculateColorTransformationMatrix(big_i420_buffer, small_i420_buffer); + + ExpectNear({{{1, 0, 0, 10}, {0, 1, 0, 10}, {0, 0, 1, 10}}}, color_matrix); +} + +TEST_F(VideoColorAlignerTest, CalculateColorTransformationMatrix) { + // Arbitrary color transformation matrix. + const ColorTransformationMatrix org_color_matrix = { + {{0.8, 0.05, 0.04, -4}, {-0.2, 0.7, 0.1, 10}, {0.1, 0.2, 0.4, 20}}}; + + const ColorTransformationMatrix result_color_matrix = + CalculateColorTransformationMatrix( + AdjustColors(org_color_matrix, reference_video_), reference_video_); + + ExpectNear(org_color_matrix, result_color_matrix); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_geometry_aligner.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_geometry_aligner.cc new file mode 100644 index 0000000000..efb033317a --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_geometry_aligner.cc @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" + +#include <map> + +#include "api/make_ref_counted.h" +#include "api/video/i420_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" +#include "third_party/libyuv/include/libyuv/scale.h" + +namespace webrtc { +namespace test { + +namespace { + +bool IsValidRegion(const CropRegion& region, + const rtc::scoped_refptr<I420BufferInterface>& frame) { + return region.left >= 0 && region.right >= 0 && region.top >= 0 && + region.bottom >= 0 && region.left + region.right < frame->width() && + region.top + region.bottom < frame->height(); +} + +} // namespace + +rtc::scoped_refptr<I420BufferInterface> CropAndZoom( + const CropRegion& crop_region, + const rtc::scoped_refptr<I420BufferInterface>& frame) { + RTC_CHECK(IsValidRegion(crop_region, frame)); + + const int uv_crop_left = crop_region.left / 2; + const int uv_crop_top = crop_region.top / 2; + + const int cropped_width = + frame->width() - crop_region.left - crop_region.right; + const int cropped_height = + frame->height() - crop_region.top - crop_region.bottom; + + // Crop by only adjusting pointers. + const uint8_t* y_plane = + frame->DataY() + frame->StrideY() * crop_region.top + crop_region.left; + const uint8_t* u_plane = + frame->DataU() + frame->StrideU() * uv_crop_top + uv_crop_left; + const uint8_t* v_plane = + frame->DataV() + frame->StrideV() * uv_crop_top + uv_crop_left; + + // Stretch the cropped frame to the original size using libyuv. + rtc::scoped_refptr<I420Buffer> adjusted_frame = + I420Buffer::Create(frame->width(), frame->height()); + libyuv::I420Scale(y_plane, frame->StrideY(), u_plane, frame->StrideU(), + v_plane, frame->StrideV(), cropped_width, cropped_height, + adjusted_frame->MutableDataY(), adjusted_frame->StrideY(), + adjusted_frame->MutableDataU(), adjusted_frame->StrideU(), + adjusted_frame->MutableDataV(), adjusted_frame->StrideV(), + frame->width(), frame->height(), libyuv::kFilterBox); + + return adjusted_frame; +} + +CropRegion CalculateCropRegion( + const rtc::scoped_refptr<I420BufferInterface>& reference_frame, + const rtc::scoped_refptr<I420BufferInterface>& test_frame) { + RTC_CHECK_EQ(reference_frame->width(), test_frame->width()); + RTC_CHECK_EQ(reference_frame->height(), test_frame->height()); + + CropRegion best_region; + double best_ssim = Ssim(reference_frame, test_frame); + + typedef int CropRegion::*CropParameter; + CropParameter crop_parameters[4] = {&CropRegion::left, &CropRegion::top, + &CropRegion::right, &CropRegion::bottom}; + + while (true) { + // Find the parameter in which direction SSIM improves the most. + CropParameter best_parameter = nullptr; + const CropRegion prev_best_region = best_region; + + for (CropParameter crop_parameter : crop_parameters) { + CropRegion test_region = prev_best_region; + ++(test_region.*crop_parameter); + + if (!IsValidRegion(test_region, reference_frame)) + continue; + + const double ssim = + Ssim(CropAndZoom(test_region, reference_frame), test_frame); + + if (ssim > best_ssim) { + best_ssim = ssim; + best_parameter = crop_parameter; + best_region = test_region; + } + } + + // No improvement among any direction, stop iteration. + if (best_parameter == nullptr) + break; + + // Iterate in the best direction as long as it improves SSIM. + for (CropRegion test_region = best_region; + IsValidRegion(test_region, reference_frame); + ++(test_region.*best_parameter)) { + const double ssim = + Ssim(CropAndZoom(test_region, reference_frame), test_frame); + if (ssim <= best_ssim) + break; + + best_ssim = ssim; + best_region = test_region; + } + } + + return best_region; +} + +rtc::scoped_refptr<I420BufferInterface> AdjustCropping( + const rtc::scoped_refptr<I420BufferInterface>& reference_frame, + const rtc::scoped_refptr<I420BufferInterface>& test_frame) { + return CropAndZoom(CalculateCropRegion(reference_frame, test_frame), + reference_frame); +} + +rtc::scoped_refptr<Video> AdjustCropping( + const rtc::scoped_refptr<Video>& reference_video, + const rtc::scoped_refptr<Video>& test_video) { + class CroppedVideo : public Video { + public: + CroppedVideo(const rtc::scoped_refptr<Video>& reference_video, + const rtc::scoped_refptr<Video>& test_video) + : reference_video_(reference_video), test_video_(test_video) { + RTC_CHECK_EQ(reference_video->number_of_frames(), + test_video->number_of_frames()); + RTC_CHECK_EQ(reference_video->width(), test_video->width()); + RTC_CHECK_EQ(reference_video->height(), test_video->height()); + } + + int width() const override { return test_video_->width(); } + int height() const override { return test_video_->height(); } + size_t number_of_frames() const override { + return test_video_->number_of_frames(); + } + + rtc::scoped_refptr<I420BufferInterface> GetFrame( + size_t index) const override { + const rtc::scoped_refptr<I420BufferInterface> reference_frame = + reference_video_->GetFrame(index); + + // Only calculate cropping region once per frame since it's expensive. + if (!crop_regions_.count(index)) { + crop_regions_[index] = + CalculateCropRegion(reference_frame, test_video_->GetFrame(index)); + } + + return CropAndZoom(crop_regions_[index], reference_frame); + } + + private: + const rtc::scoped_refptr<Video> reference_video_; + const rtc::scoped_refptr<Video> test_video_; + // Mutable since this is a cache that affects performance and not logical + // behavior. + mutable std::map<size_t, CropRegion> crop_regions_; + }; + + return rtc::make_ref_counted<CroppedVideo>(reference_video, test_video); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_geometry_aligner.h b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_geometry_aligner.h new file mode 100644 index 0000000000..47667b0d13 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_geometry_aligner.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_TOOLS_FRAME_ANALYZER_VIDEO_GEOMETRY_ALIGNER_H_ +#define RTC_TOOLS_FRAME_ANALYZER_VIDEO_GEOMETRY_ALIGNER_H_ + +#include "api/video/video_frame_buffer.h" +#include "rtc_tools/video_file_reader.h" + +namespace webrtc { +namespace test { + +struct CropRegion { + // Each value represents how much to crop from each side. Left is where x=0, + // and top is where y=0. All values equal to zero represents no cropping. + int left = 0; + int right = 0; + int top = 0; + int bottom = 0; +}; + +// Crops and zooms in on the cropped region so that the returned frame has the +// same resolution as the input frame. +rtc::scoped_refptr<I420BufferInterface> CropAndZoom( + const CropRegion& crop_region, + const rtc::scoped_refptr<I420BufferInterface>& frame); + +// Calculate the optimal cropping region on the reference frame to maximize SSIM +// to the test frame. +CropRegion CalculateCropRegion( + const rtc::scoped_refptr<I420BufferInterface>& reference_frame, + const rtc::scoped_refptr<I420BufferInterface>& test_frame); + +// Returns a cropped and zoomed version of the reference frame that matches up +// to the test frame. This is a simple helper function on top of +// CalculateCropRegion() and CropAndZoom(). +rtc::scoped_refptr<I420BufferInterface> AdjustCropping( + const rtc::scoped_refptr<I420BufferInterface>& reference_frame, + const rtc::scoped_refptr<I420BufferInterface>& test_frame); + +// Returns a cropped and zoomed version of the reference video that matches up +// to the test video. Frames are individually adjusted for cropping. +rtc::scoped_refptr<Video> AdjustCropping( + const rtc::scoped_refptr<Video>& reference_video, + const rtc::scoped_refptr<Video>& test_video); + +} // namespace test +} // namespace webrtc + +#endif // RTC_TOOLS_FRAME_ANALYZER_VIDEO_GEOMETRY_ALIGNER_H_ diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_geometry_aligner_unittest.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_geometry_aligner_unittest.cc new file mode 100644 index 0000000000..a86b8c5f4c --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_geometry_aligner_unittest.cc @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" + +#include <vector> + +#include "api/video/i420_buffer.h" +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" +#include "rtc_tools/video_file_reader.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { + +class VideoGeometryAlignerTest : public ::testing::Test { + protected: + void SetUp() { + reference_video_ = + OpenYuvFile(ResourcePath("foreman_128x96", "yuv"), 128, 96); + ASSERT_TRUE(reference_video_); + + // Very simple 4x4 frame used for verying CropAndZoom. + const uint8_t data_y[] = {0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15}; + const uint8_t data_u[] = {0, 1, 2, 3}; + const uint8_t data_v[] = {0, 1, 2, 3}; + test_frame_ = I420Buffer::Copy( + /* width= */ 4, /* height= */ 4, data_y, /* stride_y= */ 4, data_u, + /* stride_u= */ 2, data_v, /* stride_v= */ 2); + } + + rtc::scoped_refptr<Video> reference_video_; + rtc::scoped_refptr<I420BufferInterface> test_frame_; +}; + +// Teach gtest how to compare CropRegions. +bool operator==(const CropRegion& a, const CropRegion& b) { + return a.left == b.left && a.top == b.top && a.right == b.right && + a.bottom == b.bottom; +} + +TEST_F(VideoGeometryAlignerTest, CropAndZoomIdentity) { + const rtc::scoped_refptr<I420BufferInterface> frame = + reference_video_->GetFrame(0); + + // Assume perfect match, i.e. SSIM == 1. + CropRegion identity_region; + EXPECT_EQ(1.0, Ssim(frame, CropAndZoom(identity_region, frame))); +} + +TEST_F(VideoGeometryAlignerTest, CropAndZoomLeft) { + CropRegion region; + region.left = 2; + const rtc::scoped_refptr<I420BufferInterface> cropped_frame = + CropAndZoom(region, test_frame_); + EXPECT_EQ(std::vector<uint8_t>( + {2, 2, 3, 3, 6, 6, 7, 7, 10, 10, 11, 11, 14, 14, 15, 15}), + std::vector<uint8_t>(cropped_frame->DataY(), + cropped_frame->DataY() + 16)); + EXPECT_EQ( + std::vector<uint8_t>({1, 1, 3, 3}), + std::vector<uint8_t>(cropped_frame->DataU(), cropped_frame->DataU() + 4)); + EXPECT_EQ( + std::vector<uint8_t>({1, 1, 3, 3}), + std::vector<uint8_t>(cropped_frame->DataV(), cropped_frame->DataV() + 4)); +} + +// TODO(magjed): Re-enable when libyuv filtering is updated. +TEST_F(VideoGeometryAlignerTest, DISABLED_CropAndZoomTop) { + CropRegion region; + region.top = 2; + const rtc::scoped_refptr<I420BufferInterface> cropped_frame = + CropAndZoom(region, test_frame_); + EXPECT_EQ(std::vector<uint8_t>( + {8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 12, 13, 14, 15}), + std::vector<uint8_t>(cropped_frame->DataY(), + cropped_frame->DataY() + 16)); + EXPECT_EQ( + std::vector<uint8_t>({2, 3, 2, 3}), + std::vector<uint8_t>(cropped_frame->DataU(), cropped_frame->DataU() + 4)); + EXPECT_EQ( + std::vector<uint8_t>({2, 3, 2, 3}), + std::vector<uint8_t>(cropped_frame->DataV(), cropped_frame->DataV() + 4)); +} + +TEST_F(VideoGeometryAlignerTest, CropAndZoomRight) { + CropRegion region; + region.right = 2; + const rtc::scoped_refptr<I420BufferInterface> cropped_frame = + CropAndZoom(region, test_frame_); + EXPECT_EQ(std::vector<uint8_t>( + {0, 0, 1, 1, 4, 4, 5, 5, 8, 8, 9, 9, 12, 12, 13, 13}), + std::vector<uint8_t>(cropped_frame->DataY(), + cropped_frame->DataY() + 16)); + EXPECT_EQ( + std::vector<uint8_t>({0, 0, 2, 2}), + std::vector<uint8_t>(cropped_frame->DataU(), cropped_frame->DataU() + 4)); + EXPECT_EQ( + std::vector<uint8_t>({0, 0, 2, 2}), + std::vector<uint8_t>(cropped_frame->DataV(), cropped_frame->DataV() + 4)); +} + +// TODO(magjed): Re-enable when libyuv filtering is updated. +TEST_F(VideoGeometryAlignerTest, DISABLED_CropAndZoomBottom) { + CropRegion region; + region.bottom = 2; + const rtc::scoped_refptr<I420BufferInterface> cropped_frame = + CropAndZoom(region, test_frame_); + EXPECT_EQ( + std::vector<uint8_t>({0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 4, 5, 6, 7}), + std::vector<uint8_t>(cropped_frame->DataY(), + cropped_frame->DataY() + 16)); + EXPECT_EQ( + std::vector<uint8_t>({0, 1, 0, 1}), + std::vector<uint8_t>(cropped_frame->DataU(), cropped_frame->DataU() + 4)); + EXPECT_EQ( + std::vector<uint8_t>({0, 1, 0, 1}), + std::vector<uint8_t>(cropped_frame->DataV(), cropped_frame->DataV() + 4)); +} + +TEST_F(VideoGeometryAlignerTest, CalculateCropRegionIdentity) { + const rtc::scoped_refptr<I420BufferInterface> frame = + reference_video_->GetFrame(0); + CropRegion identity_region; + EXPECT_EQ(identity_region, CalculateCropRegion(frame, frame)); +} + +TEST_F(VideoGeometryAlignerTest, CalculateCropRegionArbitrary) { + // Arbitrary crop region. + CropRegion crop_region; + crop_region.left = 2; + crop_region.top = 4; + crop_region.right = 5; + crop_region.bottom = 3; + + const rtc::scoped_refptr<I420BufferInterface> frame = + reference_video_->GetFrame(0); + + EXPECT_EQ(crop_region, + CalculateCropRegion(frame, CropAndZoom(crop_region, frame))); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_quality_analysis.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_quality_analysis.cc new file mode 100644 index 0000000000..1832438b75 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_quality_analysis.cc @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" + +#include <algorithm> +#include <array> +#include <cstddef> + +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv/compare.h" + +namespace webrtc { +namespace test { + +ResultsContainer::ResultsContainer() {} +ResultsContainer::~ResultsContainer() {} + +template <typename FrameMetricFunction> +static double CalculateMetric( + const FrameMetricFunction& frame_metric_function, + const rtc::scoped_refptr<I420BufferInterface>& ref_buffer, + const rtc::scoped_refptr<I420BufferInterface>& test_buffer) { + RTC_CHECK_EQ(ref_buffer->width(), test_buffer->width()); + RTC_CHECK_EQ(ref_buffer->height(), test_buffer->height()); + return frame_metric_function( + ref_buffer->DataY(), ref_buffer->StrideY(), ref_buffer->DataU(), + ref_buffer->StrideU(), ref_buffer->DataV(), ref_buffer->StrideV(), + test_buffer->DataY(), test_buffer->StrideY(), test_buffer->DataU(), + test_buffer->StrideU(), test_buffer->DataV(), test_buffer->StrideV(), + test_buffer->width(), test_buffer->height()); +} + +double Psnr(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer, + const rtc::scoped_refptr<I420BufferInterface>& test_buffer) { + // LibYuv sets the max psnr value to 128, we restrict it to 48. + // In case of 0 mse in one frame, 128 can skew the results significantly. + return std::min(48.0, + CalculateMetric(&libyuv::I420Psnr, ref_buffer, test_buffer)); +} + +double Ssim(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer, + const rtc::scoped_refptr<I420BufferInterface>& test_buffer) { + return CalculateMetric(&libyuv::I420Ssim, ref_buffer, test_buffer); +} + +std::vector<AnalysisResult> RunAnalysis( + const rtc::scoped_refptr<webrtc::test::Video>& reference_video, + const rtc::scoped_refptr<webrtc::test::Video>& test_video, + const std::vector<size_t>& test_frame_indices) { + std::vector<AnalysisResult> results; + for (size_t i = 0; i < test_video->number_of_frames(); ++i) { + const rtc::scoped_refptr<I420BufferInterface>& test_frame = + test_video->GetFrame(i); + const rtc::scoped_refptr<I420BufferInterface>& reference_frame = + reference_video->GetFrame(i); + + // Fill in the result struct. + AnalysisResult result; + result.frame_number = test_frame_indices[i]; + result.psnr_value = Psnr(reference_frame, test_frame); + result.ssim_value = Ssim(reference_frame, test_frame); + results.push_back(result); + } + + return results; +} + +std::vector<Cluster> CalculateFrameClusters( + const std::vector<size_t>& indices) { + std::vector<Cluster> clusters; + + for (size_t index : indices) { + if (!clusters.empty() && clusters.back().index == index) { + // This frame belongs to the previous cluster. + ++clusters.back().number_of_repeated_frames; + } else { + // Start a new cluster. + clusters.push_back({index, /* number_of_repeated_frames= */ 1}); + } + } + + return clusters; +} + +int GetMaxRepeatedFrames(const std::vector<Cluster>& clusters) { + int max_number_of_repeated_frames = 0; + for (const Cluster& cluster : clusters) { + max_number_of_repeated_frames = std::max(max_number_of_repeated_frames, + cluster.number_of_repeated_frames); + } + return max_number_of_repeated_frames; +} + +int GetMaxSkippedFrames(const std::vector<Cluster>& clusters) { + size_t max_skipped_frames = 0; + for (size_t i = 1; i < clusters.size(); ++i) { + const size_t skipped_frames = clusters[i].index - clusters[i - 1].index - 1; + max_skipped_frames = std::max(max_skipped_frames, skipped_frames); + } + return static_cast<int>(max_skipped_frames); +} + +int GetTotalNumberOfSkippedFrames(const std::vector<Cluster>& clusters) { + // The number of reference frames the test video spans. + const size_t number_ref_frames = + clusters.empty() ? 0 : 1 + clusters.back().index - clusters.front().index; + return static_cast<int>(number_ref_frames - clusters.size()); +} + +void PrintAnalysisResults(const std::string& label, + ResultsContainer& results, + MetricsLogger& logger) { + if (results.frames.size() > 0u) { + logger.LogSingleValueMetric("Unique_frames_count", label, + results.frames.size(), Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + + SamplesStatsCounter psnr_values; + SamplesStatsCounter ssim_values; + for (const auto& frame : results.frames) { + psnr_values.AddSample(frame.psnr_value); + ssim_values.AddSample(frame.ssim_value); + } + + logger.LogMetric("PSNR_dB", label, psnr_values, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + logger.LogMetric("SSIM", label, ssim_values, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + } + + logger.LogSingleValueMetric("Max_repeated", label, + results.max_repeated_frames, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + logger.LogSingleValueMetric("Max_skipped", label, results.max_skipped_frames, + Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + logger.LogSingleValueMetric("Total_skipped", label, + results.total_skipped_frames, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + logger.LogSingleValueMetric("Decode_errors_reference", label, + results.decode_errors_ref, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + logger.LogSingleValueMetric("Decode_errors_test", label, + results.decode_errors_test, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_quality_analysis.h b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_quality_analysis.h new file mode 100644 index 0000000000..701b5859b0 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_quality_analysis.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_TOOLS_FRAME_ANALYZER_VIDEO_QUALITY_ANALYSIS_H_ +#define RTC_TOOLS_FRAME_ANALYZER_VIDEO_QUALITY_ANALYSIS_H_ + +#include <stdio.h> + +#include <string> +#include <vector> + +#include "api/scoped_refptr.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/video/video_frame_buffer.h" +#include "rtc_tools/video_file_reader.h" + +namespace webrtc { +namespace test { + +struct AnalysisResult { + AnalysisResult() {} + AnalysisResult(int frame_number, double psnr_value, double ssim_value) + : frame_number(frame_number), + psnr_value(psnr_value), + ssim_value(ssim_value) {} + int frame_number; + double psnr_value; + double ssim_value; +}; + +struct ResultsContainer { + ResultsContainer(); + ~ResultsContainer(); + + std::vector<AnalysisResult> frames; + int max_repeated_frames = 0; + int max_skipped_frames = 0; + int total_skipped_frames = 0; + int decode_errors_ref = 0; + int decode_errors_test = 0; +}; + +// A function to run the PSNR and SSIM analysis on the test file. The test file +// comprises the frames that were captured during the quality measurement test. +// There may be missing or duplicate frames. Also the frames start at a random +// position in the original video. We also need to provide a map from test frame +// indices to reference frame indices. +std::vector<AnalysisResult> RunAnalysis( + const rtc::scoped_refptr<webrtc::test::Video>& reference_video, + const rtc::scoped_refptr<webrtc::test::Video>& test_video, + const std::vector<size_t>& test_frame_indices); + +// Compute PSNR for an I420 buffer (all planes). The max return value (in the +// case where the test and reference frames are exactly the same) will be 48. +double Psnr(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer, + const rtc::scoped_refptr<I420BufferInterface>& test_buffer); + +// Compute SSIM for an I420 buffer (all planes). The max return value (in the +// case where the test and reference frames are exactly the same) will be 1. +double Ssim(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer, + const rtc::scoped_refptr<I420BufferInterface>& test_buffer); + +// Prints the result from the analysis in Chromium performance +// numbers compatible format to stdout. If the results object contains no frames +// no output will be written. +void PrintAnalysisResults(const std::string& label, + ResultsContainer& results, + MetricsLogger& logger); + +struct Cluster { + // Corresponding reference frame index for this cluster. + size_t index; + // The number of sequential frames that mapped to the same reference frame + // index. + int number_of_repeated_frames; +}; + +// Clusters sequentially repeated frames. For example, the sequence {100, 102, +// 102, 103} will be mapped to {{100, 1}, {102, 2}, {103, 1}}. +std::vector<Cluster> CalculateFrameClusters(const std::vector<size_t>& indices); + +// Get number of max sequentially repeated frames in the test video. This number +// will be one if we only store unique frames in the test video. +int GetMaxRepeatedFrames(const std::vector<Cluster>& clusters); + +// Get the longest sequence of skipped reference frames. This corresponds to the +// longest freeze in the test video. +int GetMaxSkippedFrames(const std::vector<Cluster>& clusters); + +// Get total number of skipped frames in the test video. +int GetTotalNumberOfSkippedFrames(const std::vector<Cluster>& clusters); + +} // namespace test +} // namespace webrtc + +#endif // RTC_TOOLS_FRAME_ANALYZER_VIDEO_QUALITY_ANALYSIS_H_ diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_quality_analysis_unittest.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_quality_analysis_unittest.cc new file mode 100644 index 0000000000..d0227fb4b3 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_quality_analysis_unittest.cc @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" + +#include <string> +#include <vector> + +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_logger.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::IsSupersetOf; + +// Metric fields to assert on +struct MetricValidationInfo { + std::string test_case; + std::string name; + Unit unit; + ImprovementDirection improvement_direction; + double mean; +}; + +bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) { + return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit && + a.improvement_direction == b.improvement_direction; +} + +std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) { + os << "{ test_case=" << m.test_case << "; name=" << m.name + << "; unit=" << test::ToString(m.unit) + << "; improvement_direction=" << test::ToString(m.improvement_direction) + << " }"; + return os; +} + +std::vector<MetricValidationInfo> ToValidationInfo( + const std::vector<Metric>& metrics) { + std::vector<MetricValidationInfo> out; + for (const Metric& m : metrics) { + out.push_back( + MetricValidationInfo{.test_case = m.test_case, + .name = m.name, + .unit = m.unit, + .improvement_direction = m.improvement_direction, + .mean = *m.stats.mean}); + } + return out; +} + +TEST(VideoQualityAnalysisTest, PrintAnalysisResultsEmpty) { + ResultsContainer result; + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + PrintAnalysisResults("Empty", result, logger); +} + +TEST(VideoQualityAnalysisTest, PrintAnalysisResultsOneFrame) { + ResultsContainer result; + result.frames.push_back(AnalysisResult(0, 35.0, 0.9)); + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + PrintAnalysisResults("OneFrame", result, logger); +} + +TEST(VideoQualityAnalysisTest, PrintAnalysisResultsThreeFrames) { + ResultsContainer result; + result.frames.push_back(AnalysisResult(0, 35.0, 0.9)); + result.frames.push_back(AnalysisResult(1, 34.0, 0.8)); + result.frames.push_back(AnalysisResult(2, 33.0, 0.7)); + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + PrintAnalysisResults("ThreeFrames", result, logger); +} + +TEST(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesSkippedFrames) { + ResultsContainer result; + + result.max_repeated_frames = 2; + result.max_skipped_frames = 2; + result.total_skipped_frames = 3; + result.decode_errors_ref = 0; + result.decode_errors_test = 0; + + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + PrintAnalysisResults("NormalStatsFile", result, logger); + + std::vector<MetricValidationInfo> metrics = + ToValidationInfo(logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + IsSupersetOf( + {MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Max_repeated", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 2}, + MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Max_skipped", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 2}, + MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Total_skipped", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 3}, + MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Decode_errors_reference", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 0}, + MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Decode_errors_test", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 0}})); +} + +TEST(VideoQualityAnalysisTest, + PrintMaxRepeatedAndSkippedFramesDecodeErrorInTest) { + ResultsContainer result; + + std::string log_filename = + TempFilename(webrtc::test::OutputPath(), "log.log"); + FILE* logfile = fopen(log_filename.c_str(), "w"); + ASSERT_TRUE(logfile != NULL); + + result.max_repeated_frames = 1; + result.max_skipped_frames = 0; + result.total_skipped_frames = 0; + result.decode_errors_ref = 0; + result.decode_errors_test = 3; + + DefaultMetricsLogger logger(Clock::GetRealTimeClock()); + PrintAnalysisResults("NormalStatsFile", result, logger); + + std::vector<MetricValidationInfo> metrics = + ToValidationInfo(logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + IsSupersetOf( + {MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Max_repeated", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 1}, + MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Max_skipped", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 0}, + MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Total_skipped", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 0}, + MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Decode_errors_reference", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 0}, + MetricValidationInfo{ + .test_case = "NormalStatsFile", + .name = "Decode_errors_test", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .mean = 3}})); +} + +TEST(VideoQualityAnalysisTest, CalculateFrameClustersOneValue) { + const std::vector<Cluster> result = CalculateFrameClusters({1}); + EXPECT_EQ(1u, result.size()); + EXPECT_EQ(1u, result[0].index); + EXPECT_EQ(1, result[0].number_of_repeated_frames); +} + +TEST(VideoQualityAnalysisTest, GetMaxRepeatedFramesOneValue) { + EXPECT_EQ(1, GetMaxRepeatedFrames(CalculateFrameClusters({1}))); +} + +TEST(VideoQualityAnalysisTest, GetMaxSkippedFramesOneValue) { + EXPECT_EQ(0, GetMaxSkippedFrames(CalculateFrameClusters({1}))); +} + +TEST(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesOneValue) { + EXPECT_EQ(0, GetTotalNumberOfSkippedFrames(CalculateFrameClusters({1}))); +} + +TEST(VideoQualityAnalysisTest, CalculateFrameClustersOneOneTwo) { + const std::vector<Cluster> result = CalculateFrameClusters({1, 1, 2}); + EXPECT_EQ(2u, result.size()); + EXPECT_EQ(1u, result[0].index); + EXPECT_EQ(2, result[0].number_of_repeated_frames); + EXPECT_EQ(2u, result[1].index); + EXPECT_EQ(1, result[1].number_of_repeated_frames); +} + +TEST(VideoQualityAnalysisTest, GetMaxRepeatedFramesOneOneTwo) { + EXPECT_EQ(2, GetMaxRepeatedFrames(CalculateFrameClusters({1, 1, 2}))); +} + +TEST(VideoQualityAnalysisTest, GetMaxSkippedFramesOneOneTwo) { + EXPECT_EQ(0, GetMaxSkippedFrames(CalculateFrameClusters({1, 1, 2}))); +} + +TEST(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesOneOneTwo) { + EXPECT_EQ(0, + GetTotalNumberOfSkippedFrames(CalculateFrameClusters({1, 1, 2}))); +} + +TEST(VideoQualityAnalysisTest, CalculateFrameClustersEmpty) { + EXPECT_TRUE(CalculateFrameClusters({}).empty()); +} + +TEST(VideoQualityAnalysisTest, GetMaxRepeatedFramesEmpty) { + EXPECT_EQ(0, GetMaxRepeatedFrames({})); +} + +TEST(VideoQualityAnalysisTest, GetMaxSkippedFramesEmpty) { + EXPECT_EQ(0, GetMaxSkippedFrames({})); +} + +TEST(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesEmpty) { + EXPECT_EQ(0, GetTotalNumberOfSkippedFrames({})); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_temporal_aligner.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_temporal_aligner.cc new file mode 100644 index 0000000000..4b940d0fe3 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_temporal_aligner.cc @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/video_temporal_aligner.h" + +#include <algorithm> +#include <cmath> +#include <cstddef> +#include <deque> +#include <iterator> +#include <limits> +#include <vector> + +#include "api/make_ref_counted.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame_buffer.h" +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" + +namespace webrtc { +namespace test { + +namespace { + +// This constant controls how many frames we look ahead while seeking for the +// match for the next frame. Note that we may span bigger gaps than this number +// since we reset the counter as soon as we find a better match. The seeking +// will stop when there is no improvement in the next kNumberOfFramesLookAhead +// frames. Typically, the SSIM will improve as we get closer and closer to the +// real match. +const int kNumberOfFramesLookAhead = 60; + +// Helper class that takes a video and generates an infinite looping video. +class LoopingVideo : public Video { + public: + explicit LoopingVideo(const rtc::scoped_refptr<Video>& video) + : video_(video) {} + + int width() const override { return video_->width(); } + int height() const override { return video_->height(); } + size_t number_of_frames() const override { + return std::numeric_limits<size_t>::max(); + } + + rtc::scoped_refptr<I420BufferInterface> GetFrame( + size_t index) const override { + return video_->GetFrame(index % video_->number_of_frames()); + } + + private: + const rtc::scoped_refptr<Video> video_; +}; + +// Helper class that take a vector of frame indices and a video and produces a +// new video where the frames have been reshuffled. +class ReorderedVideo : public Video { + public: + ReorderedVideo(const rtc::scoped_refptr<Video>& video, + const std::vector<size_t>& indices) + : video_(video), indices_(indices) {} + + int width() const override { return video_->width(); } + int height() const override { return video_->height(); } + size_t number_of_frames() const override { return indices_.size(); } + + rtc::scoped_refptr<I420BufferInterface> GetFrame( + size_t index) const override { + return video_->GetFrame(indices_.at(index)); + } + + private: + const rtc::scoped_refptr<Video> video_; + const std::vector<size_t> indices_; +}; + +// Helper class that takes a video and produces a downscaled video. +class DownscaledVideo : public Video { + public: + DownscaledVideo(float scale_factor, const rtc::scoped_refptr<Video>& video) + : downscaled_width_( + static_cast<int>(std::round(scale_factor * video->width()))), + downscaled_height_( + static_cast<int>(std::round(scale_factor * video->height()))), + video_(video) {} + + int width() const override { return downscaled_width_; } + int height() const override { return downscaled_height_; } + size_t number_of_frames() const override { + return video_->number_of_frames(); + } + + rtc::scoped_refptr<I420BufferInterface> GetFrame( + size_t index) const override { + const rtc::scoped_refptr<I420BufferInterface> frame = + video_->GetFrame(index); + rtc::scoped_refptr<I420Buffer> downscaled_frame = + I420Buffer::Create(downscaled_width_, downscaled_height_); + downscaled_frame->ScaleFrom(*frame); + return downscaled_frame; + } + + private: + const int downscaled_width_; + const int downscaled_height_; + const rtc::scoped_refptr<Video> video_; +}; + +// Helper class that takes a video and caches the latest frame access. This +// improves performance a lot since the original source is often from a file. +class CachedVideo : public Video { + public: + CachedVideo(int max_cache_size, const rtc::scoped_refptr<Video>& video) + : max_cache_size_(max_cache_size), video_(video) {} + + int width() const override { return video_->width(); } + int height() const override { return video_->height(); } + size_t number_of_frames() const override { + return video_->number_of_frames(); + } + + rtc::scoped_refptr<I420BufferInterface> GetFrame( + size_t index) const override { + for (const CachedFrame& cached_frame : cache_) { + if (cached_frame.index == index) + return cached_frame.frame; + } + + rtc::scoped_refptr<I420BufferInterface> frame = video_->GetFrame(index); + cache_.push_front({index, frame}); + if (cache_.size() > max_cache_size_) + cache_.pop_back(); + + return frame; + } + + private: + struct CachedFrame { + size_t index; + rtc::scoped_refptr<I420BufferInterface> frame; + }; + + const size_t max_cache_size_; + const rtc::scoped_refptr<Video> video_; + mutable std::deque<CachedFrame> cache_; +}; + +// Try matching the test frame against all frames in the reference video and +// return the index of the best matching frame. +size_t FindBestMatch(const rtc::scoped_refptr<I420BufferInterface>& test_frame, + const Video& reference_video) { + std::vector<double> ssim; + for (const auto& ref_frame : reference_video) + ssim.push_back(Ssim(test_frame, ref_frame)); + return std::distance(ssim.begin(), + std::max_element(ssim.begin(), ssim.end())); +} + +// Find and return the index of the frame matching the test frame. The search +// starts at the starting index and continues until there is no better match +// within the next kNumberOfFramesLookAhead frames. +size_t FindNextMatch(const rtc::scoped_refptr<I420BufferInterface>& test_frame, + const Video& reference_video, + size_t start_index) { + const double start_ssim = + Ssim(test_frame, reference_video.GetFrame(start_index)); + for (int i = 1; i < kNumberOfFramesLookAhead; ++i) { + const size_t next_index = start_index + i; + // If we find a better match, restart the search at that point. + if (start_ssim < Ssim(test_frame, reference_video.GetFrame(next_index))) + return FindNextMatch(test_frame, reference_video, next_index); + } + // The starting index was the best match. + return start_index; +} + +} // namespace + +std::vector<size_t> FindMatchingFrameIndices( + const rtc::scoped_refptr<Video>& reference_video, + const rtc::scoped_refptr<Video>& test_video) { + // This is done to get a 10x speedup. We don't need the full resolution in + // order to match frames, and we should limit file access and not read the + // same memory tens of times. + const float kScaleFactor = 0.25f; + const rtc::scoped_refptr<Video> cached_downscaled_reference_video = + rtc::make_ref_counted<CachedVideo>(kNumberOfFramesLookAhead, + rtc::make_ref_counted<DownscaledVideo>( + kScaleFactor, reference_video)); + const rtc::scoped_refptr<Video> downscaled_test_video = + rtc::make_ref_counted<DownscaledVideo>(kScaleFactor, test_video); + + // Assume the video is looping around. + const rtc::scoped_refptr<Video> looping_reference_video = + rtc::make_ref_counted<LoopingVideo>(cached_downscaled_reference_video); + + std::vector<size_t> match_indices; + for (const rtc::scoped_refptr<I420BufferInterface>& test_frame : + *downscaled_test_video) { + if (match_indices.empty()) { + // First frame. + match_indices.push_back( + FindBestMatch(test_frame, *cached_downscaled_reference_video)); + } else { + match_indices.push_back(FindNextMatch( + test_frame, *looping_reference_video, match_indices.back())); + } + } + + return match_indices; +} + +rtc::scoped_refptr<Video> ReorderVideo(const rtc::scoped_refptr<Video>& video, + const std::vector<size_t>& indices) { + return rtc::make_ref_counted<ReorderedVideo>( + rtc::make_ref_counted<LoopingVideo>(video), indices); +} + +rtc::scoped_refptr<Video> GenerateAlignedReferenceVideo( + const rtc::scoped_refptr<Video>& reference_video, + const rtc::scoped_refptr<Video>& test_video) { + return GenerateAlignedReferenceVideo( + reference_video, FindMatchingFrameIndices(reference_video, test_video)); +} + +rtc::scoped_refptr<Video> GenerateAlignedReferenceVideo( + const rtc::scoped_refptr<Video>& reference_video, + const std::vector<size_t>& indices) { + return ReorderVideo(reference_video, indices); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_temporal_aligner.h b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_temporal_aligner.h new file mode 100644 index 0000000000..26a4088815 --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_temporal_aligner.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_TOOLS_FRAME_ANALYZER_VIDEO_TEMPORAL_ALIGNER_H_ +#define RTC_TOOLS_FRAME_ANALYZER_VIDEO_TEMPORAL_ALIGNER_H_ + +#include <stddef.h> + +#include <vector> + +#include "api/scoped_refptr.h" +#include "rtc_tools/video_file_reader.h" + +namespace webrtc { +namespace test { + +// Returns a vector with the same size as the given test video. Each index +// corresponds to what reference frame that test frame matches to. These +// indices are strictly increasing and might loop around the reference video, +// e.g. their values can be bigger than the number of frames in the reference +// video and they should be interpreted modulo that size. The matching frames +// will be determined by maximizing SSIM. +std::vector<size_t> FindMatchingFrameIndices( + const rtc::scoped_refptr<Video>& reference_video, + const rtc::scoped_refptr<Video>& test_video); + +// Generate a new video using the frames from the original video. The returned +// video will have the same number of frames as the size of `indices`, and +// frame nr i in the returned video will point to frame nr indices[i] in the +// original video. +rtc::scoped_refptr<Video> ReorderVideo(const rtc::scoped_refptr<Video>& video, + const std::vector<size_t>& indices); + +// Returns a modified version of the reference video where the frames have +// been aligned to the test video. The test video is assumed to be captured +// during a quality measurement test where the reference video is the source. +// The test video may start at an arbitrary position in the reference video +// and there might be missing frames. The reference video is assumed to loop +// over when it reaches the end. The returned result is a version of the +// reference video where the missing frames are left out so it aligns to the +// test video. +rtc::scoped_refptr<Video> GenerateAlignedReferenceVideo( + const rtc::scoped_refptr<Video>& reference_video, + const rtc::scoped_refptr<Video>& test_video); + +// As above, but using precalculated indices. +rtc::scoped_refptr<Video> GenerateAlignedReferenceVideo( + const rtc::scoped_refptr<Video>& reference_video, + const std::vector<size_t>& indices); + +} // namespace test +} // namespace webrtc + +#endif // RTC_TOOLS_FRAME_ANALYZER_VIDEO_TEMPORAL_ALIGNER_H_ diff --git a/third_party/libwebrtc/rtc_tools/frame_analyzer/video_temporal_aligner_unittest.cc b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_temporal_aligner_unittest.cc new file mode 100644 index 0000000000..9519a7462b --- /dev/null +++ b/third_party/libwebrtc/rtc_tools/frame_analyzer/video_temporal_aligner_unittest.cc @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/video_temporal_aligner.h" + +#include <cstddef> + +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" +#include "rtc_tools/video_file_reader.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { + +class VideoTemporalAlignerTest : public ::testing::Test { + protected: + void SetUp() { + reference_video = + OpenYuvFile(ResourcePath("foreman_128x96", "yuv"), 128, 96); + ASSERT_TRUE(reference_video); + } + + rtc::scoped_refptr<Video> reference_video; +}; + +TEST_F(VideoTemporalAlignerTest, FindMatchingFrameIndicesEmpty) { + rtc::scoped_refptr<Video> empty_test_video = + ReorderVideo(reference_video, std::vector<size_t>()); + + const std::vector<size_t> matched_indices = + FindMatchingFrameIndices(reference_video, empty_test_video); + + EXPECT_TRUE(matched_indices.empty()); +} + +TEST_F(VideoTemporalAlignerTest, FindMatchingFrameIndicesIdentity) { + const std::vector<size_t> indices = + FindMatchingFrameIndices(reference_video, reference_video); + + EXPECT_EQ(indices.size(), reference_video->number_of_frames()); + for (size_t i = 0; i < indices.size(); ++i) + EXPECT_EQ(i, indices[i]); +} + +TEST_F(VideoTemporalAlignerTest, FindMatchingFrameIndicesDuplicateFrames) { + const std::vector<size_t> indices = {2, 2, 2, 2}; + + // Generate a test video based on this sequence. + rtc::scoped_refptr<Video> test_video = ReorderVideo(reference_video, indices); + + const std::vector<size_t> matched_indices = + FindMatchingFrameIndices(reference_video, test_video); + + EXPECT_EQ(indices, matched_indices); +} + +TEST_F(VideoTemporalAlignerTest, FindMatchingFrameIndicesLoopAround) { + std::vector<size_t> indices; + for (size_t i = 0; i < reference_video->number_of_frames() * 2; ++i) + indices.push_back(i % reference_video->number_of_frames()); + + // Generate a test video based on this sequence. + rtc::scoped_refptr<Video> test_video = ReorderVideo(reference_video, indices); + + const std::vector<size_t> matched_indices = + FindMatchingFrameIndices(reference_video, test_video); + + for (size_t i = 0; i < matched_indices.size(); ++i) + EXPECT_EQ(i, matched_indices[i]); +} + +TEST_F(VideoTemporalAlignerTest, FindMatchingFrameIndicesStressTest) { + std::vector<size_t> indices; + // Arbitrary start index. + const size_t start_index = 12345; + // Generate some generic sequence of frames. + indices.push_back(start_index % reference_video->number_of_frames()); + indices.push_back((start_index + 1) % reference_video->number_of_frames()); + indices.push_back((start_index + 2) % reference_video->number_of_frames()); + indices.push_back((start_index + 5) % reference_video->number_of_frames()); + indices.push_back((start_index + 10) % reference_video->number_of_frames()); + indices.push_back((start_index + 20) % reference_video->number_of_frames()); + indices.push_back((start_index + 20) % reference_video->number_of_frames()); + indices.push_back((start_index + 22) % reference_video->number_of_frames()); + indices.push_back((start_index + 32) % reference_video->number_of_frames()); + + // Generate a test video based on this sequence. + rtc::scoped_refptr<Video> test_video = ReorderVideo(reference_video, indices); + + const std::vector<size_t> matched_indices = + FindMatchingFrameIndices(reference_video, test_video); + + EXPECT_EQ(indices, matched_indices); +} + +TEST_F(VideoTemporalAlignerTest, GenerateAlignedReferenceVideo) { + // Arbitrary start index. + const size_t start_index = 12345; + std::vector<size_t> indices; + const size_t frame_step = 10; + for (size_t i = 0; i < reference_video->number_of_frames() / frame_step; + ++i) { + indices.push_back((start_index + i * frame_step) % + reference_video->number_of_frames()); + } + + // Generate a test video based on this sequence. + rtc::scoped_refptr<Video> test_video = ReorderVideo(reference_video, indices); + + rtc::scoped_refptr<Video> aligned_reference_video = + GenerateAlignedReferenceVideo(reference_video, test_video); + + // Assume perfect match, i.e. ssim == 1, for all frames. + for (size_t i = 0; i < test_video->number_of_frames(); ++i) { + EXPECT_EQ(1.0, Ssim(test_video->GetFrame(i), + aligned_reference_video->GetFrame(i))); + } +} + +} // namespace test +} // namespace webrtc |