diff options
Diffstat (limited to 'debian/patches/fuzzy_tests.patch')
| -rw-r--r-- | debian/patches/fuzzy_tests.patch | 181 |
1 files changed, 181 insertions, 0 deletions
diff --git a/debian/patches/fuzzy_tests.patch b/debian/patches/fuzzy_tests.patch new file mode 100644 index 0000000..9d4e884 --- /dev/null +++ b/debian/patches/fuzzy_tests.patch @@ -0,0 +1,181 @@ +From 9e26e6cb774831fc07e3788bd63b51f489369b21 Mon Sep 17 00:00:00 2001 +From: Rafael Siejakowski <rs@rs-math.net> +Date: Sun, 10 Jul 2022 17:16:55 -0300 +Subject: [PATCH 1/2] Fuzzy bitmap comparison in CLI rendering tests + +Enhance the CLI testing framework with the ability to compare raster +output (either produced by Inkscape or converted from a vector format +with ImageMagick) using the L2 distance in the image space. Each such +"fuzzy" test can set a maximum allowed percentage difference between +the two compared images. This technique is applied to four multipage +output tests, fixing the spurious failures due to subtle differences +in rasterization artifacts between platforms and library versions. + +Fixes https://gitlab.com/inkscape/inbox/-/issues/7304 +--- + testfiles/cli_tests/CMakeLists.txt | 37 +++++-- + testfiles/cli_tests/l2compare.sh | 102 ++++++++++++++++++ + .../export-filtered-clones-mp_expected.png | Bin 872 -> 5034 bytes + 3 files changed, 133 insertions(+), 6 deletions(-) + create mode 100755 testfiles/cli_tests/l2compare.sh + +--- a/testfiles/cli_tests/CMakeLists.txt ++++ b/testfiles/cli_tests/CMakeLists.txt +@@ -9,6 +9,8 @@ + # INPUT_FILENAME - name of input file (optional) + # OUTPUT_FILENAME - name of output file (optional) + # OUTPUT_PAGE - index of page in multipage output (optional), starts from 0 ++# FUZZ_PERCENTAGE - maximum allowed normalized root-mean-squared distance between compared images ++# RASTER_DPI - DPI setting for rasterizing vector formats before root-mean-squared comparison + # PARAMETERS - additional command line parameters to pass to Inkscape + # + # Pass/fail criteria: +@@ -19,6 +21,8 @@ + # REFERENCE_FILENAME - compare OUTPUT_FILENAME with this pre-rendered reference file + # both files are converted to PNG and compared with ImageMagick's 'compare' + # for multipage output, use OUTPUT_PAGE to specify a single page for comparison ++# FUZZYREF_FILENAME - comparison of OUTPUT_FILENAME with this pre-rendered reference file will be ++# performed in the L2 metric, subject to the specified FUZZ_PERCENTAGE + # EXPECTED_FILES - verify the command produced the expected files (i.e. they exist on disk) + # TEST_SCRIPT - additional script to run after performing all checks and before cleaning up + # +@@ -26,7 +30,8 @@ + # ENVIRONMENT - Additional environment variables to set while running the test + function(add_cli_test name) + # parse arguments +- set(oneValueArgs INPUT_FILENAME OUTPUT_FILENAME OUTPUT_PAGE PASS_FOR_OUTPUT FAIL_FOR_OUTPUT REFERENCE_FILENAME) ++ set(oneValueArgs INPUT_FILENAME OUTPUT_FILENAME OUTPUT_PAGE PASS_FOR_OUTPUT FAIL_FOR_OUTPUT REFERENCE_FILENAME ++ FUZZYREF_FILENAME FUZZ_PERCENTAGE RASTER_DPI) + set(multiValueArgs PARAMETERS EXPECTED_FILES TEST_SCRIPT ENVIRONMENT) + cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + +@@ -85,8 +90,23 @@ + set_tests_properties(${testname}_check_output PROPERTIES + ENVIRONMENT "${CMAKE_CTEST_ENV}" DEPENDS ${testname} SKIP_RETURN_CODE 42) + endif() +-endfunction(add_cli_test) + ++ # add a fuzzy test to check the output files ++ if(DEFINED ARG_FUZZYREF_FILENAME) ++ file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/testcases/${ARG_FUZZYREF_FILENAME}" ARG_FUZZYREF_FILENAME) ++ if(DEFINED ARG_FUZZ_PERCENTAGE) ++ set(ARG_FUZZ "${ARG_FUZZ_PERCENTAGE}") ++ else() ++ set(ARG_FUZZ "0") ++ endif() ++ add_test(NAME ${testname}_check_output ++ COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/l2compare.sh ++ "${ARG_OUTPUT_FILENAME}" "${ARG_OUTPUT_PAGE}" "${ARG_FUZZYREF_FILENAME}" "${ARG_FUZZ}" "${ARG_RASTER_DPI}") ++ set_tests_properties(${testname}_check_output PROPERTIES ++ ENVIRONMENT "${CMAKE_CTEST_ENV}" DEPENDS ${testname} SKIP_RETURN_CODE 42) ++ endif() ++ ++endfunction(add_cli_test) + + + ##### Tests follow below ##### +--- /dev/null ++++ b/testfiles/cli_tests/l2compare.sh +@@ -0,0 +1,102 @@ ++#!/bin/bash ++# SPDX-License-Identifier: GPL-2.0-or-later ++# ++# Convert an image (or a single page of a PDF/PostScript document) to a bitmap ++# and calculate the relative root-mean-squared (L2) distance from the reference. ++# ++# Authors: ++# Rafael Siejakowski <rs@rs-math.net> ++# ++# Copyright (C) 2022 Authors ++# ++# Released under GNU GPL v2+, read the file 'COPYING' for more information. ++# ++ ++ensure_command() ++{ ++ command -v $1 >/dev/null 2>&1 || { echo >&2 "Required command '$1' not found. Aborting."; exit 1; } ++} ++ ++ensure_command "convert" ++ensure_command "compare" ++ensure_command "bc" ++ensure_command "cp" ++ ++OUTPUT_FILENAME="$1" ++OUTPUT_PAGE="$2" ++REFERENCE_FILENAME="$3" ++PERCENTAGE_DIFFERENCE_ALLOWED="$4" ++DPI="$5" ++ ++if [ ! -f "${OUTPUT_FILENAME}" ] ++then ++ echo "Error: Test file '${OUTPUT_FILENAME}' not found." ++ exit 1 ++fi ++ ++if [ ! -f "${REFERENCE_FILENAME}" ] ++then ++ echo "Error: Reference file '${REFERENCE_FILENAME}' not found." ++ exit 1 ++fi ++ ++# Convert the output file to the PNG format ++CONVERSION_OPTIONS="-colorspace RGB" ++ ++# Extract a page from multipage PS/PDF if requested ++OUTFILE_SUFFIX="" ++if [[ "x$OUTPUT_PAGE" != "x" ]] ++then ++ OUTFILE_SUFFIX="[${OUTPUT_PAGE}]" # Use ImageMagick's bracket operator ++fi ++ ++DPI_OPTION="" ++if [[ "x$DPI" != "x" ]] ++then ++ DPI_OPTION="-density $DPI" ++fi ++ ++if ! convert $DPI_OPTION "${OUTPUT_FILENAME}${OUTFILE_SUFFIX}" $CONVERSION_OPTIONS "${OUTPUT_FILENAME}-output.png" ++then ++ echo "Warning: Failed to convert test file '${OUTPUT_FILENAME}' to PNG format. Skipping comparison test." ++ exit 42 ++fi ++ ++# Copy the reference file ++cp "${REFERENCE_FILENAME}" "${OUTPUT_FILENAME}-reference.png" ++ ++# Compare the two files ++COMPARE_RESULT=$(compare 2>&1 -metric RMSE "${OUTPUT_FILENAME}-output.png" "${OUTPUT_FILENAME}-reference.png" \ ++ "${OUTPUT_FILENAME}-diff.png") ++COMPARE_RESULT=${COMPARE_RESULT#*(} ++RELATIVE_ERROR=${COMPARE_RESULT%)*} ++if [[ "x$RELATIVE_ERROR" == "x" ]] ++then ++ echo "Warning: Could not parse out the relative RMS error for fuzzy comparison. Skipping comparison test." ++ exit 42 ++fi ++ ++# Check if the difference between the files is within tolerance ++CONDITION="$RELATIVE_ERROR * 100 <= $PERCENTAGE_DIFFERENCE_ALLOWED" ++WITHIN_TOLERANCE=$(echo "${CONDITION}" | bc) ++if [[ $? -ne 0 ]] ++then ++ echo "Warning: An error occurred running 'bc'. The fuzzy comparison test will be skipped." ++ exit 42 ++fi ++ ++PERCENTAGE_ERROR=$(echo "$RELATIVE_ERROR * 100" | bc) ++if (( $WITHIN_TOLERANCE )) ++then ++ # Test passed: print stats and clean up the files. ++ echo "Fuzzy comparison PASSED; error of ${PERCENTAGE_ERROR}% is within ${PERCENTAGE_DIFFERENCE_ALLOWED}% tolerance." ++ for FILE in ${OUTPUT_FILENAME}{,-reference.png,-output.png,-diff.png} ++ do ++ rm -f "${FILE}" ++ done ++else ++ # Test failed! ++ echo "Fuzzy comparison FAILED; error of ${PERCENTAGE_ERROR}% exceeds ${PERCENTAGE_DIFFERENCE_ALLOWED}% tolerance." ++ exit 1 ++fi ++ |