From 9e26e6cb774831fc07e3788bd63b51f489369b21 Mon Sep 17 00:00:00 2001 From: Rafael Siejakowski Date: Sun, 10 Jul 2022 17:16:55 -0300 Subject: [PATCH 1/2] Fuzzy bitmap comparison in CLI rendering tests Enhance the CLI testing framework with the ability to compare raster output (either produced by Inkscape or converted from a vector format with ImageMagick) using the L2 distance in the image space. Each such "fuzzy" test can set a maximum allowed percentage difference between the two compared images. This technique is applied to four multipage output tests, fixing the spurious failures due to subtle differences in rasterization artifacts between platforms and library versions. Fixes https://gitlab.com/inkscape/inbox/-/issues/7304 --- testfiles/cli_tests/CMakeLists.txt | 37 +++++-- testfiles/cli_tests/l2compare.sh | 102 ++++++++++++++++++ .../export-filtered-clones-mp_expected.png | Bin 872 -> 5034 bytes 3 files changed, 133 insertions(+), 6 deletions(-) create mode 100755 testfiles/cli_tests/l2compare.sh --- a/testfiles/cli_tests/CMakeLists.txt +++ b/testfiles/cli_tests/CMakeLists.txt @@ -9,6 +9,8 @@ # INPUT_FILENAME - name of input file (optional) # OUTPUT_FILENAME - name of output file (optional) # OUTPUT_PAGE - index of page in multipage output (optional), starts from 0 +# FUZZ_PERCENTAGE - maximum allowed normalized root-mean-squared distance between compared images +# RASTER_DPI - DPI setting for rasterizing vector formats before root-mean-squared comparison # PARAMETERS - additional command line parameters to pass to Inkscape # # Pass/fail criteria: @@ -19,6 +21,8 @@ # REFERENCE_FILENAME - compare OUTPUT_FILENAME with this pre-rendered reference file # both files are converted to PNG and compared with ImageMagick's 'compare' # for multipage output, use OUTPUT_PAGE to specify a single page for comparison +# FUZZYREF_FILENAME - comparison of OUTPUT_FILENAME with this pre-rendered reference file will be +# performed in the L2 metric, subject to the specified FUZZ_PERCENTAGE # EXPECTED_FILES - verify the command produced the expected files (i.e. they exist on disk) # TEST_SCRIPT - additional script to run after performing all checks and before cleaning up # @@ -26,7 +30,8 @@ # ENVIRONMENT - Additional environment variables to set while running the test function(add_cli_test name) # parse arguments - set(oneValueArgs INPUT_FILENAME OUTPUT_FILENAME OUTPUT_PAGE PASS_FOR_OUTPUT FAIL_FOR_OUTPUT REFERENCE_FILENAME) + set(oneValueArgs INPUT_FILENAME OUTPUT_FILENAME OUTPUT_PAGE PASS_FOR_OUTPUT FAIL_FOR_OUTPUT REFERENCE_FILENAME + FUZZYREF_FILENAME FUZZ_PERCENTAGE RASTER_DPI) set(multiValueArgs PARAMETERS EXPECTED_FILES TEST_SCRIPT ENVIRONMENT) cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) @@ -85,8 +90,23 @@ set_tests_properties(${testname}_check_output PROPERTIES ENVIRONMENT "${CMAKE_CTEST_ENV}" DEPENDS ${testname} SKIP_RETURN_CODE 42) endif() -endfunction(add_cli_test) + # add a fuzzy test to check the output files + if(DEFINED ARG_FUZZYREF_FILENAME) + file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/testcases/${ARG_FUZZYREF_FILENAME}" ARG_FUZZYREF_FILENAME) + if(DEFINED ARG_FUZZ_PERCENTAGE) + set(ARG_FUZZ "${ARG_FUZZ_PERCENTAGE}") + else() + set(ARG_FUZZ "0") + endif() + add_test(NAME ${testname}_check_output + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/l2compare.sh + "${ARG_OUTPUT_FILENAME}" "${ARG_OUTPUT_PAGE}" "${ARG_FUZZYREF_FILENAME}" "${ARG_FUZZ}" "${ARG_RASTER_DPI}") + set_tests_properties(${testname}_check_output PROPERTIES + ENVIRONMENT "${CMAKE_CTEST_ENV}" DEPENDS ${testname} SKIP_RETURN_CODE 42) + endif() + +endfunction(add_cli_test) ##### Tests follow below ##### --- /dev/null +++ b/testfiles/cli_tests/l2compare.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Convert an image (or a single page of a PDF/PostScript document) to a bitmap +# and calculate the relative root-mean-squared (L2) distance from the reference. +# +# Authors: +# Rafael Siejakowski +# +# Copyright (C) 2022 Authors +# +# Released under GNU GPL v2+, read the file 'COPYING' for more information. +# + +ensure_command() +{ + command -v $1 >/dev/null 2>&1 || { echo >&2 "Required command '$1' not found. Aborting."; exit 1; } +} + +ensure_command "convert" +ensure_command "compare" +ensure_command "bc" +ensure_command "cp" + +OUTPUT_FILENAME="$1" +OUTPUT_PAGE="$2" +REFERENCE_FILENAME="$3" +PERCENTAGE_DIFFERENCE_ALLOWED="$4" +DPI="$5" + +if [ ! -f "${OUTPUT_FILENAME}" ] +then + echo "Error: Test file '${OUTPUT_FILENAME}' not found." + exit 1 +fi + +if [ ! -f "${REFERENCE_FILENAME}" ] +then + echo "Error: Reference file '${REFERENCE_FILENAME}' not found." + exit 1 +fi + +# Convert the output file to the PNG format +CONVERSION_OPTIONS="-colorspace RGB" + +# Extract a page from multipage PS/PDF if requested +OUTFILE_SUFFIX="" +if [[ "x$OUTPUT_PAGE" != "x" ]] +then + OUTFILE_SUFFIX="[${OUTPUT_PAGE}]" # Use ImageMagick's bracket operator +fi + +DPI_OPTION="" +if [[ "x$DPI" != "x" ]] +then + DPI_OPTION="-density $DPI" +fi + +if ! convert $DPI_OPTION "${OUTPUT_FILENAME}${OUTFILE_SUFFIX}" $CONVERSION_OPTIONS "${OUTPUT_FILENAME}-output.png" +then + echo "Warning: Failed to convert test file '${OUTPUT_FILENAME}' to PNG format. Skipping comparison test." + exit 42 +fi + +# Copy the reference file +cp "${REFERENCE_FILENAME}" "${OUTPUT_FILENAME}-reference.png" + +# Compare the two files +COMPARE_RESULT=$(compare 2>&1 -metric RMSE "${OUTPUT_FILENAME}-output.png" "${OUTPUT_FILENAME}-reference.png" \ + "${OUTPUT_FILENAME}-diff.png") +COMPARE_RESULT=${COMPARE_RESULT#*(} +RELATIVE_ERROR=${COMPARE_RESULT%)*} +if [[ "x$RELATIVE_ERROR" == "x" ]] +then + echo "Warning: Could not parse out the relative RMS error for fuzzy comparison. Skipping comparison test." + exit 42 +fi + +# Check if the difference between the files is within tolerance +CONDITION="$RELATIVE_ERROR * 100 <= $PERCENTAGE_DIFFERENCE_ALLOWED" +WITHIN_TOLERANCE=$(echo "${CONDITION}" | bc) +if [[ $? -ne 0 ]] +then + echo "Warning: An error occurred running 'bc'. The fuzzy comparison test will be skipped." + exit 42 +fi + +PERCENTAGE_ERROR=$(echo "$RELATIVE_ERROR * 100" | bc) +if (( $WITHIN_TOLERANCE )) +then + # Test passed: print stats and clean up the files. + echo "Fuzzy comparison PASSED; error of ${PERCENTAGE_ERROR}% is within ${PERCENTAGE_DIFFERENCE_ALLOWED}% tolerance." + for FILE in ${OUTPUT_FILENAME}{,-reference.png,-output.png,-diff.png} + do + rm -f "${FILE}" + done +else + # Test failed! + echo "Fuzzy comparison FAILED; error of ${PERCENTAGE_ERROR}% exceeds ${PERCENTAGE_DIFFERENCE_ALLOWED}% tolerance." + exit 1 +fi +