1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
From 9e26e6cb774831fc07e3788bd63b51f489369b21 Mon Sep 17 00:00:00 2001
From: Rafael Siejakowski <rs@rs-math.net>
Date: Sun, 10 Jul 2022 17:16:55 -0300
Subject: [PATCH 1/2] Fuzzy bitmap comparison in CLI rendering tests
Enhance the CLI testing framework with the ability to compare raster
output (either produced by Inkscape or converted from a vector format
with ImageMagick) using the L2 distance in the image space. Each such
"fuzzy" test can set a maximum allowed percentage difference between
the two compared images. This technique is applied to four multipage
output tests, fixing the spurious failures due to subtle differences
in rasterization artifacts between platforms and library versions.
Fixes https://gitlab.com/inkscape/inbox/-/issues/7304
---
testfiles/cli_tests/CMakeLists.txt | 37 +++++--
testfiles/cli_tests/l2compare.sh | 102 ++++++++++++++++++
.../export-filtered-clones-mp_expected.png | Bin 872 -> 5034 bytes
3 files changed, 133 insertions(+), 6 deletions(-)
create mode 100755 testfiles/cli_tests/l2compare.sh
--- a/testfiles/cli_tests/CMakeLists.txt
+++ b/testfiles/cli_tests/CMakeLists.txt
@@ -9,6 +9,8 @@
# INPUT_FILENAME - name of input file (optional)
# OUTPUT_FILENAME - name of output file (optional)
# OUTPUT_PAGE - index of page in multipage output (optional), starts from 0
+# FUZZ_PERCENTAGE - maximum allowed normalized root-mean-squared distance between compared images
+# RASTER_DPI - DPI setting for rasterizing vector formats before root-mean-squared comparison
# PARAMETERS - additional command line parameters to pass to Inkscape
#
# Pass/fail criteria:
@@ -19,6 +21,8 @@
# REFERENCE_FILENAME - compare OUTPUT_FILENAME with this pre-rendered reference file
# both files are converted to PNG and compared with ImageMagick's 'compare'
# for multipage output, use OUTPUT_PAGE to specify a single page for comparison
+# FUZZYREF_FILENAME - comparison of OUTPUT_FILENAME with this pre-rendered reference file will be
+# performed in the L2 metric, subject to the specified FUZZ_PERCENTAGE
# EXPECTED_FILES - verify the command produced the expected files (i.e. they exist on disk)
# TEST_SCRIPT - additional script to run after performing all checks and before cleaning up
#
@@ -26,7 +30,8 @@
# ENVIRONMENT - Additional environment variables to set while running the test
function(add_cli_test name)
# parse arguments
- set(oneValueArgs INPUT_FILENAME OUTPUT_FILENAME OUTPUT_PAGE PASS_FOR_OUTPUT FAIL_FOR_OUTPUT REFERENCE_FILENAME)
+ set(oneValueArgs INPUT_FILENAME OUTPUT_FILENAME OUTPUT_PAGE PASS_FOR_OUTPUT FAIL_FOR_OUTPUT REFERENCE_FILENAME
+ FUZZYREF_FILENAME FUZZ_PERCENTAGE RASTER_DPI)
set(multiValueArgs PARAMETERS EXPECTED_FILES TEST_SCRIPT ENVIRONMENT)
cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
@@ -85,8 +90,23 @@
set_tests_properties(${testname}_check_output PROPERTIES
ENVIRONMENT "${CMAKE_CTEST_ENV}" DEPENDS ${testname} SKIP_RETURN_CODE 42)
endif()
-endfunction(add_cli_test)
+ # add a fuzzy test to check the output files
+ if(DEFINED ARG_FUZZYREF_FILENAME)
+ file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/testcases/${ARG_FUZZYREF_FILENAME}" ARG_FUZZYREF_FILENAME)
+ if(DEFINED ARG_FUZZ_PERCENTAGE)
+ set(ARG_FUZZ "${ARG_FUZZ_PERCENTAGE}")
+ else()
+ set(ARG_FUZZ "0")
+ endif()
+ add_test(NAME ${testname}_check_output
+ COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/l2compare.sh
+ "${ARG_OUTPUT_FILENAME}" "${ARG_OUTPUT_PAGE}" "${ARG_FUZZYREF_FILENAME}" "${ARG_FUZZ}" "${ARG_RASTER_DPI}")
+ set_tests_properties(${testname}_check_output PROPERTIES
+ ENVIRONMENT "${CMAKE_CTEST_ENV}" DEPENDS ${testname} SKIP_RETURN_CODE 42)
+ endif()
+
+endfunction(add_cli_test)
##### Tests follow below #####
--- /dev/null
+++ b/testfiles/cli_tests/l2compare.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Convert an image (or a single page of a PDF/PostScript document) to a bitmap
+# and calculate the relative root-mean-squared (L2) distance from the reference.
+#
+# Authors:
+# Rafael Siejakowski <rs@rs-math.net>
+#
+# Copyright (C) 2022 Authors
+#
+# Released under GNU GPL v2+, read the file 'COPYING' for more information.
+#
+
+ensure_command()
+{
+ command -v $1 >/dev/null 2>&1 || { echo >&2 "Required command '$1' not found. Aborting."; exit 1; }
+}
+
+ensure_command "convert"
+ensure_command "compare"
+ensure_command "bc"
+ensure_command "cp"
+
+OUTPUT_FILENAME="$1"
+OUTPUT_PAGE="$2"
+REFERENCE_FILENAME="$3"
+PERCENTAGE_DIFFERENCE_ALLOWED="$4"
+DPI="$5"
+
+if [ ! -f "${OUTPUT_FILENAME}" ]
+then
+ echo "Error: Test file '${OUTPUT_FILENAME}' not found."
+ exit 1
+fi
+
+if [ ! -f "${REFERENCE_FILENAME}" ]
+then
+ echo "Error: Reference file '${REFERENCE_FILENAME}' not found."
+ exit 1
+fi
+
+# Convert the output file to the PNG format
+CONVERSION_OPTIONS="-colorspace RGB"
+
+# Extract a page from multipage PS/PDF if requested
+OUTFILE_SUFFIX=""
+if [[ "x$OUTPUT_PAGE" != "x" ]]
+then
+ OUTFILE_SUFFIX="[${OUTPUT_PAGE}]" # Use ImageMagick's bracket operator
+fi
+
+DPI_OPTION=""
+if [[ "x$DPI" != "x" ]]
+then
+ DPI_OPTION="-density $DPI"
+fi
+
+if ! convert $DPI_OPTION "${OUTPUT_FILENAME}${OUTFILE_SUFFIX}" $CONVERSION_OPTIONS "${OUTPUT_FILENAME}-output.png"
+then
+ echo "Warning: Failed to convert test file '${OUTPUT_FILENAME}' to PNG format. Skipping comparison test."
+ exit 42
+fi
+
+# Copy the reference file
+cp "${REFERENCE_FILENAME}" "${OUTPUT_FILENAME}-reference.png"
+
+# Compare the two files
+COMPARE_RESULT=$(compare 2>&1 -metric RMSE "${OUTPUT_FILENAME}-output.png" "${OUTPUT_FILENAME}-reference.png" \
+ "${OUTPUT_FILENAME}-diff.png")
+COMPARE_RESULT=${COMPARE_RESULT#*(}
+RELATIVE_ERROR=${COMPARE_RESULT%)*}
+if [[ "x$RELATIVE_ERROR" == "x" ]]
+then
+ echo "Warning: Could not parse out the relative RMS error for fuzzy comparison. Skipping comparison test."
+ exit 42
+fi
+
+# Check if the difference between the files is within tolerance
+CONDITION="$RELATIVE_ERROR * 100 <= $PERCENTAGE_DIFFERENCE_ALLOWED"
+WITHIN_TOLERANCE=$(echo "${CONDITION}" | bc)
+if [[ $? -ne 0 ]]
+then
+ echo "Warning: An error occurred running 'bc'. The fuzzy comparison test will be skipped."
+ exit 42
+fi
+
+PERCENTAGE_ERROR=$(echo "$RELATIVE_ERROR * 100" | bc)
+if (( $WITHIN_TOLERANCE ))
+then
+ # Test passed: print stats and clean up the files.
+ echo "Fuzzy comparison PASSED; error of ${PERCENTAGE_ERROR}% is within ${PERCENTAGE_DIFFERENCE_ALLOWED}% tolerance."
+ for FILE in ${OUTPUT_FILENAME}{,-reference.png,-output.png,-diff.png}
+ do
+ rm -f "${FILE}"
+ done
+else
+ # Test failed!
+ echo "Fuzzy comparison FAILED; error of ${PERCENTAGE_ERROR}% exceeds ${PERCENTAGE_DIFFERENCE_ALLOWED}% tolerance."
+ exit 1
+fi
+
|
