diff options
Diffstat (limited to 'src/lib/testutils/xml_reporting_test_lib.sh.in')
-rw-r--r-- | src/lib/testutils/xml_reporting_test_lib.sh.in | 353 |
1 files changed, 353 insertions, 0 deletions
diff --git a/src/lib/testutils/xml_reporting_test_lib.sh.in b/src/lib/testutils/xml_reporting_test_lib.sh.in new file mode 100644 index 0000000..0d9b235 --- /dev/null +++ b/src/lib/testutils/xml_reporting_test_lib.sh.in @@ -0,0 +1,353 @@ +#!/bin/sh + +# Copyright (C) 2020-2021 Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# shellcheck disable=SC2039 +# SC2039: In POSIX sh, 'local' is undefined. + +# Exit with error if commands exit with non-zero and if undefined variables are +# used. +set -eu + +############################### Public functions ############################### + +# Add an entry to the XML test report. +report_test_result_in_xml() { + # If GTEST_OUTPUT is not defined... + if ! test -n "${GTEST_OUTPUT+x}"; then + # There is nowhere to report. + return + fi + + # Declarations + local test_name="${1}"; shift + local exit_code="${1}"; shift + local duration="${1}"; shift # milliseconds + local now + local test_case + local test_suite + local xml + now=$(date '+%FT%H:%M:%S') + test_suite=$(printf '%s' "${test_name}" | cut -d '.' -f 1) + test_case=$(printf '%s' "${test_name}" | cut -d '.' -f 2-) + + # Strip the 'xml:' at the start of GTEST_OUTPUT if it is there. + xml="${GTEST_OUTPUT}" + if test "$(printf '%s' "${xml}" | cut -c 1-4)" = 'xml:'; then + xml=$(printf '%s' "${xml}" | cut -c 5-) + fi + xml="${xml}/${test_suite}.sh.xml" + + # Convert to seconds, but keep the millisecond precision. + duration=$(_calculate "${duration} / 1000.0") + + # For test suites that have a single test case and no name for the test + # case, name the test case after the test suite. + if test -z "${test_case}"; then + test_case="${test_suite}" + fi + + # Determine result based on exit code. Googletest seems to omit the failed + # tests, instead we are explicitly adding them with a 'failed' result. + local result + if test "${exit_code}" -eq 0; then + result='success' + else + result='failed' + fi + + _create_xml "${xml}" "${now}" + + _add_test_suite "${test_suite}" "${xml}" "${now}" + + _add_test_case "${test_suite}" "${test_case}" "${result}" "${duration}" \ + "${xml}" "${now}" +} + +############################## Private functions ############################### + +# Add ${string} after ${reference} in ${file}. +_add_after() { + local string="${1}"; shift + local reference="${1}"; shift + local file="${1}"; shift + + # Escape all slashes. + string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g') + reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g') + + # Escape all spaces. Only trailing spaces need escaped, but that's harder + # and this still empirically works. + string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g') + reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g') + + # Linearize. To avoid this change, add one line at a time. + string=$(printf '%s' "${string}" | tr '\n' ' ') + + # Add ${string} after ${reference} in ${file}. + # The "\\" followed by newline is for BSD support. + sed "/${reference}/a\\ +${string} +" "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +} + +# Add ${string} before ${reference} in ${file}. +_add_before() { + local string="${1}"; shift + local reference="${1}"; shift + local file="${1}"; shift + + # Get the line number of the reference line. + local line_number + line_number=$(grep -Fn "${reference}" "${file}" | cut -d ':' -f 1) + + # Escape all slashes. + string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g') + reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g') + + # Escape all spaces. Only trailing spaces need escaped, but that's harder + # and this still empirically works. + string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g') + reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g') + + # Linearize. To avoid this change, add one line at a time. + string=$(printf '%s' "${string}" | tr '\n' ' ') + + # Add ${string} before ${reference} in ${file}. + # The "\\" followed by newline is for BSD support. + sed "${line_number}i\\ +${string} +" "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +} + +_add_failure_tag() { + local test_case_tag="${1}"; shift + local xml="${1}"; shift + + local closing_tag=' </testcase>' + local failure_tag + local failure_text + local linearized_failure_text + # Remove characters which are suspected to not be allowed in: + # * sed + # * XML attribute values + # * XML CDATA + failure_text=$(printf '%s\n%s' "${ERROR}" "${OUTPUT}" | \ + sed 's/"/ /g' | sed 's/\[/ /g' | sed 's/\]/ /g') + linearized_failure_text=$(printf '%s' "${failure_text}" | tr '\n' ' ') + failure_tag=$(printf ' <failure message="%s" type=""><![CDATA[%s]]></failure>' \ + "${linearized_failure_text}" "${failure_text}") + + # Add. + _add_after "${closing_tag}" "${test_case_tag}" "${xml}" + _add_after "${failure_tag}" "${test_case_tag}" "${xml}" +} + +# Add test result if not in file. +_add_test_case() { + local test_suite="${1}"; shift + local test_case="${1}"; shift + local result="${1}"; shift + local duration="${1}"; shift + local xml="${1}"; shift + local now="${1}"; shift + + # Determine the test case tag. + local closing_backslash + local closing_tag + if test "${result}" = 'success'; then + closing_backslash=' /' + else + closing_backslash= + fi + + # Create the test XML tag. + local test_case_line + test_case_line=$(printf ' <testcase name="%s" status="run" result="completed" time="%s" timestamp="%s" classname="%s"%s>' \ + "${test_case}" "${duration}" "${now}" "${test_suite}" \ + "${closing_backslash}") + + # Add this test case to all the other test cases. + local all_test_cases + all_test_cases=$(_print_lines_between_matching_patterns \ + " <testsuite name=\"${test_suite}\"" ' </testsuite>' "${xml}") + all_test_cases=$(printf '%s\n%s' "${all_test_cases}" "${test_case_line}") + + # Find the test following this one. + local following_line + following_line=$(printf '%s' "${all_test_cases}" | \ + grep -A1 -F "${test_case_line}" | \ + grep -Fv "${test_case_line}" || true) + if test -n "${following_line}"; then + # If found, add it before. + _add_before "${test_case_line}" "${following_line}" "${xml}" + else + # Find the test before this one. + local previous_line + previous_line=$(printf '%s' "${all_test_cases}" | \ + grep -B1 -F "${test_case_line}" | \ + grep -Fv "${test_case_line}" || true) + if test -n "${previous_line}"; then + # If found, add it after. + _add_after "${test_case_line}" "${previous_line}" "${xml}" + else + # If neither were found, add it as the first test case following the test + # suite line. + _add_after "${test_case_line}" " <testsuite name=\"${test_suite}\"" "${xml}" + fi + fi + + # Add the failure tag if it is the case. + if test "${result}" != 'success'; then + _add_failure_tag "${test_case_line}" "${xml}" + fi + + # Retrieve again to include the failure tag that may have just been added + # among other tags or lines. + all_test_cases=$(_print_lines_between_matching_patterns \ + " <testsuite name=\"${test_suite}\"" ' </testsuite>' "${xml}") + + # Update attributes for the parent <testsuite> and the global <testsuites>. + _update_test_suite_metrics "${test_suite}" "${all_test_cases}" "${xml}" "${now}" +} + +# Add a set of test suite tags if not already present in the XML. +_add_test_suite() { + local test_suite="${1}"; shift + local xml="${1}"; shift + local now="${1}"; shift + local test_suite_line + local all_test_suites + + # If test suite tag is already there, then there is nothing to do. + if grep -F "<testsuite name=\"${test_suite}\"" "${xml}" \ + > /dev/null 2>&1; then + return + fi + + # Create the test suite XML tag. + local test_suite_line + test_suite_line=$(printf ' <testsuite name="%s" tests="0" failures="0" disabled="0" errors="0" time="0" timestamp="%s">' \ + "${test_suite}" "${now}") + + # Add this test suite to all the other test suites and sort them. + local all_test_suites + all_test_suites=$(printf '%s\n%s' " ${test_suite_line}" \ + "$(grep -E ' <testsuite name=|</testsuites>' "${xml}")") + + # Find the test suite following this one. + local following_line + following_line=$(printf '%s' "${all_test_suites}" | \ + grep -A1 -F "${test_suite_line}" | \ + grep -Fv "${test_suite_line}" || true) + + # Add the test suite tag to the XML. + _add_before "${test_suite_line}" "${following_line}" "${xml}" + _add_after ' </testsuite>' "${test_suite_line}" "${xml}" +} + +# Calculate the given mathematical expression and print it in a format that +# matches googletest's time in the XML attribute time="..." which is seconds +# rounded to 3 decimals. +_calculate() { + awk "BEGIN{print ${*}}"; +} + +# Create XML with header and top-level tags if the file doesn't exist. +_create_xml() { + # If file exists and we have set GTEST_OUTPUT_CREATED previously, then there + # is nothing to do. + if test -f "${xml}" && test -n "${GTEST_OUTPUT_CREATED+x}"; then + return; + fi + + local xml="${1}"; shift + local now="${1}"; shift + + mkdir -p "$(dirname "${xml}")" + printf \ +'<?xml version="1.0" encoding="UTF-8"?> +<testsuites tests="0" failures="0" disabled="0" errors="0" time="0" timestamp="%s" name="AllTests"> +</testsuites> +' "${now}" > "${xml}" + + # GTEST_OUTPUT_CREATED is not a googletest variable, but our way of allowing + # to overwrite XMLs created in a previous test run. The lifetime of + # GTEST_OUTPUT_CREATED is extended to the oldest ancestor file who has + # sourced this script i.e. the *_test.sh file. So it gets lost from one + # *_test.sh to another. The consensus that need to be kept so that this + # works correctly are: + # * Needless to say, don't set this variable on your own. + # * Always call these scripts directly or through `make check`. + # Never source test files e.g. `source memfile_tests.sh` or + # `. memfile_tests.sh`. + # * The ${xml} passed here must be deterministically and uniquely + # attributed to the *_test.sh. At the time of this writing, ${xml} is the + # part of the name before the dot. So for example, for memfile, all tests + # should start with the same thing e.g. `memfile.*`. + export GTEST_OUTPUT_CREATED=true +} + +# Print the lines between two matching regex patterns from a file. Excludes the +# lines that contain the patterns themselves. Matches only the first occurrence. +_print_lines_between_matching_patterns() { + local start_pattern="${1}"; shift + local end_pattern="${1}"; shift + local file="${1}"; shift + + # Escape all slashes. + start_pattern=$(printf '%s' "${start_pattern}" | sed 's#\/#\\\/#g') + end_pattern=$(printf '%s' "${end_pattern}" | sed 's#\/#\\\/#g') + + # Print with sed. + sed -n "/${start_pattern}/,/${end_pattern}/p;/${end_pattern}/q" "${file}" \ + | sed '$d' | tail -n +2 +} + +# Update the test suite XML attributes with metrics collected from the child +# test cases. +_update_test_suite_metrics() { + local test_suite="${1}"; shift + local all_test_cases="${1}"; shift + local xml="${1}"; shift + local now="${1}"; shift + + # Get the metrics on the parent test suite. + local duration + local durations_summed + local failures + local tests + tests=$(printf '%s' "${all_test_cases}" | \ + grep -Fc '<testcase' || true) + failures=$(printf '%s' "${all_test_cases}" | \ + grep -Fc '<failure' || true) + durations_summed=$(printf '%s' "${all_test_cases}" | \ + grep -Eo 'time="[0-9.]+"' | cut -d '"' -f 2 | xargs | sed 's/ / + /g') + duration=$(_calculate "${durations_summed}") + + # Create the test suite XML tag. + local test_suite_line + test_suite_line=$(printf ' <testsuite name="%s" tests="%s" failures="%s" disabled="0" errors="0" time="%s" timestamp="%s">' \ + "${test_suite}" "${tests}" "${failures}" "${duration}" "${now}") + + # Update the test suite with the collected metrics. + sed "s# <testsuite name=\"${test_suite}\".*>#${test_suite_line}#g" \ + "${xml}" > "${xml}.tmp" + mv "${xml}.tmp" "${xml}" + + # Create the test suites XML tag. + local test_suites_line + test_suites_line=$(printf '<testsuites tests="%s" failures="%s" disabled="0" errors="0" time="%s" timestamp="%s" name="AllTests">' \ + "${tests}" "${failures}" "${duration}" "${now}") + + # Update the test suites with the collected metrics. + sed "s#<testsuites .*>#${test_suites_line}#g" \ + "${xml}" > "${xml}.tmp" + mv "${xml}.tmp" "${xml}" +} |