#!/bin/sh # Copyright (C) 2020-2024 Internet Systems Consortium, Inc. ("ISC") # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Exit with error if commands exit with non-zero and if undefined variables are # used. set -eu ############################### Public functions ############################### # Add an entry to the XML test report. report_test_result_in_xml() { # If GTEST_OUTPUT is not defined... if ! test -n "${GTEST_OUTPUT+x}"; then # There is nowhere to report. return fi # Declarations local test_name="${1}"; shift local exit_code="${1}"; shift local duration="${1}"; shift # milliseconds local now local test_case local test_suite local xml now=$(date '+%FT%H:%M:%S') test_suite=$(printf '%s' "${test_name}" | cut -d '.' -f 1) test_case=$(printf '%s' "${test_name}" | cut -d '.' -f 2-) # Strip the 'xml:' at the start of GTEST_OUTPUT if it is there. xml="${GTEST_OUTPUT}" if test "$(printf '%s' "${xml}" | cut -c 1-4)" = 'xml:'; then xml=$(printf '%s' "${xml}" | cut -c 5-) fi xml="${xml}/${test_suite}.sh.xml" # Convert to seconds, but keep the millisecond precision. duration=$(_calculate "${duration} / 1000.0") # For test suites that have a single test case and no name for the test # case, name the test case after the test suite. if test -z "${test_case}"; then test_case="${test_suite}" fi # Determine result based on exit code. Googletest seems to omit the failed # tests, instead we are explicitly adding them with a 'failed' result. local result if test "${exit_code}" -eq 0; then result='success' else result='failed' fi _create_xml "${xml}" "${now}" _add_test_suite "${test_suite}" "${xml}" "${now}" _add_test_case "${test_suite}" "${test_case}" "${result}" "${duration}" \ "${xml}" "${now}" } ############################## Private functions ############################### # Add ${string} after ${reference} in ${file}. _add_after() { local string="${1}"; shift local reference="${1}"; shift local file="${1}"; shift # Escape all slashes. string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g') reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g') # Escape all spaces. Only trailing spaces need escaped, but that's harder # and this still empirically works. string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g') reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g') # Linearize. To avoid this change, add one line at a time. string=$(printf '%s' "${string}" | tr '\n' ' ') # Add ${string} after ${reference} in ${file}. # The "\\" followed by newline is for BSD support. sed "/${reference}/a\\ ${string} " "${file}" > "${file}.tmp" mv "${file}.tmp" "${file}" } # Add ${string} before ${reference} in ${file}. _add_before() { local string="${1}"; shift local reference="${1}"; shift local file="${1}"; shift # Get the line number of the reference line. local line_number line_number=$(grep -Fn "${reference}" "${file}" | cut -d ':' -f 1) # Escape all slashes. string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g') reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g') # Escape all spaces. Only trailing spaces need escaped, but that's harder # and this still empirically works. string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g') reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g') # Linearize. To avoid this change, add one line at a time. string=$(printf '%s' "${string}" | tr '\n' ' ') # Add ${string} before ${reference} in ${file}. # The "\\" followed by newline is for BSD support. sed "${line_number}i\\ ${string} " "${file}" > "${file}.tmp" mv "${file}.tmp" "${file}" } _add_failure_tag() { local test_case_tag="${1}"; shift local xml="${1}"; shift local closing_tag=' ' local failure_tag local failure_text local linearized_failure_text # Remove characters which are suspected to not be allowed in: # * sed # * XML attribute values # * XML CDATA failure_text=$(printf '%s\n%s' "${ERROR-}" "${OUTPUT-}" | \ sed 's/"/ /g' | sed 's/\[/ /g' | sed 's/\]/ /g') linearized_failure_text=$(printf '%s' "${failure_text}" | tr '\n' ' ') failure_tag=$(printf ' ' \ "${linearized_failure_text}" "${failure_text}") # Add. _add_after "${closing_tag}" "${test_case_tag}" "${xml}" _add_after "${failure_tag}" "${test_case_tag}" "${xml}" } # Add test result if not in file. _add_test_case() { local test_suite="${1}"; shift local test_case="${1}"; shift local result="${1}"; shift local duration="${1}"; shift local xml="${1}"; shift local now="${1}"; shift # Determine the test case tag. local closing_backslash local closing_tag if test "${result}" = 'success'; then closing_backslash=' /' else closing_backslash= fi # Create the test XML tag. local test_case_line test_case_line=$(printf ' ' \ "${test_case}" "${duration}" "${now}" "${test_suite}" \ "${closing_backslash}") # Add this test case to all the other test cases. local all_test_cases all_test_cases=$(_print_lines_between_matching_patterns \ " ' "${xml}") all_test_cases=$(printf '%s\n%s' "${all_test_cases}" "${test_case_line}") # Find the test following this one. local following_line following_line=$(printf '%s' "${all_test_cases}" | \ grep -A1 -F "${test_case_line}" | \ grep -Fv "${test_case_line}" || true) if test -n "${following_line}"; then # If found, add it before. _add_before "${test_case_line}" "${following_line}" "${xml}" else # Find the test before this one. local previous_line previous_line=$(printf '%s' "${all_test_cases}" | \ grep -B1 -F "${test_case_line}" | \ grep -Fv "${test_case_line}" || true) if test -n "${previous_line}"; then # If found, add it after. _add_after "${test_case_line}" "${previous_line}" "${xml}" else # If neither were found, add it as the first test case following the test # suite line. _add_after "${test_case_line}" " ' "${xml}") # Update attributes for the parent and the global . _update_test_suite_metrics "${test_suite}" "${all_test_cases}" "${xml}" "${now}" } # Add a set of test suite tags if not already present in the XML. _add_test_suite() { local test_suite="${1}"; shift local xml="${1}"; shift local now="${1}"; shift local test_suite_line local all_test_suites # If test suite tag is already there, then there is nothing to do. if grep -F " /dev/null 2>&1; then return fi # Create the test suite XML tag. local test_suite_line test_suite_line=$(printf ' ' \ "${test_suite}" "${now}") # Add this test suite to all the other test suites and sort them. local all_test_suites all_test_suites=$(printf '%s\n%s' " ${test_suite_line}" \ "$(grep -E ' ' "${xml}")") # Find the test suite following this one. local following_line following_line=$(printf '%s' "${all_test_suites}" | \ grep -A1 -F "${test_suite_line}" | \ grep -Fv "${test_suite_line}" || true) # Add the test suite tag to the XML. _add_before "${test_suite_line}" "${following_line}" "${xml}" _add_after ' ' "${test_suite_line}" "${xml}" } # Calculate the given mathematical expression and print it in a format that # matches googletest's time in the XML attribute time="..." which is seconds # rounded to 3 decimals. _calculate() { awk "BEGIN{print ${*}}"; } # Create XML with header and top-level tags if the file doesn't exist. _create_xml() { # If file exists and we have set GTEST_OUTPUT_CREATED previously, then there # is nothing to do. if test -f "${xml}" && test -n "${GTEST_OUTPUT_CREATED+x}"; then return; fi local xml="${1}"; shift local now="${1}"; shift mkdir -p "$(dirname "${xml}")" printf \ ' ' "${now}" > "${xml}" # GTEST_OUTPUT_CREATED is not a googletest variable, but our way of allowing # to overwrite XMLs created in a previous test run. The lifetime of # GTEST_OUTPUT_CREATED is extended to the oldest ancestor file who has # sourced this script i.e. the *_test.sh file. So it gets lost from one # *_test.sh to another. The consensus that need to be kept so that this # works correctly are: # * Needless to say, don't set this variable on your own. # * Always call these scripts directly or through `make check`. # Never source test files e.g. `source memfile_tests.sh` or # `. memfile_tests.sh`. # * The ${xml} passed here must be deterministically and uniquely # attributed to the *_test.sh. At the time of this writing, ${xml} is the # part of the name before the dot. So for example, for memfile, all tests # should start with the same thing e.g. `memfile.*`. export GTEST_OUTPUT_CREATED=true } # Print the lines between two matching regex patterns from a file. Excludes the # lines that contain the patterns themselves. Matches only the first occurrence. _print_lines_between_matching_patterns() { local start_pattern="${1}"; shift local end_pattern="${1}"; shift local file="${1}"; shift # Escape all slashes. start_pattern=$(printf '%s' "${start_pattern}" | sed 's#\/#\\\/#g') end_pattern=$(printf '%s' "${end_pattern}" | sed 's#\/#\\\/#g') # Print with sed. sed -n "/${start_pattern}/,/${end_pattern}/p;/${end_pattern}/q" "${file}" \ | sed '$d' | tail -n +2 } # Update the test suite XML attributes with metrics collected from the child # test cases. _update_test_suite_metrics() { local test_suite="${1}"; shift local all_test_cases="${1}"; shift local xml="${1}"; shift local now="${1}"; shift # Get the metrics on the parent test suite. local duration local durations_summed local failures local tests tests=$(printf '%s' "${all_test_cases}" | \ grep -Fc '' \ "${test_suite}" "${tests}" "${failures}" "${duration}" "${now}") # Update the test suite with the collected metrics. sed "s# #${test_suite_line}#g" \ "${xml}" > "${xml}.tmp" mv "${xml}.tmp" "${xml}" # Create the test suites XML tag. local test_suites_line test_suites_line=$(printf '' \ "${tests}" "${failures}" "${duration}" "${now}") # Update the test suites with the collected metrics. sed "s##${test_suites_line}#g" \ "${xml}" > "${xml}.tmp" mv "${xml}.tmp" "${xml}" }