path: root/src/lib/testutils/xml_reporting_test_lib.sh.in

#!/bin/sh

# Copyright (C) 2020-2021 Internet Systems Consortium, Inc. ("ISC")
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# shellcheck disable=SC2039
# SC2039: In POSIX sh, 'local' is undefined.

# Exit with error if commands exit with non-zero and if undefined variables are
# used.
set -eu

############################### Public functions ###############################

# Add an entry to the XML test report.
report_test_result_in_xml() {
    # If GTEST_OUTPUT is not defined...
    if ! test -n "${GTEST_OUTPUT+x}"; then
        # There is nowhere to report.
        return
    fi

    # Declarations
    local test_name="${1}"; shift
    local exit_code="${1}"; shift
    local duration="${1}"; shift # milliseconds
    local now
    local test_case
    local test_suite
    local xml
    now=$(date '+%FT%H:%M:%S')
    test_suite=$(printf '%s' "${test_name}" | cut -d '.' -f 1)
    test_case=$(printf '%s' "${test_name}" | cut -d '.' -f 2-)

    # Strip the 'xml:' at the start of GTEST_OUTPUT if it is there.
    xml="${GTEST_OUTPUT}"
    if test "$(printf '%s' "${xml}" | cut -c 1-4)" = 'xml:'; then
        xml=$(printf '%s' "${xml}" | cut -c 5-)
    fi
    xml="${xml}/${test_suite}.sh.xml"

    # Convert to seconds, but keep the millisecond precision.
    duration=$(_calculate "${duration} / 1000.0")

    # For test suites that have a single test case and no name for the test
    # case, name the test case after the test suite.
    if test -z "${test_case}"; then
        test_case="${test_suite}"
    fi

    # Determine result based on exit code. Googletest seems to omit the failed
    # tests, instead we are explicitly adding them with a 'failed' result.
    local result
    if test "${exit_code}" -eq 0; then
        result='success'
    else
        result='failed'
    fi

    _create_xml "${xml}" "${now}"

    _add_test_suite "${test_suite}" "${xml}" "${now}"

    _add_test_case "${test_suite}" "${test_case}" "${result}" "${duration}" \
        "${xml}" "${now}"
}

############################## Private functions ###############################

# Add ${string} after ${reference} in ${file}.
_add_after() {
    local string="${1}"; shift
    local reference="${1}"; shift
    local file="${1}"; shift

    # Escape all slashes.
    string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g')
    reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g')

    # Escape all spaces. Only trailing spaces need escaped, but that's harder
    # and this still empirically works.
    string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g')
    reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g')

    # Linearize. To avoid this change, add one line at a time.
    string=$(printf '%s' "${string}" | tr '\n' ' ')

    # Add ${string} after ${reference} in ${file}.
    # The "\\" followed by newline is for BSD support.
    sed "/${reference}/a\\
${string}
" "${file}" > "${file}.tmp"
    mv "${file}.tmp" "${file}"
}

# Add ${string} before ${reference} in ${file}.
_add_before() {
    local string="${1}"; shift
    local reference="${1}"; shift
    local file="${1}"; shift

    # Get the line number of the reference line.
    local line_number
    line_number=$(grep -Fn "${reference}" "${file}" | cut -d ':' -f 1)

    # Escape all slashes.
    string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g')
    reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g')

    # Escape all spaces. Only trailing spaces need escaped, but that's harder
    # and this still empirically works.
    string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g')
    reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g')

    # Linearize. To avoid this change, add one line at a time.
    string=$(printf '%s' "${string}" | tr '\n' ' ')

    # Add ${string} before ${reference} in ${file}.
    # The "\\" followed by newline is for BSD support.
    sed "${line_number}i\\
${string}
" "${file}" > "${file}.tmp"
    mv "${file}.tmp" "${file}"
}

_add_failure_tag() {
    local test_case_tag="${1}"; shift
    local xml="${1}"; shift

    local closing_tag='    </testcase>'
    local failure_tag
    local failure_text
    local linearized_failure_text
    # Remove characters which are suspected to not be allowed in:
    # * sed
    # * XML attribute values
    # * XML CDATA
    failure_text=$(printf '%s\n%s' "${ERROR}" "${OUTPUT}" | \
        sed 's/"/ /g' | sed 's/\[/ /g' | sed 's/\]/ /g')
    linearized_failure_text=$(printf '%s' "${failure_text}" | tr '\n' ' ')
    failure_tag=$(printf '      <failure message="%s" type=""><![CDATA[%s]]></failure>' \
        "${linearized_failure_text}" "${failure_text}")

    # Add.
    _add_after "${closing_tag}" "${test_case_tag}" "${xml}"
    _add_after "${failure_tag}" "${test_case_tag}" "${xml}"
}

# Add test result if not in file.
_add_test_case() {
    local test_suite="${1}"; shift
    local test_case="${1}"; shift
    local result="${1}"; shift
    local duration="${1}"; shift
    local xml="${1}"; shift
    local now="${1}"; shift

    # Determine the test case tag.
    local closing_backslash
    local closing_tag
    if test "${result}" = 'success'; then
        closing_backslash=' /'
    else
        closing_backslash=
    fi

    # Create the test XML tag.
    local test_case_line
    test_case_line=$(printf '    <testcase name="%s" status="run" result="completed" time="%s" timestamp="%s" classname="%s"%s>' \
        "${test_case}" "${duration}" "${now}" "${test_suite}" \
        "${closing_backslash}")

    # Add this test case to all the other test cases.
    local all_test_cases
    all_test_cases=$(_print_lines_between_matching_patterns \
        "  <testsuite name=\"${test_suite}\"" '  </testsuite>' "${xml}")
    all_test_cases=$(printf '%s\n%s' "${all_test_cases}" "${test_case_line}")

    # Find the test following this one.
    local following_line
    following_line=$(printf '%s' "${all_test_cases}" | \
        grep -A1 -F "${test_case_line}" | \
        grep -Fv "${test_case_line}" || true)
    if test -n "${following_line}"; then
        # If found, add it before.
        _add_before "${test_case_line}" "${following_line}" "${xml}"
    else
        # Find the test before this one.
        local previous_line
        previous_line=$(printf '%s' "${all_test_cases}" | \
            grep -B1 -F "${test_case_line}" | \
            grep -Fv "${test_case_line}" || true)
        if test -n "${previous_line}"; then
            # If found, add it after.
            _add_after "${test_case_line}" "${previous_line}" "${xml}"
        else
            # If neither were found, add it as the first test case following the test
            # suite line.
            _add_after "${test_case_line}" "  <testsuite name=\"${test_suite}\"" "${xml}"
        fi
    fi

    # Add the failure tag if it is the case.
    if test "${result}" != 'success'; then
        _add_failure_tag "${test_case_line}" "${xml}"
    fi

    # Retrieve again to include the failure tag that may have just been added
    # among other tags or lines.
    all_test_cases=$(_print_lines_between_matching_patterns \
        "  <testsuite name=\"${test_suite}\"" '  </testsuite>' "${xml}")

    # Update attributes for the parent <testsuite> and the global <testsuites>.
    _update_test_suite_metrics "${test_suite}" "${all_test_cases}" "${xml}" "${now}"
}

# Add a set of test suite tags if not already present in the XML.
_add_test_suite() {
    local test_suite="${1}"; shift
    local xml="${1}"; shift
    local now="${1}"; shift
    local test_suite_line
    local all_test_suites

    # If test suite tag is already there, then there is nothing to do.
    if grep -F "<testsuite name=\"${test_suite}\"" "${xml}" \
        > /dev/null 2>&1; then
        return
    fi

    # Create the test suite XML tag.
    local test_suite_line
    test_suite_line=$(printf '  <testsuite name="%s" tests="0" failures="0" disabled="0" errors="0" time="0" timestamp="%s">' \
        "${test_suite}" "${now}")

    # Add this test suite to all the other test suites and sort them.
    local all_test_suites
    all_test_suites=$(printf '%s\n%s' "  ${test_suite_line}" \
        "$(grep -E '  <testsuite name=|</testsuites>' "${xml}")")

    # Find the test suite following this one.
    local following_line
    following_line=$(printf '%s' "${all_test_suites}" | \
        grep -A1 -F "${test_suite_line}" | \
        grep -Fv "${test_suite_line}" || true)

    # Add the test suite tag to the XML.
    _add_before "${test_suite_line}" "${following_line}" "${xml}"
    _add_after '  </testsuite>' "${test_suite_line}" "${xml}"
}

# Calculate the given mathematical expression and print it in a format that
# matches googletest's time in the XML attribute time="..." which is seconds
# rounded to 3 decimals.
_calculate() {
    awk "BEGIN{print ${*}}";
}

# Create XML with header and top-level tags if the file doesn't exist.
_create_xml() {
    # If file exists and we have set GTEST_OUTPUT_CREATED previously, then there
    # is nothing to do.
    if test -f "${xml}" && test -n "${GTEST_OUTPUT_CREATED+x}"; then
        return;
    fi

    local xml="${1}"; shift
    local now="${1}"; shift

    mkdir -p "$(dirname "${xml}")"
    printf \
'<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="0" failures="0" disabled="0" errors="0" time="0" timestamp="%s" name="AllTests">
</testsuites>
' "${now}" > "${xml}"

    # GTEST_OUTPUT_CREATED is not a googletest variable, but our way of allowing
    # to overwrite XMLs created in a previous test run. The lifetime of
    # GTEST_OUTPUT_CREATED is extended to the oldest ancestor file who has
    # sourced this script i.e. the *_test.sh file. So it gets lost from one
    # *_test.sh to another. The consensus that need to be kept so that this
    # works correctly are:
    #   * Needless to say, don't set this variable on your own.
    #   * Always call these scripts directly or through `make check`.
    # Never source test files e.g. `source memfile_tests.sh` or
    # `. memfile_tests.sh`.
    #   * The ${xml} passed here must be deterministically and uniquely
    # attributed to the *_test.sh. At the time of this writing, ${xml} is the
    # part of the name before the dot. So for example, for memfile, all tests
    # should start with the same thing e.g. `memfile.*`.
    export GTEST_OUTPUT_CREATED=true
}

# Print the lines between two matching regex patterns from a file. Excludes the
# lines that contain the patterns themselves. Matches only the first occurrence.
_print_lines_between_matching_patterns() {
    local start_pattern="${1}"; shift
    local end_pattern="${1}"; shift
    local file="${1}"; shift

    # Escape all slashes.
    start_pattern=$(printf '%s' "${start_pattern}" | sed 's#\/#\\\/#g')
    end_pattern=$(printf '%s' "${end_pattern}" | sed 's#\/#\\\/#g')

    # Print with sed.
    sed -n "/${start_pattern}/,/${end_pattern}/p;/${end_pattern}/q" "${file}" \
        | sed '$d' | tail -n +2
}

# Update the test suite XML attributes with metrics collected from the child
# test cases.
_update_test_suite_metrics() {
    local test_suite="${1}"; shift
    local all_test_cases="${1}"; shift
    local xml="${1}"; shift
    local now="${1}"; shift

    # Get the metrics on the parent test suite.
    local duration
    local durations_summed
    local failures
    local tests
    tests=$(printf '%s' "${all_test_cases}" | \
        grep -Fc '<testcase' || true)
    failures=$(printf '%s' "${all_test_cases}" | \
        grep -Fc '<failure' || true)
    durations_summed=$(printf '%s' "${all_test_cases}" | \
        grep -Eo 'time="[0-9.]+"' | cut -d '"' -f 2 | xargs | sed 's/ / + /g')
    duration=$(_calculate "${durations_summed}")

    # Create the test suite XML tag.
    local test_suite_line
    test_suite_line=$(printf '  <testsuite name="%s" tests="%s" failures="%s" disabled="0" errors="0" time="%s" timestamp="%s">' \
        "${test_suite}" "${tests}" "${failures}" "${duration}" "${now}")

    # Update the test suite with the collected metrics.
    sed "s#  <testsuite name=\"${test_suite}\".*>#${test_suite_line}#g" \
        "${xml}" > "${xml}.tmp"
    mv "${xml}.tmp" "${xml}"

    # Create the test suites XML tag.
    local test_suites_line
    test_suites_line=$(printf '<testsuites tests="%s" failures="%s" disabled="0" errors="0" time="%s" timestamp="%s" name="AllTests">' \
        "${tests}" "${failures}" "${duration}" "${now}")

    # Update the test suites with the collected metrics.
    sed "s#<testsuites .*>#${test_suites_line}#g" \
        "${xml}" > "${xml}.tmp"
    mv "${xml}.tmp" "${xml}"
}