summaryrefslogtreecommitdiffstats
path: root/tests/benchmarks/benchmark.sh
diff options
context:
space:
mode:
Diffstat (limited to '')
-rwxr-xr-xtests/benchmarks/benchmark.sh298
1 files changed, 298 insertions, 0 deletions
diff --git a/tests/benchmarks/benchmark.sh b/tests/benchmarks/benchmark.sh
new file mode 100755
index 0000000..4a89807
--- /dev/null
+++ b/tests/benchmarks/benchmark.sh
@@ -0,0 +1,298 @@
+#!/bin/bash
+
+set -eo pipefail
+
+#
+# parse the command line
+#
+
+usage() { echo "usage: $(basename "$0") [--cli <path>] [--baseline-cli <path>] [--suite <suite>] [--json <path>] [--zip <path>] [--verbose] [--debug]"; }
+
+TEST_CLI="git"
+BASELINE_CLI=
+SUITE=
+JSON_RESULT=
+ZIP_RESULT=
+OUTPUT_DIR=
+VERBOSE=
+DEBUG=
+NEXT=
+
+for a in "$@"; do
+ if [ "${NEXT}" = "cli" ]; then
+ TEST_CLI="${a}"
+ NEXT=
+ elif [ "${NEXT}" = "baseline-cli" ]; then
+ BASELINE_CLI="${a}"
+ NEXT=
+ elif [ "${NEXT}" = "suite" ]; then
+ SUITE="${a}"
+ NEXT=
+ elif [ "${NEXT}" = "json" ]; then
+ JSON_RESULT="${a}"
+ NEXT=
+ elif [ "${NEXT}" = "zip" ]; then
+ ZIP_RESULT="${a}"
+ NEXT=
+ elif [ "${NEXT}" = "output-dir" ]; then
+ OUTPUT_DIR="${a}"
+ NEXT=
+ elif [ "${a}" = "c" ] || [ "${a}" = "--cli" ]; then
+ NEXT="cli"
+ elif [[ "${a}" == "-c"* ]]; then
+ TEST_CLI="${a/-c/}"
+ elif [ "${a}" = "b" ] || [ "${a}" = "--baseline-cli" ]; then
+ NEXT="baseline-cli"
+ elif [[ "${a}" == "-b"* ]]; then
+ BASELINE_CLI="${a/-b/}"
+ elif [ "${a}" = "-s" ] || [ "${a}" = "--suite" ]; then
+ NEXT="suite"
+ elif [[ "${a}" == "-s"* ]]; then
+ SUITE="${a/-s/}"
+ elif [ "${a}" = "-v" ] || [ "${a}" == "--verbose" ]; then
+ VERBOSE=1
+ elif [ "${a}" == "--debug" ]; then
+ VERBOSE=1
+ DEBUG=1
+ elif [ "${a}" = "-j" ] || [ "${a}" == "--json" ]; then
+ NEXT="json"
+ elif [[ "${a}" == "-j"* ]]; then
+ JSON_RESULT="${a/-j/}"
+ elif [ "${a}" = "-z" ] || [ "${a}" == "--zip" ]; then
+ NEXT="zip"
+ elif [[ "${a}" == "-z"* ]]; then
+ ZIP_RESULT="${a/-z/}"
+ elif [ "${a}" = "--output-dir" ]; then
+ NEXT="output-dir"
+ else
+ echo "$(basename "$0"): unknown option: ${a}" 1>&2
+ usage 1>&2
+ exit 1
+ fi
+done
+
+if [ "${NEXT}" != "" ]; then
+ usage 1>&2
+ exit 1
+fi
+
+if [ "${OUTPUT_DIR}" = "" ]; then
+ OUTPUT_DIR=${OUTPUT_DIR:="$(mktemp -d)"}
+ CLEANUP_DIR=1
+fi
+
+#
+# collect some information about the test environment
+#
+
+SYSTEM_OS=$(uname -s)
+if [ "${SYSTEM_OS}" = "Darwin" ]; then SYSTEM_OS="macOS"; fi
+
+SYSTEM_KERNEL=$(uname -v)
+
+fullpath() {
+ if [[ "$(uname -s)" == "MINGW"* && $(cygpath -u "${TEST_CLI}") == "/"* ]]; then
+ echo "${TEST_CLI}"
+ elif [[ "${TEST_CLI}" == "/"* ]]; then
+ echo "${TEST_CLI}"
+ else
+ which "${TEST_CLI}"
+ fi
+}
+
+cli_version() {
+ if [[ "$(uname -s)" == "MINGW"* ]]; then
+ $(cygpath -u "$1") --version
+ else
+ "$1" --version
+ fi
+}
+
+TEST_CLI_NAME=$(basename "${TEST_CLI}")
+TEST_CLI_PATH=$(fullpath "${TEST_CLI}")
+TEST_CLI_VERSION=$(cli_version "${TEST_CLI}")
+
+if [ "${BASELINE_CLI}" != "" ]; then
+ if [[ "${BASELINE_CLI}" == "/"* ]]; then
+ BASELINE_CLI_PATH="${BASELINE_CLI}"
+ else
+ BASELINE_CLI_PATH=$(which "${BASELINE_CLI}")
+ fi
+
+ BASELINE_CLI_NAME=$(basename "${BASELINE_CLI}")
+ BASELINE_CLI_PATH=$(fullpath "${BASELINE_CLI}")
+ BASELINE_CLI_VERSION=$(cli_version "${BASELINE_CLI}")
+fi
+
+#
+# run the benchmarks
+#
+
+echo "##############################################################################"
+if [ "${SUITE}" != "" ]; then
+ SUITE_PREFIX="${SUITE/::/__}"
+ echo "## Running ${SUITE} benchmarks"
+else
+ echo "## Running all benchmarks"
+fi
+echo "##############################################################################"
+echo ""
+
+if [ "${BASELINE_CLI}" != "" ]; then
+ echo "# Baseline CLI: ${BASELINE_CLI} (${BASELINE_CLI_VERSION})"
+fi
+echo "# Test CLI: ${TEST_CLI} (${TEST_CLI_VERSION})"
+echo ""
+
+BENCHMARK_DIR=${BENCHMARK_DIR:=$(dirname "$0")}
+ANY_FOUND=
+ANY_FAILED=
+
+indent() { sed "s/^/ /"; }
+time_in_ms() { if [ "$(uname -s)" = "Darwin" ]; then date "+%s000"; else date "+%s%N" ; fi; }
+humanize_secs() {
+ units=('s' 'ms' 'us' 'ns')
+ unit=0
+ time="${1}"
+
+ if [ "${time}" = "" ]; then
+ echo ""
+ return
+ fi
+
+ # bash doesn't do floating point arithmetic. ick.
+ while [[ "${time}" == "0."* ]] && [ "$((unit+1))" != "${#units[*]}" ]; do
+ time="$(echo | awk "{ print ${time} * 1000 }")"
+ unit=$((unit+1))
+ done
+
+ echo "${time} ${units[$unit]}"
+}
+
+TIME_START=$(time_in_ms)
+
+for TEST_PATH in "${BENCHMARK_DIR}"/*; do
+ TEST_FILE=$(basename "${TEST_PATH}")
+
+ if [ ! -f "${TEST_PATH}" ] || [ ! -x "${TEST_PATH}" ]; then
+ continue
+ fi
+
+ if [[ "${TEST_FILE}" != *"__"* ]]; then
+ continue
+ fi
+
+ if [[ "${TEST_FILE}" != "${SUITE_PREFIX}"* ]]; then
+ continue
+ fi
+
+ ANY_FOUND=1
+ TEST_NAME="${TEST_FILE/__/::}"
+
+ echo -n "${TEST_NAME}:"
+ if [ "${VERBOSE}" = "1" ]; then
+ echo ""
+ else
+ echo -n " "
+ fi
+
+ if [ "${DEBUG}" = "1" ]; then
+ SHOW_OUTPUT="--show-output"
+ fi
+
+ OUTPUT_FILE="${OUTPUT_DIR}/${TEST_FILE}.out"
+ JSON_FILE="${OUTPUT_DIR}/${TEST_FILE}.json"
+ ERROR_FILE="${OUTPUT_DIR}/${TEST_FILE}.err"
+
+ FAILED=
+ ${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}" || FAILED=1
+
+ if [ "${FAILED}" = "1" ]; then
+ if [ "${VERBOSE}" != "1" ]; then
+ echo "failed!"
+ fi
+
+ indent < "${ERROR_FILE}"
+ ANY_FAILED=1
+ continue
+ fi
+
+ # in verbose mode, just print the hyperfine results; otherwise,
+ # pull the useful information out of its json and summarize it
+ if [ "${VERBOSE}" = "1" ]; then
+ indent < "${OUTPUT_FILE}"
+ else
+ jq -r '[ .results[0].mean, .results[0].stddev, .results[1].mean, .results[1].stddev ] | @tsv' < "${JSON_FILE}" | while IFS=$'\t' read -r one_mean one_stddev two_mean two_stddev; do
+ one_mean=$(humanize_secs "${one_mean}")
+ one_stddev=$(humanize_secs "${one_stddev}")
+
+ if [ "${two_mean}" != "" ]; then
+ two_mean=$(humanize_secs "${two_mean}")
+ two_stddev=$(humanize_secs "${two_stddev}")
+
+ echo "${one_mean} ± ${one_stddev} vs ${two_mean} ± ${two_stddev}"
+ else
+ echo "${one_mean} ± ${one_stddev}"
+ fi
+ done
+ fi
+
+ # add our metadata to the hyperfine json result
+ jq ". |= { \"name\": \"${TEST_NAME}\" } + ." < "${JSON_FILE}" > "${JSON_FILE}.new" && mv "${JSON_FILE}.new" "${JSON_FILE}"
+done
+
+TIME_END=$(time_in_ms)
+
+if [ "$ANY_FOUND" != "1" ]; then
+ echo ""
+ echo "error: no benchmark suite \"${SUITE}\"."
+ echo ""
+ exit 1
+fi
+
+escape() {
+ echo "${1//\\/\\\\}"
+}
+
+# combine all the individual benchmark results into a single json file
+if [ "${JSON_RESULT}" != "" ]; then
+ if [ "${VERBOSE}" = "1" ]; then
+ echo ""
+ echo "# Writing JSON results: ${JSON_RESULT}"
+ fi
+
+ SYSTEM_JSON="{ \"os\": \"${SYSTEM_OS}\", \"kernel\": \"${SYSTEM_KERNEL}\" }"
+ TIME_JSON="{ \"start\": ${TIME_START}, \"end\": ${TIME_END} }"
+ TEST_CLI_JSON="{ \"name\": \"${TEST_CLI_NAME}\", \"path\": \"$(escape "${TEST_CLI_PATH}")\", \"version\": \"${TEST_CLI_VERSION}\" }"
+ BASELINE_CLI_JSON="{ \"name\": \"${BASELINE_CLI_NAME}\", \"path\": \"$(escape "${BASELINE_CLI_PATH}")\", \"version\": \"${BASELINE_CLI_VERSION}\" }"
+
+ if [ "${BASELINE_CLI}" != "" ]; then
+ EXECUTOR_JSON="{ \"baseline\": ${BASELINE_CLI_JSON}, \"cli\": ${TEST_CLI_JSON} }"
+ else
+ EXECUTOR_JSON="{ \"cli\": ${TEST_CLI_JSON} }"
+ fi
+
+ # add our metadata to all the test results
+ jq -n "{ \"system\": ${SYSTEM_JSON}, \"time\": ${TIME_JSON}, \"executor\": ${EXECUTOR_JSON}, \"tests\": [inputs] }" "${OUTPUT_DIR}"/*.json > "${JSON_RESULT}"
+fi
+
+# combine all the data into a zip if requested
+if [ "${ZIP_RESULT}" != "" ]; then
+ if [ "${VERBOSE}" = "1" ]; then
+ if [ "${JSON_RESULT}" = "" ]; then echo ""; fi
+ echo "# Writing ZIP results: ${ZIP_RESULT}"
+ fi
+
+ zip -jr "${ZIP_RESULT}" "${OUTPUT_DIR}" >/dev/null
+fi
+
+if [ "$CLEANUP_DIR" = "1" ]; then
+ rm -f "${OUTPUT_DIR}"/*.out
+ rm -f "${OUTPUT_DIR}"/*.err
+ rm -f "${OUTPUT_DIR}"/*.json
+ rmdir "${OUTPUT_DIR}"
+fi
+
+if [ "$ANY_FAILED" = "1" ]; then
+ exit 1
+fi