summaryrefslogtreecommitdiffstats
path: root/third_party/jpeg-xl/ci.sh
diff options
context:
space:
mode:
Diffstat (limited to 'third_party/jpeg-xl/ci.sh')
-rwxr-xr-xthird_party/jpeg-xl/ci.sh75
1 files changed, 49 insertions, 26 deletions
diff --git a/third_party/jpeg-xl/ci.sh b/third_party/jpeg-xl/ci.sh
index f33efd9693..448a7f9927 100755
--- a/third_party/jpeg-xl/ci.sh
+++ b/third_party/jpeg-xl/ci.sh
@@ -16,6 +16,8 @@ MYDIR=$(dirname $(realpath "$0"))
### Environment parameters:
TEST_STACK_LIMIT="${TEST_STACK_LIMIT:-256}"
+BENCHMARK_NUM_THREADS="${BENCHMARK_NUM_THREADS:-0}"
+BUILD_CONFIG=${BUILD_CONFIG:-}
CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo}
CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-}
CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-}
@@ -79,6 +81,12 @@ if [[ "${ENABLE_WASM_SIMD}" -eq "2" ]]; then
CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -DHWY_WANT_WASM2"
fi
+if [[ -z "${BUILD_CONFIG}" ]]; then
+ TOOLS_DIR="${BUILD_DIR}/tools"
+else
+ TOOLS_DIR="${BUILD_DIR}/tools/${BUILD_CONFIG}"
+fi
+
if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then
CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}"
fi
@@ -128,17 +136,17 @@ if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then
)
fi
-CLANG_TIDY_BIN=$(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 clang-tidy | head -n 1)
+CLANG_TIDY_BIN=$(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 clang-tidy 2>/dev/null | head -n 1)
# Default to "cat" if "colordiff" is not installed or if stdout is not a tty.
if [[ -t 1 ]]; then
- COLORDIFF_BIN=$(which colordiff cat | head -n 1)
+ COLORDIFF_BIN=$(which colordiff cat 2>/dev/null | head -n 1)
else
COLORDIFF_BIN="cat"
fi
-FIND_BIN=$(which gfind find | head -n 1)
+FIND_BIN=$(which gfind find 2>/dev/null | head -n 1)
# "false" will disable wine64 when not installed. This won't allow
# cross-compiling.
-WINE_BIN=$(which wine64 false | head -n 1)
+WINE_BIN=$(which wine64 false 2>/dev/null | head -n 1)
CLANG_VERSION="${CLANG_VERSION:-}"
# Detect the clang version suffix and store it in CLANG_VERSION. For example,
@@ -411,7 +419,7 @@ cmake_build_and_test() {
if [[ "${PACK_TEST:-}" == "1" ]]; then
(cd "${BUILD_DIR}"
${FIND_BIN} -name '*.cmake' -a '!' -path '*CMakeFiles*'
- # gtest / gmock / gtest_main shared libs
+ # gtest / gtest_main shared libs
${FIND_BIN} lib/ -name 'libg*.so*'
${FIND_BIN} -type d -name tests -a '!' -path '*CMakeFiles*'
) | tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \
@@ -791,9 +799,13 @@ cmd_ossfuzz_ninja() {
cmd_fast_benchmark() {
local small_corpus_tar="${BENCHMARK_CORPORA}/jyrki-full.tar"
+ local small_corpus_url="https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar"
mkdir -p "${BENCHMARK_CORPORA}"
- curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" \
- "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar"
+ if [ -f "${small_corpus_tar}" ]; then
+ curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" "${small_corpus_url}"
+ else
+ curl --show-error -o "${small_corpus_tar}" "${small_corpus_url}"
+ fi
local tmpdir=$(mktemp -d)
CLEANUP_FILES+=("${tmpdir}")
@@ -831,7 +843,7 @@ cmd_benchmark() {
png_filename="${filename%.ppm}.png"
png_filename=$(echo "${png_filename}" | tr '/' '_')
sem --bg --id "${sem_id}" -j"${nprocs}" -- \
- "${BUILD_DIR}/tools/decode_and_encode" \
+ "${TOOLS_DIR}/decode_and_encode" \
"${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}"
images+=( "${png_filename}" )
done < <(cd "${tmpdir}"; ${FIND_BIN} . -name '*.ppm' -type f)
@@ -844,6 +856,8 @@ cmd_benchmark() {
get_mem_available() {
if [[ "${OS}" == "Darwin" ]]; then
echo $(vm_stat | grep -F 'Pages free:' | awk '{print $3 * 4}')
+ elif [[ "${OS}" == MINGW* ]]; then
+ echo $(vmstat | tail -n 1 | awk '{print $4 * 4}')
else
echo $(grep -F MemAvailable: /proc/meminfo | awk '{print $2}')
fi
@@ -856,15 +870,24 @@ run_benchmark() {
local output_dir="${BUILD_DIR}/benchmark_results"
mkdir -p "${output_dir}"
- # The memory available at the beginning of the benchmark run in kB. The number
- # of threads depends on the available memory, and the passed memory per
- # thread. We also add a 2 GiB of constant memory.
- local mem_available="$(get_mem_available)"
- # Check that we actually have a MemAvailable value.
- [[ -n "${mem_available}" ]]
- local num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} ))
- if [[ ${num_threads} -le 0 ]]; then
- num_threads=1
+ if [[ "${OS}" == MINGW* ]]; then
+ src_img_dir=`cygpath -w "${src_img_dir}"`
+ fi
+
+ local num_threads=1
+ if [[ ${BENCHMARK_NUM_THREADS} -gt 0 ]]; then
+ num_threads=${BENCHMARK_NUM_THREADS}
+ else
+ # The memory available at the beginning of the benchmark run in kB. The number
+ # of threads depends on the available memory, and the passed memory per
+ # thread. We also add a 2 GiB of constant memory.
+ local mem_available="$(get_mem_available)"
+ # Check that we actually have a MemAvailable value.
+ [[ -n "${mem_available}" ]]
+ num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} ))
+ if [[ ${num_threads} -le 0 ]]; then
+ num_threads=1
+ fi
fi
local benchmark_args=(
@@ -873,20 +896,20 @@ run_benchmark() {
--output_dir "${output_dir}"
--show_progress
--num_threads="${num_threads}"
+ --decode_reps=11
+ --encode_reps=11
)
if [[ "${STORE_IMAGES}" == "1" ]]; then
benchmark_args+=(--save_decompressed --save_compressed)
fi
(
[[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}"
- "${BUILD_DIR}/tools/benchmark_xl" "${benchmark_args[@]}" | \
+ "${TOOLS_DIR}/benchmark_xl" "${benchmark_args[@]}" | \
tee "${output_dir}/results.txt"
# Check error code for benckmark_xl command. This will exit if not.
return ${PIPESTATUS[0]}
)
-
-
}
# Helper function to wait for the CPU temperature to cool down on ARM.
@@ -1027,7 +1050,7 @@ cmd_arm_benchmark() {
local src_img
for src_img in "${jpg_images[@]}" "${images[@]}"; do
local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ')
- local enc_binaries=("${BUILD_DIR}/tools/cjxl")
+ local enc_binaries=("${TOOLS_DIR}/cjxl")
local src_ext="${src_img##*.}"
for enc_binary in "${enc_binaries[@]}"; do
local enc_binary_base=$(basename "${enc_binary}")
@@ -1076,7 +1099,7 @@ cmd_arm_benchmark() {
local dec_output
wait_for_temp
- dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \
+ dec_output=$("${TOOLS_DIR}/djxl" "${enc_file}" \
--num_reps=5 --num_threads="${num_threads}" 2>&1 | tee /dev/stderr |
grep -E "M[BP]/s \[")
local img_size=$(echo "${dec_output}" | cut -f 1 -d ',')
@@ -1092,7 +1115,7 @@ cmd_arm_benchmark() {
if [[ "${src_ext}" == "jpg" ]]; then
wait_for_temp
local dec_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jpg"
- dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \
+ dec_output=$("${TOOLS_DIR}/djxl" "${enc_file}" \
"${dec_file}" --num_reps=5 --num_threads="${num_threads}" 2>&1 | \
tee /dev/stderr | grep -E "M[BP]/s \[")
local jpeg_dec_mps_speed=$(_speed_from_output "${dec_output}")
@@ -1122,12 +1145,12 @@ cmd_fuzz() {
local fuzzer_crash_dir=$(realpath "${BUILD_DIR}/fuzzer_crash")
mkdir -p "${corpus_dir}" "${fuzzer_crash_dir}"
# Generate step.
- "${BUILD_DIR}/tools/fuzzer_corpus" "${corpus_dir}"
+ "${TOOLS_DIR}/fuzzer_corpus" "${corpus_dir}"
# Run step:
local nprocs=$(nproc --all || echo 1)
(
- cd "${BUILD_DIR}"
- "tools/djxl_fuzzer" "${fuzzer_crash_dir}" "${corpus_dir}" \
+ cd "${TOOLS_DIR}"
+ djxl_fuzzer "${fuzzer_crash_dir}" "${corpus_dir}" \
-max_total_time="${FUZZER_MAX_TIME}" -jobs=${nprocs} \
-artifact_prefix="${fuzzer_crash_dir}/"
)