diff options
Diffstat (limited to 'third_party/jpeg-xl/.github/workflows/build_test.yml')
| -rw-r--r-- | third_party/jpeg-xl/.github/workflows/build_test.yml | 433 |
1 files changed, 433 insertions, 0 deletions
diff --git a/third_party/jpeg-xl/.github/workflows/build_test.yml b/third_party/jpeg-xl/.github/workflows/build_test.yml new file mode 100644 index 0000000000..ddc81b6fb0 --- /dev/null +++ b/third_party/jpeg-xl/.github/workflows/build_test.yml @@ -0,0 +1,433 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Workflow for building and running tests. + +name: Build/Test +on: + push: + branches: + - main + - v*.*.x + pull_request: + types: [opened, reopened, labeled, synchronize] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + ubuntu_build: + name: Ubuntu Build ${{ matrix.name }} + runs-on: ${{ matrix.os || 'ubuntu-latest' }} + strategy: + matrix: + # We have one job per "name" in the matrix. Attributes are set on the + # specific job names. + name: [release, debug, asan, msan, scalar] + include: + - name: release + test_in_pr: true + # Track static stack size on build and check it doesn't exceed 3 kB. + env_stack_size: 1 + max_stack: 3000 + # Conformance tooling test requires numpy. + apt_pkgs: graphviz python3-numpy + - name: lowprecision + mode: release + test_in_pr: true + cmake_args: -DCMAKE_CXX_FLAGS=-DJXL_HIGH_PRECISION=0 + - name: debug + # Runs on AVX3 CPUs require more stack than others. Make sure to + # test on AVX3-enabled CPUs when changing this value. + env_test_stack_size: 4000 + # Build scalar-only hwy instructions. + - name: scalar + mode: release + cxxflags: -DHWY_COMPILE_ONLY_SCALAR + # Disabling optional features to speed up msan build a little bit. + - name: msan + skip_install: true + cmake_args: >- + -DJPEGXL_ENABLE_DEVTOOLS=OFF -DJPEGXL_ENABLE_PLUGINS=OFF + -DJPEGXL_ENABLE_VIEWERS=OFF + - name: asan + skip_install: true + - name: coverage + apt_pkgs: gcovr + # Coverage builds require a bit more RAM. + env_test_stack_size: 2048 + # Build with support for decoding to JPEG bytes disabled. Produces a + # smaller build if only decoding to pixels is needed. + - name: release-nojpeg + mode: release + cxxflags: -DJXL_DEBUG_ON_ABORT=0 + cmake_args: >- + -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF + -DJPEGXL_ENABLE_PLUGINS=OFF + -DJPEGXL_ENABLE_VIEWERS=OFF + # Build optimized for binary size, all features not needed for + # reconstructing pixels is disabled. + - name: release:minimal + mode: release + cxxflags: -DJXL_DEBUG_ON_ABORT=0 + cmake_args: >- + -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF + -DJPEGXL_ENABLE_BOXES=OFF + -DJPEGXL_ENABLE_PLUGINS=OFF + -DJPEGXL_ENABLE_VIEWERS=OFF + # Builds with gcc in release mode + - name: release:gcc8 + mode: release + apt_pkgs: gcc-8 g++-8 + cmake_args: >- + -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 + # Builds with clang-5 in release mode + - name: release:clang-5 + os: ubuntu-18.04 + mode: release + # TODO(eustas): investigate, why static brotli library is not found. + skip_install: true + apt_pkgs: clang-5.0 + cmake_args: >- + -DCMAKE_C_COMPILER=clang-5.0 -DCMAKE_CXX_COMPILER=clang++-5.0 + -DJPEGXL_ENABLE_PLUGINS=OFF + + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + # Whether we track the stack size. + STACK_SIZE: ${{ matrix.env_stack_size }} + TEST_STACK_LIMIT: ${{ matrix.env_test_stack_size }} + WILL_RUN_TESTS: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && (matrix.test_in_pr || contains(github.event.pull_request.labels.*.name, 'CI:full'))) }} + + steps: + - name: Install build deps + run: | + sudo apt update + sudo apt install -y \ + ccache \ + clang-7 \ + cmake \ + doxygen \ + libbenchmark-dev \ + libbenchmark-tools \ + libbrotli-dev \ + libgdk-pixbuf2.0-dev \ + libgif-dev \ + libgtest-dev \ + libgtk2.0-dev \ + libjpeg-dev \ + libopenexr-dev \ + libpng-dev \ + libwebp-dev \ + ninja-build \ + pkg-config \ + xvfb \ + ${{ matrix.apt_pkgs }} \ + # + echo "CC=clang-7" >> $GITHUB_ENV + echo "CXX=clang++-7" >> $GITHUB_ENV + - name: Checkout the source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 2 + + - name: Setup the LLVM source path + if: matrix.name == 'msan' + run: | + LLVM_ROOT=${GITHUB_WORKSPACE}/llvm_root + mkdir -p ${LLVM_ROOT} + echo "LLVM_ROOT=${LLVM_ROOT}" >> $GITHUB_ENV + - name: Cache LLVM sources + if: matrix.name == 'msan' + uses: actions/cache@v2 + with: + path: ${{ env.LLVM_ROOT }} + key: llvm + - name: Checkout the LLVM source + if: matrix.name == 'msan' + uses: actions/checkout@v2 + with: + submodules: false + repository: llvm/llvm-project + ref: llvmorg-7.0.1 + path: llvm_root + + - name: Sphinx dependencies + # Dependencies for sphinx HTML documentation + if: matrix.name == 'release' + run: | + pip3 install -r doc/sphinx/requirements.txt + - name: Git environment + id: git-env + run: | + echo "::set-output name=parent::$(git rev-parse ${{ github.sha }}^)" + shell: bash + - name: ccache + uses: actions/cache@v2 + with: + path: ${{ env.CCACHE_DIR }} + # When the cache hits the key it is not updated, so if this is a rebuild + # of the same Pull Request it will reuse the cache if still around. For + # either Pull Requests or new pushes to main, this will use the parent + # hash as the starting point from the restore-keys entry. + key: build-${{ runner.os }}-${{ github.sha }}-${{ matrix.name }} + restore-keys: | + build-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }} + - name: Build + if: matrix.name != 'coverage' || env.WILL_RUN_TESTS == 'true' + run: | + mkdir -p ${CCACHE_DIR} + echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf + mode="${{ matrix.mode }}" + build_tests=$([ "$WILL_RUN_TESTS" == "true" ] && echo "ON" || echo "OFF") + [[ -n "${mode}" ]] || mode="${{ matrix.name }}" + ./ci.sh ${mode} -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DBUILD_TESTING=${build_tests} \ + ${{ matrix.cmake_args }} + env: + SKIP_TEST: 1 + CMAKE_CXX_FLAGS: ${{ matrix.cxxflags }} + - name: Build stats + run: | + awk '!/^#/ {total[$4]+=($2-$1);cntr[$4]+=1} END {for (key in total) print total[key]/cntr[key] " " key}' build/.ninja_log | sort -n | tail -n 25 + - name: ccache stats + run: ccache --show-stats + - name: Build stats ${{ matrix.name }} + if: matrix.mode == 'release' || matrix.name == 'release' + run: | + tools/build_stats.py --save build/stats.json \ + --max-stack ${{ matrix.max_stack || '0' }} \ + cjxl djxl libjxl.so libjxl_dec.so + # Check that we can build the example project against the installed libs. + - name: Install and build examples + if: | + (matrix.mode == 'release' || matrix.name == 'release') && + !matrix.skip_install + run: | + set -x + sudo cmake --build build -- install + cmake -Bbuild-example -Hexamples -G Ninja + cmake --build build-example + if ldd build-example/decode_oneshot_static | grep libjxl; then + echo "decode_oneshot_static is not using the static lib" >&2 + exit 1 + fi + # Test that the built binaries run. + echo -e -n "PF\n1 1\n-1.0\n\0\0\x80\x3f\0\0\x80\x3f\0\0\x80\x3f" > test.pfm + build-example/encode_oneshot test.pfm test.jxl + build-example/encode_oneshot_static test.pfm test-static.jxl + build-example/decode_oneshot test.jxl dec.pfm dec.icc + build-example/decode_oneshot_static test.jxl dec-static.pfm dec-static.icc + # Run the tests on push and when requested in pull_request. + - name: Test ${{ matrix.mode }} + if: env.WILL_RUN_TESTS == 'true' + run: | + ./ci.sh test ${{ matrix.ctest_args }} + # Print the running time summary for the slowest tests. + - name: Test runtime stats + run: | + sort build/Testing/Temporary/CTestCostData.txt -k 3 -n | tail -n 20 || true + - name: Build HTML documentation (sphinx/readthetdocs) + if: matrix.name == 'release' + run: | + cmake --build build -- rtd-html + - name: Coverage report + if: github.event_name == 'push' && matrix.name == 'coverage' + run: | + ./ci.sh coverage_report + - name: Coverage upload to Codecov + if: github.event_name == 'push' && matrix.name == 'coverage' + uses: codecov/codecov-action@v2 + with: + flags: unittests + files: build/coverage.xml + - name: Fast benchmark ${{ matrix.mode }} + if: | + matrix.name != 'coverage' && (github.event_name == 'push' || + (github.event_name == 'pull_request' && ( + matrix.test_in_pr || + contains(github.event.pull_request.labels.*.name, 'CI:full')))) + run: | + STORE_IMAGES=0 ./ci.sh fast_benchmark + # Run gbench once, just to make sure it runs, not for actual benchmarking. + # This doesn't work on msan because we use gbench library from the system + # which is not instrumented by msan. + - name: gbench check + if: | + matrix.name == 'release' || ( + github.event_name == 'push' && matrix.name != 'msan') + run: | + ./ci.sh gbench --benchmark_min_time=0 + + windows_msys: + name: Windows MSYS2 / ${{ matrix.msystem }} + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - msystem: mingw64 + - msystem: clang64 + - msystem: mingw32 + # TODO(eustas): investigate HWY Mul failures + disable_tests: + - HwyMulTestGroup/HwyMulTest\.TestAllMulHigh/EMU128 + - HwyMulTestGroup/HwyMulTest\.TestAllMulFixedPoint15/EMU128 + - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO2 + - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO3 + - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO7 + - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO8 + - msystem: clang32 + # TODO(eustas): investigate HWY Sort and JXL ANS failures + disable_tests: + - SortTestGroup/SortTest\.TestAllSort/.* + - ANSTest\.RandomUnbalancedStreamRoundtrip3 + - ANSTest\.RandomUnbalancedStreamRoundtripBig + + defaults: + run: + shell: msys2 {0} + steps: + - name: Checkout the source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 1 + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + update: true + path-type: inherit + install: >- + base-devel + git + pacboy: >- + brotli:p + cmake:p + giflib:p + gtest:p + libavif:p + libjpeg-turbo:p + libpng:p + libwebp:p + ninja:p + toolchain:p + + - name: CMake configure + run: | + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DJPEGXL_ENABLE_JNI=OFF \ + -DJPEGXL_ENABLE_MANPAGES=OFF \ + -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \ + -DJPEGXL_FORCE_SYSTEM_GTEST=ON \ + -B build \ + -G Ninja + - name: CMake build + run: cmake --build build + - name: Test + if: | + github.event_name == 'push' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'CI:full')) + run: ctest --test-dir build --parallel 2 --output-on-failure -E "${{ join(matrix.disable_tests, '|') }}" + + wasm32_build: + name: WASM wasm32/${{ matrix.variant }} + runs-on: ubuntu-latest + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + BUILD_TARGET: wasm32 + EM_VERSION: 3.1.1 + NODE_VERSION: 18 + + strategy: + matrix: + include: + - variant: scalar + - variant: simd + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 1 + - name: Install build deps + shell: bash + run: | + set -x + sudo apt update + pkgs=( + # Build dependencies + ccache + cmake + doxygen + ninja-build + pkg-config + ) + DEBIAN_FRONTEND=noninteractive sudo apt install -y "${pkgs[@]}" + + - name: Git environment + id: git-env + run: | + echo "::set-output name=parent::$(git rev-parse ${{ github.sha }}^)" + shell: bash + - name: ccache + uses: actions/cache@v2 + with: + path: ${{ env.CCACHE_DIR }} + key: build-wasm-${{ runner.os }}-${{ github.sha }}-${{ matrix.variant }} + restore-keys: | + build-wasm-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.variant }} + + - name: Install node + uses: actions/setup-node@v3 + with: + node-version: ${{env.NODE_VERSION}} + + - name: Get non-EMSDK node path + run: which node >> $HOME/.base_node_path + + - name: Install emsdk + uses: mymindstorm/setup-emsdk@v11 + # TODO(deymo): We could cache this action but it doesn't work when running + # in a matrix. + with: + version: ${{env.EM_VERSION}} + no-cache: true + + - name: Set EMSDK node version + run: | + echo "NODE_JS='$(cat $HOME/.base_node_path)'" >> $EMSDK/.emscripten + emsdk construct_env + + # TODO(deymo): Build and install other dependencies like libpng, libjpeg, + # etc. + - name: Build + run: | + mkdir -p ${CCACHE_DIR} + echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf + if [[ "${{ matrix.variant }}" == "simd" ]]; then + export ENABLE_WASM_SIMD=1 + fi + ./ci.sh release \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache + env: + SKIP_TEST: 1 + - name: ccache stats + run: ccache --show-stats + + - name: Test + if: | + github.event_name == 'push' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'CI:full')) + run: | + ./ci.sh test |