path: root/third_party/jpeg-xl/.github/workflows/build_test.yml

# Copyright (c) the JPEG XL Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# Workflow for building and running tests.

name: Build/Test
on:
  push:
    branches:
      - main
      - v*.*.x
  pull_request:
    types: [opened, reopened, labeled, synchronize]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  ubuntu_build:
    name: Ubuntu Build ${{ matrix.name }}
    runs-on: ${{ matrix.os || 'ubuntu-latest' }}
    strategy:
      matrix:
        # We have one job per "name" in the matrix. Attributes are set on the
        # specific job names.
        name: [release, debug, asan, msan, scalar]
        include:
          - name: release
            test_in_pr: true
            # Track static stack size on build and check it doesn't exceed 3 kB.
            env_stack_size: 1
            max_stack: 3000
            # Conformance tooling test requires numpy.
            apt_pkgs: graphviz python3-numpy
          - name: lowprecision
            mode: release
            test_in_pr: true
            cmake_args: -DCMAKE_CXX_FLAGS=-DJXL_HIGH_PRECISION=0
          - name: debug
            # Runs on AVX3 CPUs require more stack than others. Make sure to
            # test on AVX3-enabled CPUs when changing this value.
            env_test_stack_size: 4000
          # Build scalar-only hwy instructions.
          - name: scalar
            mode: release
            cxxflags: -DHWY_COMPILE_ONLY_SCALAR
          # Disabling optional features to speed up msan build a little bit.
          - name: msan
            skip_install: true
            cmake_args: >-
              -DJPEGXL_ENABLE_DEVTOOLS=OFF -DJPEGXL_ENABLE_PLUGINS=OFF
              -DJPEGXL_ENABLE_VIEWERS=OFF
          - name: asan
            skip_install: true
          - name: coverage
            apt_pkgs: gcovr
            # Coverage builds require a bit more RAM.
            env_test_stack_size: 2048
          # Build with support for decoding to JPEG bytes disabled. Produces a
          # smaller build if only decoding to pixels is needed.
          - name: release-nojpeg
            mode: release
            cxxflags: -DJXL_DEBUG_ON_ABORT=0
            cmake_args: >-
              -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF
              -DJPEGXL_ENABLE_PLUGINS=OFF
              -DJPEGXL_ENABLE_VIEWERS=OFF
          # Build optimized for binary size, all features not needed for
          # reconstructing pixels is disabled.
          - name: release:minimal
            mode: release
            cxxflags: -DJXL_DEBUG_ON_ABORT=0
            cmake_args: >-
              -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF
              -DJPEGXL_ENABLE_BOXES=OFF
              -DJPEGXL_ENABLE_PLUGINS=OFF
              -DJPEGXL_ENABLE_VIEWERS=OFF
          # Builds with gcc in release mode
          - name: release:gcc8
            mode: release
            apt_pkgs: gcc-8 g++-8
            cmake_args: >-
              -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8
          # Builds with clang-5 in release mode
          - name: release:clang-5
            os: ubuntu-18.04
            mode: release
            # TODO(eustas): investigate, why static brotli library is not found.
            skip_install: true
            apt_pkgs: clang-5.0
            cmake_args: >-
              -DCMAKE_C_COMPILER=clang-5.0 -DCMAKE_CXX_COMPILER=clang++-5.0
              -DJPEGXL_ENABLE_PLUGINS=OFF

    env:
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      # Whether we track the stack size.
      STACK_SIZE: ${{ matrix.env_stack_size }}
      TEST_STACK_LIMIT: ${{ matrix.env_test_stack_size }}
      WILL_RUN_TESTS: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && (matrix.test_in_pr || contains(github.event.pull_request.labels.*.name, 'CI:full'))) }}

    steps:
    - name: Install build deps
      run: |
        sudo apt update
        sudo apt install -y \
          ccache \
          clang-7 \
          cmake \
          doxygen \
          libbenchmark-dev \
          libbenchmark-tools \
          libbrotli-dev \
          libgdk-pixbuf2.0-dev \
          libgif-dev \
          libgtest-dev \
          libgtk2.0-dev  \
          libjpeg-dev \
          libopenexr-dev \
          libpng-dev \
          libwebp-dev \
          ninja-build \
          pkg-config \
          xvfb \
          ${{ matrix.apt_pkgs }} \
        #
        echo "CC=clang-7" >> $GITHUB_ENV
        echo "CXX=clang++-7" >> $GITHUB_ENV
    - name: Checkout the source
      uses: actions/checkout@v2
      with:
        submodules: true
        fetch-depth: 2

    - name: Setup the LLVM source path
      if: matrix.name == 'msan'
      run: |
        LLVM_ROOT=${GITHUB_WORKSPACE}/llvm_root
        mkdir -p ${LLVM_ROOT}
        echo "LLVM_ROOT=${LLVM_ROOT}" >> $GITHUB_ENV
    - name: Cache LLVM sources
      if: matrix.name == 'msan'
      uses: actions/cache@v2
      with:
        path: ${{ env.LLVM_ROOT }}
        key: llvm
    - name: Checkout the LLVM source
      if: matrix.name == 'msan'
      uses: actions/checkout@v2
      with:
        submodules: false
        repository: llvm/llvm-project
        ref: llvmorg-7.0.1
        path: llvm_root

    - name: Sphinx dependencies
      # Dependencies for sphinx HTML documentation
      if: matrix.name == 'release'
      run: |
        pip3 install -r doc/sphinx/requirements.txt
    - name: Git environment
      id: git-env
      run: |
        echo "::set-output name=parent::$(git rev-parse ${{ github.sha }}^)"
      shell: bash
    - name: ccache
      uses: actions/cache@v2
      with:
        path: ${{ env.CCACHE_DIR }}
        # When the cache hits the key it is not updated, so if this is a rebuild
        # of the same Pull Request it will reuse the cache if still around. For
        # either Pull Requests or new pushes to main, this will use the parent
        # hash as the starting point from the restore-keys entry.
        key: build-${{ runner.os }}-${{ github.sha }}-${{ matrix.name }}
        restore-keys: |
          build-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }}
    - name: Build
      if: matrix.name != 'coverage' || env.WILL_RUN_TESTS == 'true'
      run: |
        mkdir -p ${CCACHE_DIR}
        echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf
        mode="${{ matrix.mode }}"
        build_tests=$([ "$WILL_RUN_TESTS" == "true" ] && echo "ON" || echo "OFF")
        [[ -n "${mode}" ]] || mode="${{ matrix.name }}"
        ./ci.sh ${mode} -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
          -DBUILD_TESTING=${build_tests} \
          ${{ matrix.cmake_args }}
      env:
        SKIP_TEST: 1
        CMAKE_CXX_FLAGS: ${{ matrix.cxxflags }}
    - name: Build stats
      run: |
        awk '!/^#/ {total[$4]+=($2-$1);cntr[$4]+=1} END {for (key in total) print total[key]/cntr[key] " " key}' build/.ninja_log | sort -n | tail -n 25
    - name: ccache stats
      run: ccache --show-stats
    - name: Build stats ${{ matrix.name }}
      if: matrix.mode == 'release' || matrix.name == 'release'
      run: |
        tools/build_stats.py --save build/stats.json \
          --max-stack ${{ matrix.max_stack || '0' }} \
          cjxl djxl libjxl.so libjxl_dec.so
    # Check that we can build the example project against the installed libs.
    - name: Install and build examples
      if: |
        (matrix.mode == 'release' || matrix.name == 'release') &&
        !matrix.skip_install
      run: |
        set -x
        sudo cmake --build build -- install
        cmake -Bbuild-example -Hexamples -G Ninja
        cmake --build build-example
        if ldd build-example/decode_oneshot_static | grep libjxl; then
          echo "decode_oneshot_static is not using the static lib" >&2
          exit 1
        fi
        # Test that the built binaries run.
        echo -e -n "PF\n1 1\n-1.0\n\0\0\x80\x3f\0\0\x80\x3f\0\0\x80\x3f" > test.pfm
        build-example/encode_oneshot test.pfm test.jxl
        build-example/encode_oneshot_static test.pfm test-static.jxl
        build-example/decode_oneshot test.jxl dec.pfm dec.icc
        build-example/decode_oneshot_static test.jxl dec-static.pfm dec-static.icc
    # Run the tests on push and when requested in pull_request.
    - name: Test ${{ matrix.mode }}
      if: env.WILL_RUN_TESTS == 'true'
      run: |
        ./ci.sh test ${{ matrix.ctest_args }}
    # Print the running time summary for the slowest tests.
    - name: Test runtime stats
      run: |
        sort build/Testing/Temporary/CTestCostData.txt -k 3 -n | tail -n 20 || true
    - name: Build HTML documentation (sphinx/readthetdocs)
      if: matrix.name == 'release'
      run: |
        cmake --build build -- rtd-html
    - name: Coverage report
      if: github.event_name == 'push' && matrix.name == 'coverage'
      run: |
        ./ci.sh coverage_report
    - name: Coverage upload to Codecov
      if: github.event_name == 'push' && matrix.name == 'coverage'
      uses: codecov/codecov-action@v2
      with:
        flags: unittests
        files: build/coverage.xml
    - name: Fast benchmark ${{ matrix.mode }}
      if: |
        matrix.name != 'coverage' && (github.event_name == 'push' ||
        (github.event_name == 'pull_request' && (
         matrix.test_in_pr ||
         contains(github.event.pull_request.labels.*.name, 'CI:full'))))
      run: |
        STORE_IMAGES=0 ./ci.sh fast_benchmark
    # Run gbench once, just to make sure it runs, not for actual benchmarking.
    # This doesn't work on msan because we use gbench library from the system
    # which is not instrumented by msan.
    - name: gbench check
      if: |
        matrix.name == 'release' || (
          github.event_name == 'push' && matrix.name != 'msan')
      run: |
        ./ci.sh gbench --benchmark_min_time=0

  windows_msys:
    name: Windows MSYS2 / ${{ matrix.msystem }}
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        include:
         - msystem: mingw64
         - msystem: clang64
         - msystem: mingw32
           # TODO(eustas): investigate HWY Mul failures
           disable_tests:
             - HwyMulTestGroup/HwyMulTest\.TestAllMulHigh/EMU128
             - HwyMulTestGroup/HwyMulTest\.TestAllMulFixedPoint15/EMU128
             - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO2
             - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO3
             - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO7
             - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO8
         - msystem: clang32
           # TODO(eustas): investigate HWY Sort and JXL ANS failures
           disable_tests:
             - SortTestGroup/SortTest\.TestAllSort/.*
             - ANSTest\.RandomUnbalancedStreamRoundtrip3
             - ANSTest\.RandomUnbalancedStreamRoundtripBig

    defaults:
      run:
        shell: msys2 {0}
    steps:
      - name: Checkout the source
        uses: actions/checkout@v2
        with:
          submodules: true
          fetch-depth: 1
      - uses: msys2/setup-msys2@v2
        with:
          msystem: ${{ matrix.msystem }}
          update: true
          path-type: inherit
          install: >-
            base-devel
            git
          pacboy: >-
            brotli:p
            cmake:p
            giflib:p
            gtest:p
            libavif:p
            libjpeg-turbo:p
            libpng:p
            libwebp:p
            ninja:p
            toolchain:p

      - name: CMake configure
        run: |
          cmake \
            -DCMAKE_BUILD_TYPE=Release \
            -DJPEGXL_ENABLE_JNI=OFF \
            -DJPEGXL_ENABLE_MANPAGES=OFF \
            -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
            -DJPEGXL_FORCE_SYSTEM_GTEST=ON \
            -B build \
            -G Ninja
      - name: CMake build
        run: cmake --build build
      - name: Test
        if: |
          github.event_name == 'push' ||
          (github.event_name == 'pull_request' &&
           contains(github.event.pull_request.labels.*.name, 'CI:full'))
        run: ctest --test-dir build --parallel 2 --output-on-failure -E "${{ join(matrix.disable_tests, '|') }}"

  wasm32_build:
    name: WASM wasm32/${{ matrix.variant }}
    runs-on: ubuntu-latest
    env:
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      BUILD_TARGET: wasm32
      EM_VERSION: 3.1.1
      NODE_VERSION: 18

    strategy:
      matrix:
        include:
          - variant: scalar
          - variant: simd

    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
        fetch-depth: 1
    - name: Install build deps
      shell: bash
      run: |
        set -x
        sudo apt update
        pkgs=(
          # Build dependencies
          ccache
          cmake
          doxygen
          ninja-build
          pkg-config
        )
        DEBIAN_FRONTEND=noninteractive sudo apt install -y "${pkgs[@]}"

    - name: Git environment
      id: git-env
      run: |
        echo "::set-output name=parent::$(git rev-parse ${{ github.sha }}^)"
      shell: bash
    - name: ccache
      uses: actions/cache@v2
      with:
        path: ${{ env.CCACHE_DIR }}
        key: build-wasm-${{ runner.os }}-${{ github.sha }}-${{ matrix.variant }}
        restore-keys: |
          build-wasm-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.variant }}

    - name: Install node
      uses: actions/setup-node@v3
      with:
        node-version: ${{env.NODE_VERSION}}

    - name: Get non-EMSDK node path
      run: which node >> $HOME/.base_node_path

    - name: Install emsdk
      uses: mymindstorm/setup-emsdk@v11
      # TODO(deymo): We could cache this action but it doesn't work when running
      # in a matrix.
      with:
        version: ${{env.EM_VERSION}}
        no-cache: true

    - name: Set EMSDK node version
      run: |
        echo "NODE_JS='$(cat $HOME/.base_node_path)'" >> $EMSDK/.emscripten
        emsdk construct_env

    # TODO(deymo): Build and install other dependencies like libpng, libjpeg,
    # etc.
    - name: Build
      run: |
        mkdir -p ${CCACHE_DIR}
        echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf
        if [[ "${{ matrix.variant }}" == "simd" ]]; then
          export ENABLE_WASM_SIMD=1
        fi
        ./ci.sh release \
          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
          -DCMAKE_C_COMPILER_LAUNCHER=ccache
      env:
        SKIP_TEST: 1
    - name: ccache stats
      run: ccache --show-stats

    - name: Test
      if: |
        github.event_name == 'push' ||
        (github.event_name == 'pull_request' &&
         contains(github.event.pull_request.labels.*.name, 'CI:full'))
      run: |
        ./ci.sh test