diff options
Diffstat (limited to 'third_party/jpeg-xl/.github/workflows')
| -rw-r--r-- | third_party/jpeg-xl/.github/workflows/build_test.yml | 433 | ||||
| -rw-r--r-- | third_party/jpeg-xl/.github/workflows/build_test_cross.yml | 183 | ||||
| -rw-r--r-- | third_party/jpeg-xl/.github/workflows/conformance.yml | 185 | ||||
| -rw-r--r-- | third_party/jpeg-xl/.github/workflows/debug_ci.yml | 59 | ||||
| -rw-r--r-- | third_party/jpeg-xl/.github/workflows/fuzz.yml | 56 | ||||
| -rw-r--r-- | third_party/jpeg-xl/.github/workflows/pull_request.yml | 42 | ||||
| -rw-r--r-- | third_party/jpeg-xl/.github/workflows/release.yaml | 378 |
7 files changed, 1336 insertions, 0 deletions
diff --git a/third_party/jpeg-xl/.github/workflows/build_test.yml b/third_party/jpeg-xl/.github/workflows/build_test.yml new file mode 100644 index 0000000000..ddc81b6fb0 --- /dev/null +++ b/third_party/jpeg-xl/.github/workflows/build_test.yml @@ -0,0 +1,433 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Workflow for building and running tests. + +name: Build/Test +on: + push: + branches: + - main + - v*.*.x + pull_request: + types: [opened, reopened, labeled, synchronize] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + ubuntu_build: + name: Ubuntu Build ${{ matrix.name }} + runs-on: ${{ matrix.os || 'ubuntu-latest' }} + strategy: + matrix: + # We have one job per "name" in the matrix. Attributes are set on the + # specific job names. + name: [release, debug, asan, msan, scalar] + include: + - name: release + test_in_pr: true + # Track static stack size on build and check it doesn't exceed 3 kB. + env_stack_size: 1 + max_stack: 3000 + # Conformance tooling test requires numpy. + apt_pkgs: graphviz python3-numpy + - name: lowprecision + mode: release + test_in_pr: true + cmake_args: -DCMAKE_CXX_FLAGS=-DJXL_HIGH_PRECISION=0 + - name: debug + # Runs on AVX3 CPUs require more stack than others. Make sure to + # test on AVX3-enabled CPUs when changing this value. + env_test_stack_size: 4000 + # Build scalar-only hwy instructions. + - name: scalar + mode: release + cxxflags: -DHWY_COMPILE_ONLY_SCALAR + # Disabling optional features to speed up msan build a little bit. + - name: msan + skip_install: true + cmake_args: >- + -DJPEGXL_ENABLE_DEVTOOLS=OFF -DJPEGXL_ENABLE_PLUGINS=OFF + -DJPEGXL_ENABLE_VIEWERS=OFF + - name: asan + skip_install: true + - name: coverage + apt_pkgs: gcovr + # Coverage builds require a bit more RAM. + env_test_stack_size: 2048 + # Build with support for decoding to JPEG bytes disabled. Produces a + # smaller build if only decoding to pixels is needed. + - name: release-nojpeg + mode: release + cxxflags: -DJXL_DEBUG_ON_ABORT=0 + cmake_args: >- + -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF + -DJPEGXL_ENABLE_PLUGINS=OFF + -DJPEGXL_ENABLE_VIEWERS=OFF + # Build optimized for binary size, all features not needed for + # reconstructing pixels is disabled. + - name: release:minimal + mode: release + cxxflags: -DJXL_DEBUG_ON_ABORT=0 + cmake_args: >- + -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF + -DJPEGXL_ENABLE_BOXES=OFF + -DJPEGXL_ENABLE_PLUGINS=OFF + -DJPEGXL_ENABLE_VIEWERS=OFF + # Builds with gcc in release mode + - name: release:gcc8 + mode: release + apt_pkgs: gcc-8 g++-8 + cmake_args: >- + -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 + # Builds with clang-5 in release mode + - name: release:clang-5 + os: ubuntu-18.04 + mode: release + # TODO(eustas): investigate, why static brotli library is not found. + skip_install: true + apt_pkgs: clang-5.0 + cmake_args: >- + -DCMAKE_C_COMPILER=clang-5.0 -DCMAKE_CXX_COMPILER=clang++-5.0 + -DJPEGXL_ENABLE_PLUGINS=OFF + + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + # Whether we track the stack size. + STACK_SIZE: ${{ matrix.env_stack_size }} + TEST_STACK_LIMIT: ${{ matrix.env_test_stack_size }} + WILL_RUN_TESTS: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && (matrix.test_in_pr || contains(github.event.pull_request.labels.*.name, 'CI:full'))) }} + + steps: + - name: Install build deps + run: | + sudo apt update + sudo apt install -y \ + ccache \ + clang-7 \ + cmake \ + doxygen \ + libbenchmark-dev \ + libbenchmark-tools \ + libbrotli-dev \ + libgdk-pixbuf2.0-dev \ + libgif-dev \ + libgtest-dev \ + libgtk2.0-dev \ + libjpeg-dev \ + libopenexr-dev \ + libpng-dev \ + libwebp-dev \ + ninja-build \ + pkg-config \ + xvfb \ + ${{ matrix.apt_pkgs }} \ + # + echo "CC=clang-7" >> $GITHUB_ENV + echo "CXX=clang++-7" >> $GITHUB_ENV + - name: Checkout the source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 2 + + - name: Setup the LLVM source path + if: matrix.name == 'msan' + run: | + LLVM_ROOT=${GITHUB_WORKSPACE}/llvm_root + mkdir -p ${LLVM_ROOT} + echo "LLVM_ROOT=${LLVM_ROOT}" >> $GITHUB_ENV + - name: Cache LLVM sources + if: matrix.name == 'msan' + uses: actions/cache@v2 + with: + path: ${{ env.LLVM_ROOT }} + key: llvm + - name: Checkout the LLVM source + if: matrix.name == 'msan' + uses: actions/checkout@v2 + with: + submodules: false + repository: llvm/llvm-project + ref: llvmorg-7.0.1 + path: llvm_root + + - name: Sphinx dependencies + # Dependencies for sphinx HTML documentation + if: matrix.name == 'release' + run: | + pip3 install -r doc/sphinx/requirements.txt + - name: Git environment + id: git-env + run: | + echo "::set-output name=parent::$(git rev-parse ${{ github.sha }}^)" + shell: bash + - name: ccache + uses: actions/cache@v2 + with: + path: ${{ env.CCACHE_DIR }} + # When the cache hits the key it is not updated, so if this is a rebuild + # of the same Pull Request it will reuse the cache if still around. For + # either Pull Requests or new pushes to main, this will use the parent + # hash as the starting point from the restore-keys entry. + key: build-${{ runner.os }}-${{ github.sha }}-${{ matrix.name }} + restore-keys: | + build-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }} + - name: Build + if: matrix.name != 'coverage' || env.WILL_RUN_TESTS == 'true' + run: | + mkdir -p ${CCACHE_DIR} + echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf + mode="${{ matrix.mode }}" + build_tests=$([ "$WILL_RUN_TESTS" == "true" ] && echo "ON" || echo "OFF") + [[ -n "${mode}" ]] || mode="${{ matrix.name }}" + ./ci.sh ${mode} -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DBUILD_TESTING=${build_tests} \ + ${{ matrix.cmake_args }} + env: + SKIP_TEST: 1 + CMAKE_CXX_FLAGS: ${{ matrix.cxxflags }} + - name: Build stats + run: | + awk '!/^#/ {total[$4]+=($2-$1);cntr[$4]+=1} END {for (key in total) print total[key]/cntr[key] " " key}' build/.ninja_log | sort -n | tail -n 25 + - name: ccache stats + run: ccache --show-stats + - name: Build stats ${{ matrix.name }} + if: matrix.mode == 'release' || matrix.name == 'release' + run: | + tools/build_stats.py --save build/stats.json \ + --max-stack ${{ matrix.max_stack || '0' }} \ + cjxl djxl libjxl.so libjxl_dec.so + # Check that we can build the example project against the installed libs. + - name: Install and build examples + if: | + (matrix.mode == 'release' || matrix.name == 'release') && + !matrix.skip_install + run: | + set -x + sudo cmake --build build -- install + cmake -Bbuild-example -Hexamples -G Ninja + cmake --build build-example + if ldd build-example/decode_oneshot_static | grep libjxl; then + echo "decode_oneshot_static is not using the static lib" >&2 + exit 1 + fi + # Test that the built binaries run. + echo -e -n "PF\n1 1\n-1.0\n\0\0\x80\x3f\0\0\x80\x3f\0\0\x80\x3f" > test.pfm + build-example/encode_oneshot test.pfm test.jxl + build-example/encode_oneshot_static test.pfm test-static.jxl + build-example/decode_oneshot test.jxl dec.pfm dec.icc + build-example/decode_oneshot_static test.jxl dec-static.pfm dec-static.icc + # Run the tests on push and when requested in pull_request. + - name: Test ${{ matrix.mode }} + if: env.WILL_RUN_TESTS == 'true' + run: | + ./ci.sh test ${{ matrix.ctest_args }} + # Print the running time summary for the slowest tests. + - name: Test runtime stats + run: | + sort build/Testing/Temporary/CTestCostData.txt -k 3 -n | tail -n 20 || true + - name: Build HTML documentation (sphinx/readthetdocs) + if: matrix.name == 'release' + run: | + cmake --build build -- rtd-html + - name: Coverage report + if: github.event_name == 'push' && matrix.name == 'coverage' + run: | + ./ci.sh coverage_report + - name: Coverage upload to Codecov + if: github.event_name == 'push' && matrix.name == 'coverage' + uses: codecov/codecov-action@v2 + with: + flags: unittests + files: build/coverage.xml + - name: Fast benchmark ${{ matrix.mode }} + if: | + matrix.name != 'coverage' && (github.event_name == 'push' || + (github.event_name == 'pull_request' && ( + matrix.test_in_pr || + contains(github.event.pull_request.labels.*.name, 'CI:full')))) + run: | + STORE_IMAGES=0 ./ci.sh fast_benchmark + # Run gbench once, just to make sure it runs, not for actual benchmarking. + # This doesn't work on msan because we use gbench library from the system + # which is not instrumented by msan. + - name: gbench check + if: | + matrix.name == 'release' || ( + github.event_name == 'push' && matrix.name != 'msan') + run: | + ./ci.sh gbench --benchmark_min_time=0 + + windows_msys: + name: Windows MSYS2 / ${{ matrix.msystem }} + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - msystem: mingw64 + - msystem: clang64 + - msystem: mingw32 + # TODO(eustas): investigate HWY Mul failures + disable_tests: + - HwyMulTestGroup/HwyMulTest\.TestAllMulHigh/EMU128 + - HwyMulTestGroup/HwyMulTest\.TestAllMulFixedPoint15/EMU128 + - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO2 + - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO3 + - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO7 + - DecodeTest/DecodeTestParam\.PixelTest/280x12RGBAtoRGBAf32BECallbackEarlyBufferO8 + - msystem: clang32 + # TODO(eustas): investigate HWY Sort and JXL ANS failures + disable_tests: + - SortTestGroup/SortTest\.TestAllSort/.* + - ANSTest\.RandomUnbalancedStreamRoundtrip3 + - ANSTest\.RandomUnbalancedStreamRoundtripBig + + defaults: + run: + shell: msys2 {0} + steps: + - name: Checkout the source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 1 + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + update: true + path-type: inherit + install: >- + base-devel + git + pacboy: >- + brotli:p + cmake:p + giflib:p + gtest:p + libavif:p + libjpeg-turbo:p + libpng:p + libwebp:p + ninja:p + toolchain:p + + - name: CMake configure + run: | + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DJPEGXL_ENABLE_JNI=OFF \ + -DJPEGXL_ENABLE_MANPAGES=OFF \ + -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \ + -DJPEGXL_FORCE_SYSTEM_GTEST=ON \ + -B build \ + -G Ninja + - name: CMake build + run: cmake --build build + - name: Test + if: | + github.event_name == 'push' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'CI:full')) + run: ctest --test-dir build --parallel 2 --output-on-failure -E "${{ join(matrix.disable_tests, '|') }}" + + wasm32_build: + name: WASM wasm32/${{ matrix.variant }} + runs-on: ubuntu-latest + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + BUILD_TARGET: wasm32 + EM_VERSION: 3.1.1 + NODE_VERSION: 18 + + strategy: + matrix: + include: + - variant: scalar + - variant: simd + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 1 + - name: Install build deps + shell: bash + run: | + set -x + sudo apt update + pkgs=( + # Build dependencies + ccache + cmake + doxygen + ninja-build + pkg-config + ) + DEBIAN_FRONTEND=noninteractive sudo apt install -y "${pkgs[@]}" + + - name: Git environment + id: git-env + run: | + echo "::set-output name=parent::$(git rev-parse ${{ github.sha }}^)" + shell: bash + - name: ccache + uses: actions/cache@v2 + with: + path: ${{ env.CCACHE_DIR }} + key: build-wasm-${{ runner.os }}-${{ github.sha }}-${{ matrix.variant }} + restore-keys: | + build-wasm-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.variant }} + + - name: Install node + uses: actions/setup-node@v3 + with: + node-version: ${{env.NODE_VERSION}} + + - name: Get non-EMSDK node path + run: which node >> $HOME/.base_node_path + + - name: Install emsdk + uses: mymindstorm/setup-emsdk@v11 + # TODO(deymo): We could cache this action but it doesn't work when running + # in a matrix. + with: + version: ${{env.EM_VERSION}} + no-cache: true + + - name: Set EMSDK node version + run: | + echo "NODE_JS='$(cat $HOME/.base_node_path)'" >> $EMSDK/.emscripten + emsdk construct_env + + # TODO(deymo): Build and install other dependencies like libpng, libjpeg, + # etc. + - name: Build + run: | + mkdir -p ${CCACHE_DIR} + echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf + if [[ "${{ matrix.variant }}" == "simd" ]]; then + export ENABLE_WASM_SIMD=1 + fi + ./ci.sh release \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache + env: + SKIP_TEST: 1 + - name: ccache stats + run: ccache --show-stats + + - name: Test + if: | + github.event_name == 'push' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'CI:full')) + run: | + ./ci.sh test diff --git a/third_party/jpeg-xl/.github/workflows/build_test_cross.yml b/third_party/jpeg-xl/.github/workflows/build_test_cross.yml new file mode 100644 index 0000000000..5b537205c4 --- /dev/null +++ b/third_party/jpeg-xl/.github/workflows/build_test_cross.yml @@ -0,0 +1,183 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Workflow for building and running tests. + +name: Build/Test Cross +on: + push: + branches: + - main + - v*.*.x + pull_request: + types: [opened, reopened, labeled, synchronize] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + cross_compile_ubuntu: + name: Cross-compiling ${{ matrix.build_target }} ${{ matrix.variant }} + runs-on: [ubuntu-22.04] + container: + image: debian:bullseye + strategy: + fail-fast: false + matrix: + include: + - arch: arm64 + build_target: aarch64-linux-gnu + cmake_args: + - -DCMAKE_CROSSCOMPILING_EMULATOR=/usr/bin/qemu-aarch64-static + + - arch: arm64 + variant: SVE + build_target: aarch64-linux-gnu + cmake_args: + - -DCMAKE_CROSSCOMPILING_EMULATOR=/usr/bin/qemu-aarch64-static + - -DJPEGXL_ENABLE_OPENEXR=off + - -DJPEGXL_ENABLE_SIZELESS_VECTORS=on + cmake_flags: -march=armv8-a+sve + c_compiler: aarch64-linux-gnu-gcc + cxx_compiler: aarch64-linux-gnu-g++ + disable_tests: true + + - arch: arm64 + variant: lowprecision + build_target: aarch64-linux-gnu + cmake_args: + - -DCMAKE_CROSSCOMPILING_EMULATOR=/usr/bin/qemu-aarch64-static + - -DCMAKE_CXX_FLAGS=-DJXL_HIGH_PRECISION=0 + + - arch: armhf + build_target: arm-linux-gnueabihf + cmake_args: [-DCMAKE_CROSSCOMPILING_EMULATOR=/usr/bin/qemu-arm-static] + + - arch: i386 + test_in_pr: true + build_target: i686-linux-gnu + + env: + BUILD_DIR: build + WILL_RUN_TESTS: ${{ (github.event_name == 'push' || (github.event_name == 'pull_request' && (matrix.test_in_pr || contains(github.event.pull_request.labels.*.name, 'CI:full')))) && !matrix.disable_tests }} + + steps: + - name: Setup apt + shell: bash + run: | + set -x + apt-get update -y + apt-get install -y ca-certificates debian-ports-archive-keyring + + dpkg --add-architecture "${{ matrix.arch }}" + + # Update the sources.list with the split of supported architectures. + bkplist="/etc/apt/sources.list.bkp" + mv /etc/apt/sources.list "${bkplist}" + + newlist="/etc/apt/sources.list" + rm -f "${newlist}" + + main_list="amd64,${{ matrix.arch }}" + port_list="" + if [[ "${{ matrix.arch }}" == "i386" ]]; then + main_list="amd64,i386" + else + port_list="${{ matrix.arch }}" + fi + + grep -v -E '^#' "${bkplist}" | + sed -E "s;^deb (http[^ ]+) (.*)\$;deb [arch=${main_list}] \\1 \\2\ndeb-src [arch=${main_list}] \\1 \\2;" \ + | tee -a "${newlist}" + + - name: Install build deps + shell: bash + run: | + set -x + apt update + pkgs=( + # Build dependencies + cmake + doxygen + git + graphviz + ninja-build + pkg-config + qemu-user-static + xdg-utils + xvfb + + # Toolchain for cross-compiling. + clang-11 + g++-aarch64-linux-gnu + libc6-dev-${{ matrix.arch }}-cross + libstdc++-10-dev-${{ matrix.arch }}-cross + libstdc++-10-dev:${{ matrix.arch }} + + # Dependencies + libbrotli-dev:${{ matrix.arch }} + libgif-dev:${{ matrix.arch }} + libjpeg-dev:${{ matrix.arch }} + libpng-dev:${{ matrix.arch }} + libwebp-dev:${{ matrix.arch }} + + # For OpenEXR: + libilmbase-dev:${{ matrix.arch }} + libopenexr-dev:${{ matrix.arch }} + + # GTK plugins + libgdk-pixbuf2.0-dev:${{ matrix.arch }} + libgtk2.0-dev:${{ matrix.arch }} + + # QT + libqt5x11extras5-dev:${{ matrix.arch }} + qtbase5-dev:${{ matrix.arch }} + ) + if [[ "${{ matrix.build_target }}" != "x86_64-linux-gnu" ]]; then + pkgs+=( + binutils-${{ matrix.build_target }} + gcc-${{ matrix.build_target }} + ) + fi + if [[ "${{ matrix.arch }}" != "i386" ]]; then + pkgs+=( + # TCMalloc + libgoogle-perftools-dev:${{ matrix.arch }} + libgoogle-perftools4:${{ matrix.arch }} + libtcmalloc-minimal4:${{ matrix.arch }} + libunwind-dev:${{ matrix.arch }} + ) + fi + DEBIAN_FRONTEND=noninteractive apt install -y "${pkgs[@]}" + echo "CC=${{ matrix.c_compiler || 'clang-11' }}" >> $GITHUB_ENV + echo "CXX=${{ matrix.cxx_compiler || 'clang++-11' }}" >> $GITHUB_ENV + - name: Checkout the source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 1 + - name: Build + run: | + CMAKE_FLAGS="${{ matrix.cmake_flags }}" ./ci.sh release \ + -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \ + -DJPEGXL_ENABLE_JNI=OFF \ + ${{ join(matrix.cmake_args, ' ') }} + env: + SKIP_TEST: 1 + BUILD_TARGET: ${{ matrix.build_target }} + - name: Build stats ${{ matrix.build_target }} + run: | + tools/build_stats.py --save build/stats.json \ + --binutils ${{ matrix.build_target }}- \ + --max-stack ${{ matrix.max_stack || '0' }} \ + cjxl djxl libjxl.so libjxl_dec.so + # Run the tests on push and when requested in pull_request. + - name: Test + if: env.WILL_RUN_TESTS == 'true' + run: | + ./ci.sh test + env: + BUILD_TARGET: ${{ matrix.build_target }} diff --git a/third_party/jpeg-xl/.github/workflows/conformance.yml b/third_party/jpeg-xl/.github/workflows/conformance.yml new file mode 100644 index 0000000000..9c85878f98 --- /dev/null +++ b/third_party/jpeg-xl/.github/workflows/conformance.yml @@ -0,0 +1,185 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Workflow for running conformance tests. + +name: Conformance +on: + push: + branches: + - main + - v*.*.x + pull_request: + types: [opened, reopened, labeled, synchronize] + +env: + LIBJXL_VERSION: 0.8.0 + LIBJXL_ABI_VERSION: 0.8 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + warmup: # If necessary, fetch files just once, before tests are run. + name: Warmup caches + runs-on: ubuntu-latest + steps: + - name: Checkout the conformance source + uses: actions/checkout@v2 + with: + repository: libjxl/conformance + # TODO(eustas): move ref to a global variable / file? + ref: 43d8135b50e53167ee6fee0b842b9eb15cfc4aa2 + path: conformance + - name: Cache + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/conformance/.objects + key: conformance-refs + - name: Download and link conformance files + run: | + ${{ github.workspace }}/conformance/scripts/download_and_symlink.sh + + build: + name: Conformance Build ${{ matrix.name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: AVX3 + cflags: -DHWY_DISABLED_TARGETS=HWY_AVX3-1 + - name: AVX2 + cflags: -DHWY_DISABLED_TARGETS=HWY_AVX2-1 + - name: SSE4 + cflags: -DHWY_DISABLED_TARGETS=HWY_SSE4-1 + - name: SSSE3 + cflags: -DHWY_DISABLED_TARGETS=HWY_SSSE3-1 + - name: EMU128 + cflags: -DHWY_COMPILE_ONLY_EMU128=1 + - name: SCALAR + cflags: -DHWY_COMPILE_ONLY_SCALAR=1 + - name: SCALAR_ASAN + cflags: -DHWY_COMPILE_ONLY_SCALAR=1 + build_type: asan + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + steps: + - name: Install build deps + run: | + sudo apt update + sudo apt install -y \ + ccache \ + clang-7 \ + cmake \ + doxygen \ + libbenchmark-dev \ + libbenchmark-tools \ + libbrotli-dev \ + libgdk-pixbuf2.0-dev \ + libgif-dev \ + libgtest-dev \ + libgtk2.0-dev \ + libjpeg-dev \ + libopenexr-dev \ + libpng-dev \ + libwebp-dev \ + ninja-build \ + pkg-config \ + xvfb \ + ${{ matrix.apt_pkgs }} \ + # + echo "CC=clang-7" >> $GITHUB_ENV + echo "CXX=clang++-7" >> $GITHUB_ENV + - name: Checkout the jxl source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 2 + - name: Git environment + id: git-env + run: | + echo "::set-output name=parent::$(git rev-parse ${{ github.sha }}^)" + shell: bash + - name: ccache + uses: actions/cache@v2 + with: + path: ${{ env.CCACHE_DIR }} + # When the cache hits the key it is not updated, so if this is a rebuild + # of the same Pull Request it will reuse the cache if still around. For + # either Pull Requests or new pushes to main, this will use the parent + # hash as the starting point from the restore-keys entry. + key: conformance-${{ runner.os }}-${{ github.sha }}-${{ matrix.name }} + restore-keys: | + conformance-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }} + - name: Build + run: | + mkdir -p ${CCACHE_DIR} + echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf + CMAKE_FLAGS="${{ matrix.cflags }}" \ + ./ci.sh ${{ matrix.build_type || 'release' }} -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DBUILD_TESTING=OFF + # Flatten the artifacts directory structure + cp tools/conformance/conformance.py build/tools/conformance + cp tools/conformance/lcms2.py build/tools/conformance + cp build/tools/djxl build/tools/conformance + cp build/libjxl.so.${{ env.LIBJXL_VERSION }} build/tools/conformance + cp build/libjxl_threads.so.${{ env.LIBJXL_VERSION }} build/tools/conformance + env: + SKIP_TEST: 1 + - uses: actions/upload-artifact@v2 + with: + name: conformance_binary-${{ matrix.name }} + path: | + build/tools/conformance/conformance.py + build/tools/conformance/lcms2.py + build/tools/conformance/djxl + build/tools/conformance/libjxl.so.${{ env.LIBJXL_VERSION }} + build/tools/conformance/libjxl_threads.so.${{ env.LIBJXL_VERSION }} + - name: ccache stats + run: ccache --show-stats + + run: + name: Conformance Test ${{ matrix.name }} on ${{ matrix.target }} + needs: [warmup, build] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + name: [main_level5, main_level10] + target: [AVX3, AVX2, SSE4, SSSE3, EMU128, SCALAR, SCALAR_ASAN] + steps: + - name: Install deps + run: | + pip install numpy + - name: Checkout the conformance source + uses: actions/checkout@v2 + with: + repository: libjxl/conformance + ref: 43d8135b50e53167ee6fee0b842b9eb15cfc4aa2 + path: conformance + - name: Cache + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/conformance/.objects + key: conformance-refs + - name: Download and link conformance files + run: | + ${{ github.workspace }}/conformance/scripts/download_and_symlink.sh + - uses: actions/download-artifact@v2 + with: + name: conformance_binary-${{ matrix.target }} + - name: Run conformance tests + run: | + chmod +x djxl + ln -s libjxl.so.${{ env.LIBJXL_VERSION }} libjxl.so.${{ env.LIBJXL_ABI_VERSION }} + ln -s libjxl_threads.so.${{ env.LIBJXL_VERSION }} libjxl_threads.so.${{ env.LIBJXL_ABI_VERSION }} + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd` + python conformance.py \ + --decoder=`pwd`/djxl \ + --corpus=`pwd`/conformance/testcases/${{ matrix.name }}.txt diff --git a/third_party/jpeg-xl/.github/workflows/debug_ci.yml b/third_party/jpeg-xl/.github/workflows/debug_ci.yml new file mode 100644 index 0000000000..fb3522eb28 --- /dev/null +++ b/third_party/jpeg-xl/.github/workflows/debug_ci.yml @@ -0,0 +1,59 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Workflow for building and then debugging on a specific commit. + +name: Build and Test debugging +on: + push: + branches: + - ci-*-debug + +jobs: + ubuntu_build: + name: Ubuntu Build and SSH + runs-on: [ubuntu-latest] + + steps: + - name: Install build deps + run: | + sudo apt update + sudo apt install -y \ + ccache \ + clang-7 \ + cmake \ + doxygen \ + libbrotli-dev \ + libgdk-pixbuf2.0-dev \ + libgif-dev \ + libgtest-dev \ + libgtk2.0-dev \ + libjpeg-dev \ + libopenexr-dev \ + libpng-dev \ + libwebp-dev \ + ninja-build \ + pkg-config \ + xvfb \ + ${{ matrix.apt_pkgs }} \ + # + echo "CC=clang-7" >> $GITHUB_ENV + echo "CXX=clang++-7" >> $GITHUB_ENV + - name: Checkout the source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 2 + - name: Build + run: | + ./ci.sh $(echo ${{ github.ref }} | sed 's_refs/heads/ci-\([a-z_]*\)-debug_\1_') \ + -DJPEGXL_FORCE_SYSTEM_BROTLI=ON + env: + SKIP_TEST: 1 + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + + + diff --git a/third_party/jpeg-xl/.github/workflows/fuzz.yml b/third_party/jpeg-xl/.github/workflows/fuzz.yml new file mode 100644 index 0000000000..188a4c79c7 --- /dev/null +++ b/third_party/jpeg-xl/.github/workflows/fuzz.yml @@ -0,0 +1,56 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# CI on pull-requests to run the fuzzer from oss-fuzz. See: +# +# https://google.github.io/oss-fuzz/getting-started/continuous-integration/ + +name: CIFuzz +on: + pull_request: + types: [opened, reopened, synchronize] + paths: + - '**.c' + - '**.cc' + - '**.cmake' + - '**.h' + - '**CMakeLists.txt' + - .github/workflows/fuzz.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + fuzzing: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v2 + id: checkout + with: + # The build_fuzzers action checks out the code to the storage/libjxl + # directory already, but doesn't check out the submodules. This step + # is a workaround for checking out the submodules. + path: storage/libjxl + submodules: true + - name: Build Fuzzers + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: 'libjxl' + language: c++ + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: 'libjxl' + language: c++ + fuzz-seconds: 600 + - name: Upload Crash + uses: actions/upload-artifact@v1 + if: failure() && steps.build.outcome == 'success' + with: + name: artifacts + path: ./out/artifacts diff --git a/third_party/jpeg-xl/.github/workflows/pull_request.yml b/third_party/jpeg-xl/.github/workflows/pull_request.yml new file mode 100644 index 0000000000..b1214e1061 --- /dev/null +++ b/third_party/jpeg-xl/.github/workflows/pull_request.yml @@ -0,0 +1,42 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Workflow to run pull-requests specific checks. + +name: PR +on: + pull_request: + types: [opened, reopened, synchronize] + +jobs: + # Checks that the AUTHORS files is updated with new contributors. + authors: + runs-on: [ubuntu-latest] + steps: + - name: Checkout the source + uses: actions/checkout@v2 + - name: Check AUTHORS file + run: + ./ci.sh authors + + format: + runs-on: [ubuntu-latest] + steps: + - name: Install build deps + run: | + sudo apt update + sudo apt install -y \ + clang-format \ + clang-format-7 \ + clang-format-8 \ + clang-format-9 \ + clang-format-10 \ + clang-format-11 \ + # + - name: Checkout the source + uses: actions/checkout@v2 + - name: clang-format + run: + ./ci.sh lint >&2 diff --git a/third_party/jpeg-xl/.github/workflows/release.yaml b/third_party/jpeg-xl/.github/workflows/release.yaml new file mode 100644 index 0000000000..4222266598 --- /dev/null +++ b/third_party/jpeg-xl/.github/workflows/release.yaml @@ -0,0 +1,378 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Workflow for building the release binaries. +# +# This workflow runs as a post-submit step, when pushing to main or the release +# branches (v*.*.x), and when creating a release in GitHub. +# +# In the GitHub release case, in addition to build the release binaries it also +# uploads the binaries to the given release automatically. + +name: Release build / deploy +on: + push: + branches: + - main + - v*.*.x + release: + types: [ published ] + +jobs: + ubuntu_static_x86_64: + name: Release linux x86_64 static + runs-on: [ubuntu-latest] + steps: + - name: Install build deps + run: | + sudo apt update + sudo apt install -y \ + asciidoc \ + clang \ + cmake \ + doxygen \ + libbrotli-dev \ + libgdk-pixbuf2.0-dev \ + libgif-dev \ + libgtest-dev \ + libgtk2.0-dev \ + libjpeg-dev \ + libopenexr-dev \ + libpng-dev \ + libwebp-dev \ + ninja-build \ + pkg-config \ + # + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + + - name: Checkout the source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 1 + + - name: Build + env: + SKIP_TEST: 1 + run: | + ./ci.sh release \ + -DJPEGXL_DEP_LICENSE_DIR=/usr/share/doc \ + -DJPEGXL_STATIC=ON \ + -DBUILD_TESTING=OFF \ + -DJPEGXL_ENABLE_VIEWERS=OFF \ + -DJPEGXL_ENABLE_PLUGINS=OFF \ + -DJPEGXL_ENABLE_OPENEXR=OFF \ + + - name: Package release tarball + run: | + cd build + tar -zcvf ${{ runner.workspace }}/release_file.tar.gz \ + LICENSE* tools/{cjxl,djxl,benchmark_xl} + ln -s ${{ runner.workspace }}/release_file.tar.gz \ + ${{ runner.workspace }}/jxl-linux-x86_64-static-${{ github.event.release.tag_name }}.tar.gz + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: jxl-linux-x86_64-static + path: ${{ runner.workspace }}/release_file.tar.gz + + - name: Upload binaries to release + if: github.event_name == 'release' + uses: AButler/upload-release-assets@v2.0 + with: + files: ${{ runner.workspace }}/jxl-linux-x86_64-static-${{ github.event.release.tag_name }}.tar.gz + repo-token: ${{ secrets.GITHUB_TOKEN }} + + + # Build .deb packages Ubuntu/Debian + release_ubuntu_pkg: + name: .deb packages / ${{ matrix.os }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: + - ubuntu:20.04 + - ubuntu:18.04 + - debian:buster + - debian:bullseye + - debian:bookworm + - debian:sid + + container: + image: ${{ matrix.os }} + + steps: + - name: Set env + shell: 'bash' + id: 'env' + run: | + artifact_name="jxl-debs-amd64-${matrix_os/:/-}" + echo ${artifact_name} + echo "::set-output name=artifact_name::${artifact_name}" + env: + matrix_os: ${{ matrix.os }} + + - name: Install build deps + run: | + apt update + DEBIAN_FRONTEND=noninteractive apt install -y \ + build-essential \ + devscripts \ + # + + - name: Install git (only 18.04) + if: matrix.os == 'ubuntu:18.04' + # Ubuntu 18.04 ships with git 2.17 but we need 2.18 or newer for + # actions/checkout@v2 to work + shell: 'bash' + run: | + apt install -y \ + libcurl4-openssl-dev \ + libexpat1-dev \ + libssl-dev \ + wget \ + zlib1g-dev \ + # + git_version="2.32.0" + wget -nv \ + "https://github.com/git/git/archive/refs/tags/v${git_version}.tar.gz" + tar -zxf "v${git_version}.tar.gz" + cd "git-${git_version}" + make prefix=/usr -j4 install + + - name: Install gcc-8 (only 18.04) + if: matrix.os == 'ubuntu:18.04' + # Compiler bug workaround: install and use gcc-8 + shell: 'bash' + run: | + apt install -y \ + gcc-8 \ + g++-8 \ + # + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 100 + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 100 + update-alternatives --set g++ /usr/bin/g++-8 + update-alternatives --set gcc /usr/bin/gcc-8 + + - name: Set git safe dir + run: | + export GIT_CEILING_DIRECTORIES=/__w # only work before git v2.35.2 + git config --global --add safe.directory /__w/libjxl/libjxl + + - name: Checkout the source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 1 + + - name: Stamp non-release versions + # Stamps the built package with the commit date as part of the version + # after the version number so newer release candidates can override older + # ones. + if: github.event_name != 'release' + shell: 'bash' + run: | + # Committer timestamp. + set -x + commit_timestamp=$(git show -s --format=%ct) + commit_datetime=$(date --utc "--date=@${commit_timestamp}" '+%Y%m%d%H%M%S') + commit_ref=$(git rev-parse --short HEAD) + sem_version=$(dpkg-parsechangelog --show-field Version) + sem_version="${sem_version%%-*}" + deb_version="${sem_version}~alpha${commit_datetime}-0+git${commit_ref}" + dch -M --distribution unstable -b --newversion "${deb_version}" \ + "Stamping build with version ${deb_version}" + + - name: Stamp release versions + # Mark the version as released + if: github.event_name == 'release' + shell: 'bash' + run: | + if head -n1 debian/changelog | grep UNRELEASED; then + dch -M --distribution unstable --release '' + fi + + - name: Install gtest (only 18.04) + if: matrix.os == 'ubuntu:18.04' + # In Ubuntu 18.04 no package installed the libgtest.a. libgtest-dev + # installs the source files only. + run: | + apt install -y libgtest-dev cmake + for prj in googletest googlemock; do + (cd /usr/src/googletest/${prj}/ && + cmake CMakeLists.txt -DCMAKE_INSTALL_PREFIX=/usr && + make all install) + done + # Remove libgmock-dev dependency in Ubuntu 18.04. It doesn't exist there. + sed '/libgmock-dev,/d' -i debian/control + + - name: Install gmock-dev (debian:sid) + # gtest-dev cmake depends on gmock-dev, but it is not installed by the + # package. + if: matrix.os == 'debian:sid' + run: | + apt install -y libgmock-dev + + - name: Remove libjxl-gimp-plugin package (only 18.04) + if: matrix.os == 'ubuntu:18.04' + run: | + # Gimp 2.8 is not supported. + sed -i '/Package: libjxl-gimp-plugin/,/^$/d' debian/control + + - name: Build hwy + run: | + apt build-dep -y ./third_party/highway + ./ci.sh debian_build highway + dpkg -i build/debs/libhwy-dev_*_amd64.deb + + - name: Build libjxl + run: | + apt build-dep -y . + ./ci.sh debian_build jpeg-xl + + - name: Stats + run: | + ./ci.sh debian_stats + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{ steps.env.outputs.artifact_name }} + path: | + build/debs/*jxl*.* + + - name: Package release tarball + if: github.event_name == 'release' + run: | + (cd build/debs/; find -maxdepth 1 -name '*jxl*.*') | \ + tar -zcvf release_file.tar.gz -C build/debs/ -T - + ln -s release_file.tar.gz \ + ${{ steps.env.outputs.artifact_name }}-${{ github.event.release.tag_name }}.tar.gz + + - name: Upload binaries to release + if: github.event_name == 'release' + uses: AButler/upload-release-assets@v2.0 + with: + files: ${{ steps.env.outputs.artifact_name }}-${{ github.event.release.tag_name }}.tar.gz + repo-token: ${{ secrets.GITHUB_TOKEN }} + + + windows_build: + name: Windows Build (vcpkg / ${{ matrix.triplet }}) + runs-on: [windows-2019] + strategy: + fail-fast: false + matrix: + include: + - triplet: x86-windows-static + arch: '-A Win32' + - triplet: x64-windows-static + arch: '-A x64' + + env: + VCPKG_VERSION: '2022.06.16.1' + VCPKG_ROOT: vcpkg + VCPKG_DISABLE_METRICS: 1 + + steps: + - name: Checkout the source + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 2 + + - uses: actions/cache@v2 + id: cache-vcpkg + with: + path: vcpkg + key: release-${{ runner.os }}-vcpkg-${{ env.VCPKG_VERSION }}-${{ matrix.triplet }} + + - name: Download vcpkg + if: steps.cache-vcpkg.outputs.cache-hit != 'true' + # wget doesn't seem to work under bash. + shell: 'powershell' + run: | + C:\msys64\usr\bin\wget.exe -nv ` + https://github.com/microsoft/vcpkg/archive/refs/tags/${{ env.VCPKG_VERSION }}.zip ` + -O vcpkg.zip + - name: Bootstrap vcpkg + if: steps.cache-vcpkg.outputs.cache-hit != 'true' + shell: 'bash' + run: | + set -x + unzip -q vcpkg.zip + rm -rf ${VCPKG_ROOT} + mv vcpkg-${VCPKG_VERSION} ${VCPKG_ROOT} + ${VCPKG_ROOT}/bootstrap-vcpkg.sh + + - name: Install libraries with vcpkg + shell: 'bash' + run: | + set -x + ${VCPKG_ROOT}/vcpkg --triplet ${{ matrix.triplet }} install \ + giflib \ + libjpeg-turbo \ + libpng \ + libwebp \ + # + + - name: Configure + shell: 'bash' + run: | + set -x + mkdir build + cmake -Bbuild -H. ${{ matrix.arch }} \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=`pwd`/prefix \ + -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \ + -DJPEGXL_ENABLE_OPENEXR=OFF \ + -DJPEGXL_ENABLE_PLUGINS=OFF \ + -DJPEGXL_ENABLE_TCMALLOC=OFF \ + -DJPEGXL_ENABLE_VIEWERS=OFF \ + -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} \ + # + - name: Build + shell: 'bash' + run: | + set -x + cmake --build build --config Release + - name: Install + shell: 'bash' + run: | + set -x + cmake --build build --config Release --target install + for pkg in giflib libjpeg-turbo libpng libwebp zlib; do + cp vcpkg/installed/${{matrix.triplet}}/share/${pkg}/copyright \ + prefix/bin/LICENSE.${pkg} + done + cp third_party/sjpeg/COPYING prefix/bin/LICENSE.sjpeg + cp third_party/skcms/LICENSE prefix/bin/LICENSE.skcms + cp third_party/highway/LICENSE prefix/bin/LICENSE.highway + cp third_party/brotli/LICENSE prefix/bin/LICENSE.brotli + cp LICENSE prefix/bin/LICENSE.libjxl + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: jxl-${{matrix.triplet}} + path: | + prefix/bin/* + + - name: Package release zip + if: github.event_name == 'release' + shell: 'powershell' + run: | + Compress-Archive -Path prefix\bin\* ` + -DestinationPath jxl-${{matrix.triplet}}.zip + + - name: Upload binaries to release + if: github.event_name == 'release' + uses: AButler/upload-release-assets@v2.0 + with: + files: jxl-${{matrix.triplet}}.zip + repo-token: ${{ secrets.GITHUB_TOKEN }} |