diff options
Diffstat (limited to 'third_party/jpeg-xl/docker/scripts/jpegxl_builder.sh')
| -rwxr-xr-x | third_party/jpeg-xl/docker/scripts/jpegxl_builder.sh | 516 |
1 files changed, 516 insertions, 0 deletions
diff --git a/third_party/jpeg-xl/docker/scripts/jpegxl_builder.sh b/third_party/jpeg-xl/docker/scripts/jpegxl_builder.sh new file mode 100755 index 0000000000..949c811eae --- /dev/null +++ b/third_party/jpeg-xl/docker/scripts/jpegxl_builder.sh @@ -0,0 +1,516 @@ +#!/usr/bin/env bash +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Main entry point for all the Dockerfile for jpegxl-builder. This centralized +# file helps sharing code and configuration between Dockerfiles. + +set -eux + +MYDIR=$(dirname $(realpath "$0")) + +# libjpeg-turbo. +JPEG_TURBO_RELEASE="2.0.4" +JPEG_TURBO_URL="https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${JPEG_TURBO_RELEASE}.tar.gz" +JPEG_TURBO_SHA256="7777c3c19762940cff42b3ba4d7cd5c52d1671b39a79532050c85efb99079064" + +# zlib (dependency of libpng) +ZLIB_RELEASE="1.2.11" +ZLIB_URL="https://www.zlib.net/zlib-${ZLIB_RELEASE}.tar.gz" +ZLIB_SHA256="c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1" +# The name in the .pc and the .dll generated don't match in zlib for Windows +# because they use different .dll names in Windows. We avoid that by defining +# UNIX=1. We also install all the .dll files to ${prefix}/lib instead of the +# default ${prefix}/bin. +ZLIB_FLAGS='-DUNIX=1 -DINSTALL_PKGCONFIG_DIR=/${CMAKE_INSTALL_PREFIX}/lib/pkgconfig -DINSTALL_BIN_DIR=/${CMAKE_INSTALL_PREFIX}/lib' + +# libpng +LIBPNG_RELEASE="1.6.37" +LIBPNG_URL="https://github.com/glennrp/libpng/archive/v${LIBPNG_RELEASE}.tar.gz" +LIBPNG_SHA256="ca74a0dace179a8422187671aee97dd3892b53e168627145271cad5b5ac81307" + +# giflib +GIFLIB_RELEASE="5.2.1" +GIFLIB_URL="https://netcologne.dl.sourceforge.net/project/giflib/giflib-${GIFLIB_RELEASE}.tar.gz" +GIFLIB_SHA256="31da5562f44c5f15d63340a09a4fd62b48c45620cd302f77a6d9acf0077879bd" + +# A patch needed to compile GIFLIB in mingw. +GIFLIB_PATCH_URL="https://github.com/msys2/MINGW-packages/raw/3afde38fcee7b3ba2cafd97d76cca8f06934504f/mingw-w64-giflib/001-mingw-build.patch" +GIFLIB_PATCH_SHA256="2b2262ddea87fc07be82e10aeb39eb699239f883c899aa18a16e4d4e40af8ec8" + +# webp +WEBP_RELEASE="1.0.2" +WEBP_URL="https://codeload.github.com/webmproject/libwebp/tar.gz/v${WEBP_RELEASE}" +WEBP_SHA256="347cf85ddc3497832b5fa9eee62164a37b249c83adae0ba583093e039bf4881f" + +# Google benchmark +BENCHMARK_RELEASE="1.5.2" +BENCHMARK_URL="https://github.com/google/benchmark/archive/v${BENCHMARK_RELEASE}.tar.gz" +BENCHMARK_SHA256="dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c" +BENCHMARK_FLAGS="-DGOOGLETEST_PATH=${MYDIR}/../../third_party/googletest" +# attribute(format(__MINGW_PRINTF_FORMAT, ...)) doesn't work in our +# environment, so we disable the warning. +BENCHMARK_FLAGS="-DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_TESTING=OFF \ + -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON" + +# V8 +V8_VERSION="9.3.22" + +# Temporary files cleanup hooks. +CLEANUP_FILES=() +cleanup() { + if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then + rm -fr "${CLEANUP_FILES[@]}" + fi +} +trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT + +# List of Ubuntu arch names supported by the builder (such as "i386"). +LIST_ARCHS=( + amd64 + i386 + arm64 + armhf +) + +# List of target triplets supported by the builder. +LIST_TARGETS=( + x86_64-linux-gnu + i686-linux-gnu + arm-linux-gnueabihf + aarch64-linux-gnu +) +LIST_MINGW_TARGETS=( + i686-w64-mingw32 + x86_64-w64-mingw32 +) +LIST_WASM_TARGETS=( + wasm32 +) + +# Setup the apt repositories and supported architectures. +setup_apt() { + apt-get update -y + apt-get install -y curl gnupg ca-certificates + + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1E9377A2BA9EF27F + + # node sources. + cat >/etc/apt/sources.list.d/nodesource.list <<EOF + deb https://deb.nodesource.com/node_14.x bionic main + deb-src https://deb.nodesource.com/node_14.x bionic main +EOF + curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - + + local port_list=() + local main_list=() + local ubarch + for ubarch in "${LIST_ARCHS[@]}"; do + if [[ "${ubarch}" != "amd64" && "${ubarch}" != "i386" ]]; then + # other archs are not part of the main mirrors, but available in + # ports.ubuntu.com. + port_list+=("${ubarch}") + else + main_list+=("${ubarch}") + fi + # Add the arch to the system. + if [[ "${ubarch}" != "amd64" ]]; then + dpkg --add-architecture "${ubarch}" + fi + done + + # Update the sources.list with the split of supported architectures. + local bkplist="/etc/apt/sources.list.bkp" + [[ -e "${bkplist}" ]] || \ + mv /etc/apt/sources.list "${bkplist}" + + local newlist="/etc/apt/sources.list.tmp" + rm -f "${newlist}" + port_list=$(echo "${port_list[@]}" | tr ' ' ,) + if [[ -n "${port_list}" ]]; then + local port_url="http://ports.ubuntu.com/ubuntu-ports/" + grep -v -E '^#' "${bkplist}" | + sed -E "s;^deb (http[^ ]+) (.*)\$;deb [arch=${port_list}] ${port_url} \\2;" \ + >>"${newlist}" + fi + + main_list=$(echo "${main_list[@]}" | tr ' ' ,) + grep -v -E '^#' "${bkplist}" | + sed -E "s;^deb (http[^ ]+) (.*)\$;deb [arch=${main_list}] \\1 \\2\ndeb-src [arch=${main_list}] \\1 \\2;" \ + >>"${newlist}" + mv "${newlist}" /etc/apt/sources.list +} + +install_pkgs() { + packages=( + # Native compilers (minimum for SIMD is clang-7) + clang-7 clang-format-7 clang-tidy-7 + + # TODO: Consider adding clang-8 to every builder: + # clang-8 clang-format-8 clang-tidy-8 + + # For cross-compiling to Windows with mingw. + mingw-w64 + wine64 + wine-binfmt + + # Native tools. + bsdmainutils + cmake + extra-cmake-modules + git + llvm + nasm + ninja-build + parallel + pkg-config + + # For compiling / testing JNI wrapper. JDK8 is almost 2x smaller than JDK11 + # openjdk-8-jdk-headless would be 50MB smaller, unfortunately, CMake + # does mistakenly thinks it does not contain JNI feature. + openjdk-8-jdk + + # These are used by the ./ci.sh lint in the native builder. + clang-format-7 + clang-format-8 + + # For coverage builds + gcovr + + # For compiling giflib documentation. + xmlto + + # Common libraries. + libstdc++-8-dev + + # We don't use tcmalloc on archs other than amd64. This installs + # libgoogle-perftools4:amd64. + google-perftools + + # NodeJS for running WASM tests + nodejs + + # To generate API documentation. + doxygen + + # Freezes version that builds (passes tests). Newer version + # (2.30-21ubuntu1~18.04.4) claims to fix "On Intel Skylake + # (-march=native) generated avx512 instruction can be wrong", + # but newly added tests does not pass. Perhaps the problem is + # that mingw package is not updated. + binutils-source=2.30-15ubuntu1 + ) + + # Install packages that are arch-dependent. + local ubarch + for ubarch in "${LIST_ARCHS[@]}"; do + packages+=( + # Library dependencies. These normally depend on the target architecture + # we are compiling for and can't usually be installed for multiple + # architectures at the same time. + libgif7:"${ubarch}" + libjpeg-dev:"${ubarch}" + libpng-dev:"${ubarch}" + + libstdc++-8-dev:"${ubarch}" + + # For OpenEXR: + libilmbase12:"${ubarch}" + libopenexr22:"${ubarch}" + + # TCMalloc dependency + libunwind-dev:"${ubarch}" + + # Cross-compiling tools per arch. + libc6-dev-"${ubarch}"-cross + libstdc++-8-dev-"${ubarch}"-cross + ) + done + + local target + for target in "${LIST_TARGETS[@]}"; do + # Per target cross-compiling tools. + if [[ "${target}" != "x86_64-linux-gnu" ]]; then + packages+=( + binutils-"${target}" + gcc-"${target}" + ) + fi + done + + # Install all the manual packages via "apt install" for the main arch. These + # will be installed for other archs via manual download and unpack. + apt install -y "${packages[@]}" "${UNPACK_PKGS[@]}" +} + +# binutils <2.32 need a patch. +install_binutils() { + local workdir=$(mktemp -d --suffix=_install) + CLEANUP_FILES+=("${workdir}") + pushd "${workdir}" + apt source binutils-mingw-w64 + apt -y build-dep binutils-mingw-w64 + cd binutils-mingw-w64-8ubuntu1 + cp "${MYDIR}/binutils_align_fix.patch" debian/patches + echo binutils_align_fix.patch >> debian/patches/series + dpkg-buildpackage -b + cd .. + dpkg -i *deb + popd +} + +# Install a library from the source code for multiple targets. +# Usage: install_from_source <tar_url> <sha256> <target> [<target...>] +install_from_source() { + local package="$1" + shift + + local url + eval "url=\${${package}_URL}" + local sha256 + eval "sha256=\${${package}_SHA256}" + # Optional package flags + local pkgflags + eval "pkgflags=\${${package}_FLAGS:-}" + + local workdir=$(mktemp -d --suffix=_install) + CLEANUP_FILES+=("${workdir}") + + local tarfile="${workdir}"/$(basename "${url}") + curl -L --output "${tarfile}" "${url}" + if ! echo "${sha256} ${tarfile}" | sha256sum -c --status -; then + echo "SHA256 mismatch for ${url}: expected ${sha256} but found:" + sha256sum "${tarfile}" + exit 1 + fi + + local target + for target in "$@"; do + echo "Installing ${package} for target ${target} from ${url}" + + local srcdir="${workdir}/source-${target}" + mkdir -p "${srcdir}" + tar -zxf "${tarfile}" -C "${srcdir}" --strip-components=1 + + local prefix="/usr" + if [[ "${target}" != "x86_64-linux-gnu" ]]; then + prefix="/usr/${target}" + fi + + # Apply patches to buildfiles. + if [[ "${package}" == "GIFLIB" && "${target}" == *mingw32 ]]; then + # GIFLIB Makefile has several problems so we need to fix them here. We are + # using a patch from MSYS2 that already fixes the compilation for mingw. + local make_patch="${srcdir}/libgif.patch" + curl -L "${GIFLIB_PATCH_URL}" -o "${make_patch}" + echo "${GIFLIB_PATCH_SHA256} ${make_patch}" | sha256sum -c --status - + patch "${srcdir}/Makefile" < "${make_patch}" + elif [[ "${package}" == "LIBPNG" && "${target}" == wasm* ]]; then + # Cut the dependency to libm; there is pull request to fix it, so this + # might not be needed in the future. + sed -i 's/APPLE/EMSCRIPTEN/g' "${srcdir}/CMakeLists.txt" + fi + + local cmake_args=() + local export_args=("CC=clang-7" "CXX=clang++-7") + local cmake="cmake" + local make="make" + local system_name="Linux" + if [[ "${target}" == *mingw32 ]]; then + system_name="Windows" + # When compiling with clang, CMake doesn't detect that we are using mingw. + cmake_args+=( + -DMINGW=1 + # Googletest needs this when cross-compiling to windows + -DCMAKE_CROSSCOMPILING=1 + -DHAVE_STD_REGEX=0 + -DHAVE_POSIX_REGEX=0 + -DHAVE_GNU_POSIX_REGEX=0 + ) + local windres=$(which ${target}-windres || true) + if [[ -n "${windres}" ]]; then + cmake_args+=(-DCMAKE_RC_COMPILER="${windres}") + fi + fi + if [[ "${target}" == wasm* ]]; then + system_name="WASM" + cmake="emcmake cmake" + make="emmake make" + export_args=() + cmake_args+=( + -DCMAKE_FIND_ROOT_PATH="${prefix}" + -DCMAKE_PREFIX_PATH="${prefix}" + ) + # Static and shared library link to the same file -> race condition. + nproc=1 + else + nproc=`nproc --all` + fi + cmake_args+=(-DCMAKE_SYSTEM_NAME="${system_name}") + + if [[ "${target}" != "x86_64-linux-gnu" ]]; then + # Cross-compiling. + cmake_args+=( + -DCMAKE_C_COMPILER_TARGET="${target}" + -DCMAKE_CXX_COMPILER_TARGET="${target}" + -DCMAKE_SYSTEM_PROCESSOR="${target%%-*}" + ) + fi + + if [[ -e "${srcdir}/CMakeLists.txt" ]]; then + # Most packages use cmake for building which is easier to configure for + # cross-compiling. + if [[ "${package}" == "JPEG_TURBO" && "${target}" == wasm* ]]; then + # JT erroneously detects WASM CPU as i386 and tries to use asm. + # Wasm/Emscripten support for dynamic linking is incomplete; disable + # to avoid CMake warning. + cmake_args+=(-DWITH_SIMD=0 -DENABLE_SHARED=OFF) + fi + ( + cd "${srcdir}" + export ${export_args[@]} + ${cmake} \ + -DCMAKE_INSTALL_PREFIX="${prefix}" \ + "${cmake_args[@]}" ${pkgflags} + ${make} -j${nproc} + ${make} install + ) + elif [[ "${package}" == "GIFLIB" ]]; then + # GIFLIB doesn't yet have a cmake build system. There is a pull + # request in giflib for adding CMakeLists.txt so this might not be + # needed in the future. + ( + cd "${srcdir}" + local giflib_make_flags=( + CFLAGS="-O2 --target=${target} -std=gnu99" + PREFIX="${prefix}" + ) + if [[ "${target}" != wasm* ]]; then + giflib_make_flags+=(CC=clang-7) + fi + # giflib make dependencies are not properly set up so parallel building + # doesn't work for everything. + ${make} -j${nproc} libgif.a "${giflib_make_flags[@]}" + ${make} -j${nproc} all "${giflib_make_flags[@]}" + ${make} install "${giflib_make_flags[@]}" + ) + else + echo "Don't know how to install ${package}" + exit 1 + fi + + # CMake mistakenly uses ".so" libraries and EMCC fails to link properly. + if [[ "${target}" == wasm* ]]; then + rm -f "${prefix}/lib"/*.so* + fi + done +} + +# Packages that are manually unpacked for each architecture. +UNPACK_PKGS=( + libgif-dev + libclang-common-7-dev + + # For OpenEXR: + libilmbase-dev + libopenexr-dev + + # TCMalloc + libgoogle-perftools-dev + libtcmalloc-minimal4 + libgoogle-perftools4 +) + +# Main script entry point. +main() { + cd "${MYDIR}" + + # Configure the repositories with the sources for multi-arch cross + # compilation. + setup_apt + apt-get update -y + apt-get dist-upgrade -y + + install_pkgs + install_binutils + apt clean + + # Remove prebuilt Java classes cache. + rm /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/classes.jsa + + # Manually extract packages for the target arch that can't install it directly + # at the same time as the native ones. + local ubarch + for ubarch in "${LIST_ARCHS[@]}"; do + if [[ "${ubarch}" != "amd64" ]]; then + local pkg + for pkg in "${UNPACK_PKGS[@]}"; do + apt download "${pkg}":"${ubarch}" + dpkg -x "${pkg}"_*_"${ubarch}".deb / + done + fi + done + # TODO: Add clang from the llvm repos. This is problematic since we are + # installing libclang-common-7-dev:"${ubarch}" from the ubuntu ports repos + # which is not available in the llvm repos so it might have a different + # version than the ubuntu ones. + + # Remove the win32 libgcc version. The gcc-mingw-w64-x86-64 (and i686) + # packages install two libgcc versions: + # /usr/lib/gcc/x86_64-w64-mingw32/7.3-posix + # /usr/lib/gcc/x86_64-w64-mingw32/7.3-win32 + # (exact libgcc version number depends on the package version). + # + # Clang will pick the best libgcc, sorting by version, but it doesn't + # seem to be a way to specify one or the other one, except by passing + # -nostdlib and setting all the include paths from the command line. + # To check which one is being used you can run: + # clang++-7 --target=x86_64-w64-mingw32 -v -print-libgcc-file-name + # We need to use the "posix" versions for thread support, so here we + # just remove the other one. + local target + for target in "${LIST_MINGW_TARGETS[@]}"; do + update-alternatives --set "${target}-gcc" $(which "${target}-gcc-posix") + local gcc_win32_path=$("${target}-cpp-win32" -print-libgcc-file-name) + rm -rf $(dirname "${gcc_win32_path}") + done + + # TODO: Add msan for the target when cross-compiling. This only installs it + # for amd64. + ./msan_install.sh + + # Build and install qemu user-linux targets. + ./qemu_install.sh + + # Install emscripten SDK. + ./emsdk_install.sh + + # Setup environment for building WASM libraries from sources. + source /opt/emsdk/emsdk_env.sh + + # Install some dependency libraries manually for the different targets. + + install_from_source JPEG_TURBO "${LIST_MINGW_TARGETS[@]}" "${LIST_WASM_TARGETS[@]}" + install_from_source ZLIB "${LIST_MINGW_TARGETS[@]}" "${LIST_WASM_TARGETS[@]}" + install_from_source LIBPNG "${LIST_MINGW_TARGETS[@]}" "${LIST_WASM_TARGETS[@]}" + install_from_source GIFLIB "${LIST_MINGW_TARGETS[@]}" "${LIST_WASM_TARGETS[@]}" + # webp in Ubuntu is relatively old so we install it from source for everybody. + install_from_source WEBP "${LIST_TARGETS[@]}" "${LIST_MINGW_TARGETS[@]}" + + install_from_source BENCHMARK "${LIST_TARGETS[@]}" "${LIST_MINGW_TARGETS[@]}" + + # Install v8. v8 has better WASM SIMD support than NodeJS 14 (LTS). + # First we need the installer to install v8. + npm install jsvu -g + # install specific version; + HOME=/opt jsvu --os=linux64 "v8@${V8_VERSION}" + ln -s "/opt/.jsvu/v8-${V8_VERSION}" "/opt/.jsvu/v8" + + # Cleanup. + find /var/lib/apt/lists/ -mindepth 1 -delete +} + +main "$@" |