diff options
Diffstat (limited to 'third_party/jpeg-xl/docker')
| -rw-r--r-- | third_party/jpeg-xl/docker/Dockerfile.jpegxl-builder | 21 | ||||
| -rw-r--r-- | third_party/jpeg-xl/docker/Dockerfile.jpegxl-builder-run-aarch64 | 37 | ||||
| -rw-r--r-- | third_party/jpeg-xl/docker/README.md | 7 | ||||
| -rwxr-xr-x | third_party/jpeg-xl/docker/build.sh | 83 | ||||
| -rw-r--r-- | third_party/jpeg-xl/docker/scripts/99_norecommends | 1 | ||||
| -rw-r--r-- | third_party/jpeg-xl/docker/scripts/binutils_align_fix.patch | 28 | ||||
| -rwxr-xr-x | third_party/jpeg-xl/docker/scripts/emsdk_install.sh | 37 | ||||
| -rwxr-xr-x | third_party/jpeg-xl/docker/scripts/jpegxl_builder.sh | 518 | ||||
| -rwxr-xr-x | third_party/jpeg-xl/docker/scripts/msan_install.sh | 131 | ||||
| -rwxr-xr-x | third_party/jpeg-xl/docker/scripts/qemu_install.sh | 83 |
10 files changed, 946 insertions, 0 deletions
diff --git a/third_party/jpeg-xl/docker/Dockerfile.jpegxl-builder b/third_party/jpeg-xl/docker/Dockerfile.jpegxl-builder new file mode 100644 index 0000000000..16e0077eea --- /dev/null +++ b/third_party/jpeg-xl/docker/Dockerfile.jpegxl-builder @@ -0,0 +1,21 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Build an Ubuntu-based docker image with the installed software needed to +# develop and test JPEG XL. + +FROM ubuntu:bionic + +# Set a prompt for when using it locally. +ENV PS1="\[\033[01;33m\]\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ " + +COPY scripts/99_norecommends /etc/apt/apt.conf.d/99_norecommends + +COPY scripts /jpegxl_scripts + +ARG DEBIAN_FRONTEND=noninteractive + +RUN /jpegxl_scripts/jpegxl_builder.sh && \ + rm -rf /jpegxl_scripts diff --git a/third_party/jpeg-xl/docker/Dockerfile.jpegxl-builder-run-aarch64 b/third_party/jpeg-xl/docker/Dockerfile.jpegxl-builder-run-aarch64 new file mode 100644 index 0000000000..a9f38a4015 --- /dev/null +++ b/third_party/jpeg-xl/docker/Dockerfile.jpegxl-builder-run-aarch64 @@ -0,0 +1,37 @@ +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Build an Ubuntu-based docker image for aarch64 with the installed software +# needed to run JPEG XL. This is only useful when running on actual aarch64 +# hardware. + +FROM arm64v8/ubuntu:bionic + +COPY scripts/99_norecommends /etc/apt/apt.conf.d/99_norecommends + +# Set a prompt for when using it locally. +ENV PS1="\[\033[01;33m\]\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ " + +ARG DEBIAN_FRONTEND=noninteractive + +RUN set -ex; \ + apt-get update -y; \ + apt-get install -y \ + bsdmainutils \ + cmake \ + curl \ + ca-certificates \ + extra-cmake-modules \ + git \ + imagemagick \ + libjpeg8 \ + libgif7 \ + libgoogle-perftools4 \ + libopenexr22 \ + libpng16-16 \ + libqt5x11extras5 \ + libsdl2-2.0-0 \ + parallel; \ + rm -rf /var/lib/apt/lists/*; diff --git a/third_party/jpeg-xl/docker/README.md b/third_party/jpeg-xl/docker/README.md new file mode 100644 index 0000000000..874df1cb80 --- /dev/null +++ b/third_party/jpeg-xl/docker/README.md @@ -0,0 +1,7 @@ +### Docker container infrastructure for JPEG XL + +This directory contains the requirements to build a docker image for the +JPEG XL project builder. + +Docker images need to be created and upload manually. See ./build.sh for +details. diff --git a/third_party/jpeg-xl/docker/build.sh b/third_party/jpeg-xl/docker/build.sh new file mode 100755 index 0000000000..3d4727f6a4 --- /dev/null +++ b/third_party/jpeg-xl/docker/build.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -eu + +MYDIR=$(dirname $(realpath "$0")) + +declare -a TARGETS + +load_targets() { + # Built-in OSX "find" does not support "-m". + FIND=$(which "gfind" || which "find") + for f in $(${FIND} -maxdepth 1 -name 'Dockerfile.*' | sort); do + local target="${f#*Dockerfile.}" + TARGETS+=("${target}") + done +} + +usage() { + cat >&2 <<EOF +Use: $1 [targets] + +Available targets: + * all +EOF + for target in "${TARGETS[@]}"; do + echo " * ${target}" >&2 + done +} + +build_target() { + local target="$1" + + local dockerfile="${MYDIR}/Dockerfile.${target}" + # JPEG XL builder images are stored in the gcr.io/jpegxl project. + local tag="gcr.io/jpegxl/${target}" + + echo "Building ${target}" + if ! sudo docker build --no-cache -t "${tag}" -f "${dockerfile}" "${MYDIR}" \ + >"${target}.log" 2>&1; then + echo "${target} failed. See ${target}.log" >&2 + else + echo "Done, to upload image run:" >&2 + echo " sudo docker push ${tag}" + if [[ "${JPEGXL_PUSH:-}" == "1" ]]; then + echo "sudo docker push ${tag}" >&2 + sudo docker push "${tag}" + # The RepoDigest is only created after it is pushed. + local fulltag=$(sudo docker inspect --format="{{.RepoDigests}}" "${tag}") + fulltag="${fulltag#[}" + fulltag="${fulltag%]}" + echo "Updating .gitlab-ci.yml to ${fulltag}" >&2 + sed -E "s;${tag}@sha256:[0-9a-f]+;${fulltag};" \ + -i "${MYDIR}/../.gitlab-ci.yml" + fi + fi +} + +main() { + cd "${MYDIR}" + local target="${1:-}" + + load_targets + if [[ -z "${target}" ]]; then + usage $0 + exit 1 + fi + + if [[ "${target}" == "all" ]]; then + for target in "${TARGETS[@]}"; do + build_target "${target}" + done + else + for target in "$@"; do + build_target "${target}" + done + fi +} + +main "$@" diff --git a/third_party/jpeg-xl/docker/scripts/99_norecommends b/third_party/jpeg-xl/docker/scripts/99_norecommends new file mode 100644 index 0000000000..96d672811d --- /dev/null +++ b/third_party/jpeg-xl/docker/scripts/99_norecommends @@ -0,0 +1 @@ +APT::Install-Recommends "false"; diff --git a/third_party/jpeg-xl/docker/scripts/binutils_align_fix.patch b/third_party/jpeg-xl/docker/scripts/binutils_align_fix.patch new file mode 100644 index 0000000000..6066252db8 --- /dev/null +++ b/third_party/jpeg-xl/docker/scripts/binutils_align_fix.patch @@ -0,0 +1,28 @@ +Description: fix lack of alignment in relocations (crashes on mingw) +See https://sourceware.org/git/?p=binutils-gdb.git;a=patch;h=73af69e74974eaa155eec89867e3ccc77ab39f6d +From: Marc <marc@groundctl.com> +Date: Fri, 9 Nov 2018 11:13:50 +0000 +Subject: [PATCH] Allow for compilers that do not produce aligned .rdat + sections in PE format files. + +--- a/upstream/ld/scripttempl/pe.sc 2020-05-12 18:45:12.000000000 +0200 ++++ b/upstream/ld/scripttempl/pe.sc 2020-05-12 18:47:12.000000000 +0200 +@@ -143,6 +143,7 @@ + .rdata ${RELOCATING+BLOCK(__section_alignment__)} : + { + ${R_RDATA} ++ . = ALIGN(4); + ${RELOCATING+__rt_psrelocs_start = .;} + ${RELOCATING+KEEP(*(.rdata_runtime_pseudo_reloc))} + ${RELOCATING+__rt_psrelocs_end = .;} +--- a/upstream/ld/scripttempl/pep.sc 2020-05-12 18:45:19.000000000 +0200 ++++ b/upstream/ld/scripttempl/pep.sc 2020-05-12 18:47:18.000000000 +0200 +@@ -143,6 +143,7 @@ + .rdata ${RELOCATING+BLOCK(__section_alignment__)} : + { + ${R_RDATA} ++ . = ALIGN(4); + ${RELOCATING+__rt_psrelocs_start = .;} + ${RELOCATING+KEEP(*(.rdata_runtime_pseudo_reloc))} + ${RELOCATING+__rt_psrelocs_end = .;} + diff --git a/third_party/jpeg-xl/docker/scripts/emsdk_install.sh b/third_party/jpeg-xl/docker/scripts/emsdk_install.sh new file mode 100755 index 0000000000..6cf225a9d9 --- /dev/null +++ b/third_party/jpeg-xl/docker/scripts/emsdk_install.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +EMSDK_URL="https://github.com/emscripten-core/emsdk/archive/main.tar.gz" +EMSDK_DIR="/opt/emsdk" + +EMSDK_RELEASE="2.0.23" + +set -eu -x + +# Temporary files cleanup hooks. +CLEANUP_FILES=() +cleanup() { + if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then + rm -fr "${CLEANUP_FILES[@]}" + fi +} +trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT + +main() { + local workdir=$(mktemp -d --suffix=emsdk) + CLEANUP_FILES+=("${workdir}") + + local emsdktar="${workdir}/emsdk.tar.gz" + curl --output "${emsdktar}" "${EMSDK_URL}" --location + mkdir -p "${EMSDK_DIR}" + tar -zxf "${emsdktar}" -C "${EMSDK_DIR}" --strip-components=1 + + cd "${EMSDK_DIR}" + ./emsdk install --shallow "${EMSDK_RELEASE}" + ./emsdk activate --embedded "${EMSDK_RELEASE}" +} + +main "$@" diff --git a/third_party/jpeg-xl/docker/scripts/jpegxl_builder.sh b/third_party/jpeg-xl/docker/scripts/jpegxl_builder.sh new file mode 100755 index 0000000000..bf9f19d4ed --- /dev/null +++ b/third_party/jpeg-xl/docker/scripts/jpegxl_builder.sh @@ -0,0 +1,518 @@ +#!/usr/bin/env bash +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Main entry point for all the Dockerfile for jpegxl-builder. This centralized +# file helps sharing code and configuration between Dockerfiles. + +set -eux + +MYDIR=$(dirname $(realpath "$0")) + +# libjpeg-turbo. +JPEG_TURBO_RELEASE="2.0.4" +JPEG_TURBO_URL="https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${JPEG_TURBO_RELEASE}.tar.gz" +JPEG_TURBO_SHA256="7777c3c19762940cff42b3ba4d7cd5c52d1671b39a79532050c85efb99079064" + +# zlib (dependency of libpng) +ZLIB_RELEASE="1.2.11" +ZLIB_URL="https://www.zlib.net/zlib-${ZLIB_RELEASE}.tar.gz" +ZLIB_SHA256="c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1" +# The name in the .pc and the .dll generated don't match in zlib for Windows +# because they use different .dll names in Windows. We avoid that by defining +# UNIX=1. We also install all the .dll files to ${prefix}/lib instead of the +# default ${prefix}/bin. +ZLIB_FLAGS='-DUNIX=1 -DINSTALL_PKGCONFIG_DIR=/${CMAKE_INSTALL_PREFIX}/lib/pkgconfig -DINSTALL_BIN_DIR=/${CMAKE_INSTALL_PREFIX}/lib' + +# libpng +LIBPNG_RELEASE="1.6.37" +LIBPNG_URL="https://github.com/glennrp/libpng/archive/v${LIBPNG_RELEASE}.tar.gz" +LIBPNG_SHA256="ca74a0dace179a8422187671aee97dd3892b53e168627145271cad5b5ac81307" + +# giflib +GIFLIB_RELEASE="5.2.1" +GIFLIB_URL="https://netcologne.dl.sourceforge.net/project/giflib/giflib-${GIFLIB_RELEASE}.tar.gz" +GIFLIB_SHA256="31da5562f44c5f15d63340a09a4fd62b48c45620cd302f77a6d9acf0077879bd" + +# A patch needed to compile GIFLIB in mingw. +GIFLIB_PATCH_URL="https://github.com/msys2/MINGW-packages/raw/3afde38fcee7b3ba2cafd97d76cca8f06934504f/mingw-w64-giflib/001-mingw-build.patch" +GIFLIB_PATCH_SHA256="2b2262ddea87fc07be82e10aeb39eb699239f883c899aa18a16e4d4e40af8ec8" + +# webp +WEBP_RELEASE="1.0.2" +WEBP_URL="https://codeload.github.com/webmproject/libwebp/tar.gz/v${WEBP_RELEASE}" +WEBP_SHA256="347cf85ddc3497832b5fa9eee62164a37b249c83adae0ba583093e039bf4881f" + +# Google benchmark +BENCHMARK_RELEASE="1.5.2" +BENCHMARK_URL="https://github.com/google/benchmark/archive/v${BENCHMARK_RELEASE}.tar.gz" +BENCHMARK_SHA256="dccbdab796baa1043f04982147e67bb6e118fe610da2c65f88912d73987e700c" +BENCHMARK_FLAGS="-DGOOGLETEST_PATH=${MYDIR}/../../third_party/googletest" +# attribute(format(__MINGW_PRINTF_FORMAT, ...)) doesn't work in our +# environment, so we disable the warning. +BENCHMARK_FLAGS="-DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_TESTING=OFF \ + -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON" + +# V8 +V8_VERSION="9.3.22" + +# Temporary files cleanup hooks. +CLEANUP_FILES=() +cleanup() { + if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then + rm -fr "${CLEANUP_FILES[@]}" + fi +} +trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT + +# List of Ubuntu arch names supported by the builder (such as "i386"). +LIST_ARCHS=( + amd64 + i386 + arm64 + armhf +) + +# List of target triplets supported by the builder. +LIST_TARGETS=( + x86_64-linux-gnu + i686-linux-gnu + arm-linux-gnueabihf + aarch64-linux-gnu +) +LIST_MINGW_TARGETS=( + i686-w64-mingw32 + x86_64-w64-mingw32 +) +LIST_WASM_TARGETS=( + wasm32 +) + +# Setup the apt repositories and supported architectures. +setup_apt() { + apt-get update -y + apt-get install -y curl gnupg ca-certificates + + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1E9377A2BA9EF27F + + # node sources. + cat >/etc/apt/sources.list.d/nodesource.list <<EOF + deb https://deb.nodesource.com/node_14.x bionic main + deb-src https://deb.nodesource.com/node_14.x bionic main +EOF + curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - + + local port_list=() + local main_list=() + local ubarch + for ubarch in "${LIST_ARCHS[@]}"; do + if [[ "${ubarch}" != "amd64" && "${ubarch}" != "i386" ]]; then + # other archs are not part of the main mirrors, but available in + # ports.ubuntu.com. + port_list+=("${ubarch}") + else + main_list+=("${ubarch}") + fi + # Add the arch to the system. + if [[ "${ubarch}" != "amd64" ]]; then + dpkg --add-architecture "${ubarch}" + fi + done + + # Update the sources.list with the split of supported architectures. + local bkplist="/etc/apt/sources.list.bkp" + [[ -e "${bkplist}" ]] || \ + mv /etc/apt/sources.list "${bkplist}" + + local newlist="/etc/apt/sources.list.tmp" + rm -f "${newlist}" + port_list=$(echo "${port_list[@]}" | tr ' ' ,) + if [[ -n "${port_list}" ]]; then + local port_url="http://ports.ubuntu.com/ubuntu-ports/" + grep -v -E '^#' "${bkplist}" | + sed -E "s;^deb (http[^ ]+) (.*)\$;deb [arch=${port_list}] ${port_url} \\2;" \ + >>"${newlist}" + fi + + main_list=$(echo "${main_list[@]}" | tr ' ' ,) + grep -v -E '^#' "${bkplist}" | + sed -E "s;^deb (http[^ ]+) (.*)\$;deb [arch=${main_list}] \\1 \\2\ndeb-src [arch=${main_list}] \\1 \\2;" \ + >>"${newlist}" + mv "${newlist}" /etc/apt/sources.list +} + +install_pkgs() { + packages=( + # Native compilers (minimum for SIMD is clang-7) + clang-7 clang-format-7 clang-tidy-7 + + # TODO: Consider adding clang-8 to every builder: + # clang-8 clang-format-8 clang-tidy-8 + + # For cross-compiling to Windows with mingw. + mingw-w64 + wine64 + wine-binfmt + + # Native tools. + bsdmainutils + cmake + extra-cmake-modules + git + llvm + nasm + ninja-build + parallel + pkg-config + + # For compiling / testing JNI wrapper. JDK8 is almost 2x smaller than JDK11 + # openjdk-8-jdk-headless would be 50MB smaller, unfortunately, CMake + # does mistakenly thinks it does not contain JNI feature. + openjdk-8-jdk + + # These are used by the ./ci.sh lint in the native builder. + clang-format-7 + clang-format-8 + + # For coverage builds + gcovr + + # For compiling giflib documentation. + xmlto + + # Common libraries. + libstdc++-8-dev + + # We don't use tcmalloc on archs other than amd64. This installs + # libgoogle-perftools4:amd64. + google-perftools + + # NodeJS for running WASM tests + nodejs + + # To generate API documentation. + doxygen + + # Freezes version that builds (passes tests). Newer version + # (2.30-21ubuntu1~18.04.4) claims to fix "On Intel Skylake + # (-march=native) generated avx512 instruction can be wrong", + # but newly added tests does not pass. Perhaps the problem is + # that mingw package is not updated. + binutils-source=2.30-15ubuntu1 + ) + + # Install packages that are arch-dependent. + local ubarch + for ubarch in "${LIST_ARCHS[@]}"; do + packages+=( + # Library dependencies. These normally depend on the target architecture + # we are compiling for and can't usually be installed for multiple + # architectures at the same time. + libgif7:"${ubarch}" + libjpeg-dev:"${ubarch}" + libpng-dev:"${ubarch}" + libqt5x11extras5-dev:"${ubarch}" + + libstdc++-8-dev:"${ubarch}" + qtbase5-dev:"${ubarch}" + + # For OpenEXR: + libilmbase12:"${ubarch}" + libopenexr22:"${ubarch}" + + # TCMalloc dependency + libunwind-dev:"${ubarch}" + + # Cross-compiling tools per arch. + libc6-dev-"${ubarch}"-cross + libstdc++-8-dev-"${ubarch}"-cross + ) + done + + local target + for target in "${LIST_TARGETS[@]}"; do + # Per target cross-compiling tools. + if [[ "${target}" != "x86_64-linux-gnu" ]]; then + packages+=( + binutils-"${target}" + gcc-"${target}" + ) + fi + done + + # Install all the manual packages via "apt install" for the main arch. These + # will be installed for other archs via manual download and unpack. + apt install -y "${packages[@]}" "${UNPACK_PKGS[@]}" +} + +# binutils <2.32 need a patch. +install_binutils() { + local workdir=$(mktemp -d --suffix=_install) + CLEANUP_FILES+=("${workdir}") + pushd "${workdir}" + apt source binutils-mingw-w64 + apt -y build-dep binutils-mingw-w64 + cd binutils-mingw-w64-8ubuntu1 + cp "${MYDIR}/binutils_align_fix.patch" debian/patches + echo binutils_align_fix.patch >> debian/patches/series + dpkg-buildpackage -b + cd .. + dpkg -i *deb + popd +} + +# Install a library from the source code for multiple targets. +# Usage: install_from_source <tar_url> <sha256> <target> [<target...>] +install_from_source() { + local package="$1" + shift + + local url + eval "url=\${${package}_URL}" + local sha256 + eval "sha256=\${${package}_SHA256}" + # Optional package flags + local pkgflags + eval "pkgflags=\${${package}_FLAGS:-}" + + local workdir=$(mktemp -d --suffix=_install) + CLEANUP_FILES+=("${workdir}") + + local tarfile="${workdir}"/$(basename "${url}") + curl -L --output "${tarfile}" "${url}" + if ! echo "${sha256} ${tarfile}" | sha256sum -c --status -; then + echo "SHA256 mismatch for ${url}: expected ${sha256} but found:" + sha256sum "${tarfile}" + exit 1 + fi + + local target + for target in "$@"; do + echo "Installing ${package} for target ${target} from ${url}" + + local srcdir="${workdir}/source-${target}" + mkdir -p "${srcdir}" + tar -zxf "${tarfile}" -C "${srcdir}" --strip-components=1 + + local prefix="/usr" + if [[ "${target}" != "x86_64-linux-gnu" ]]; then + prefix="/usr/${target}" + fi + + # Apply patches to buildfiles. + if [[ "${package}" == "GIFLIB" && "${target}" == *mingw32 ]]; then + # GIFLIB Makefile has several problems so we need to fix them here. We are + # using a patch from MSYS2 that already fixes the compilation for mingw. + local make_patch="${srcdir}/libgif.patch" + curl -L "${GIFLIB_PATCH_URL}" -o "${make_patch}" + echo "${GIFLIB_PATCH_SHA256} ${make_patch}" | sha256sum -c --status - + patch "${srcdir}/Makefile" < "${make_patch}" + elif [[ "${package}" == "LIBPNG" && "${target}" == wasm* ]]; then + # Cut the dependency to libm; there is pull request to fix it, so this + # might not be needed in the future. + sed -i 's/APPLE/EMSCRIPTEN/g' "${srcdir}/CMakeLists.txt" + fi + + local cmake_args=() + local export_args=("CC=clang-7" "CXX=clang++-7") + local cmake="cmake" + local make="make" + local system_name="Linux" + if [[ "${target}" == *mingw32 ]]; then + system_name="Windows" + # When compiling with clang, CMake doesn't detect that we are using mingw. + cmake_args+=( + -DMINGW=1 + # Googletest needs this when cross-compiling to windows + -DCMAKE_CROSSCOMPILING=1 + -DHAVE_STD_REGEX=0 + -DHAVE_POSIX_REGEX=0 + -DHAVE_GNU_POSIX_REGEX=0 + ) + local windres=$(which ${target}-windres || true) + if [[ -n "${windres}" ]]; then + cmake_args+=(-DCMAKE_RC_COMPILER="${windres}") + fi + fi + if [[ "${target}" == wasm* ]]; then + system_name="WASM" + cmake="emcmake cmake" + make="emmake make" + export_args=() + cmake_args+=( + -DCMAKE_FIND_ROOT_PATH="${prefix}" + -DCMAKE_PREFIX_PATH="${prefix}" + ) + # Static and shared library link to the same file -> race condition. + nproc=1 + else + nproc=`nproc --all` + fi + cmake_args+=(-DCMAKE_SYSTEM_NAME="${system_name}") + + if [[ "${target}" != "x86_64-linux-gnu" ]]; then + # Cross-compiling. + cmake_args+=( + -DCMAKE_C_COMPILER_TARGET="${target}" + -DCMAKE_CXX_COMPILER_TARGET="${target}" + -DCMAKE_SYSTEM_PROCESSOR="${target%%-*}" + ) + fi + + if [[ -e "${srcdir}/CMakeLists.txt" ]]; then + # Most packages use cmake for building which is easier to configure for + # cross-compiling. + if [[ "${package}" == "JPEG_TURBO" && "${target}" == wasm* ]]; then + # JT erroneously detects WASM CPU as i386 and tries to use asm. + # Wasm/Emscripten support for dynamic linking is incomplete; disable + # to avoid CMake warning. + cmake_args+=(-DWITH_SIMD=0 -DENABLE_SHARED=OFF) + fi + ( + cd "${srcdir}" + export ${export_args[@]} + ${cmake} \ + -DCMAKE_INSTALL_PREFIX="${prefix}" \ + "${cmake_args[@]}" ${pkgflags} + ${make} -j${nproc} + ${make} install + ) + elif [[ "${package}" == "GIFLIB" ]]; then + # GIFLIB doesn't yet have a cmake build system. There is a pull + # request in giflib for adding CMakeLists.txt so this might not be + # needed in the future. + ( + cd "${srcdir}" + local giflib_make_flags=( + CFLAGS="-O2 --target=${target} -std=gnu99" + PREFIX="${prefix}" + ) + if [[ "${target}" != wasm* ]]; then + giflib_make_flags+=(CC=clang-7) + fi + # giflib make dependencies are not properly set up so parallel building + # doesn't work for everything. + ${make} -j${nproc} libgif.a "${giflib_make_flags[@]}" + ${make} -j${nproc} all "${giflib_make_flags[@]}" + ${make} install "${giflib_make_flags[@]}" + ) + else + echo "Don't know how to install ${package}" + exit 1 + fi + + # CMake mistakenly uses ".so" libraries and EMCC fails to link properly. + if [[ "${target}" == wasm* ]]; then + rm -f "${prefix}/lib"/*.so* + fi + done +} + +# Packages that are manually unpacked for each architecture. +UNPACK_PKGS=( + libgif-dev + libclang-common-7-dev + + # For OpenEXR: + libilmbase-dev + libopenexr-dev + + # TCMalloc + libgoogle-perftools-dev + libtcmalloc-minimal4 + libgoogle-perftools4 +) + +# Main script entry point. +main() { + cd "${MYDIR}" + + # Configure the repositories with the sources for multi-arch cross + # compilation. + setup_apt + apt-get update -y + apt-get dist-upgrade -y + + install_pkgs + install_binutils + apt clean + + # Remove prebuilt Java classes cache. + rm /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/classes.jsa + + # Manually extract packages for the target arch that can't install it directly + # at the same time as the native ones. + local ubarch + for ubarch in "${LIST_ARCHS[@]}"; do + if [[ "${ubarch}" != "amd64" ]]; then + local pkg + for pkg in "${UNPACK_PKGS[@]}"; do + apt download "${pkg}":"${ubarch}" + dpkg -x "${pkg}"_*_"${ubarch}".deb / + done + fi + done + # TODO: Add clang from the llvm repos. This is problematic since we are + # installing libclang-common-7-dev:"${ubarch}" from the ubuntu ports repos + # which is not available in the llvm repos so it might have a different + # version than the ubuntu ones. + + # Remove the win32 libgcc version. The gcc-mingw-w64-x86-64 (and i686) + # packages install two libgcc versions: + # /usr/lib/gcc/x86_64-w64-mingw32/7.3-posix + # /usr/lib/gcc/x86_64-w64-mingw32/7.3-win32 + # (exact libgcc version number depends on the package version). + # + # Clang will pick the best libgcc, sorting by version, but it doesn't + # seem to be a way to specify one or the other one, except by passing + # -nostdlib and setting all the include paths from the command line. + # To check which one is being used you can run: + # clang++-7 --target=x86_64-w64-mingw32 -v -print-libgcc-file-name + # We need to use the "posix" versions for thread support, so here we + # just remove the other one. + local target + for target in "${LIST_MINGW_TARGETS[@]}"; do + update-alternatives --set "${target}-gcc" $(which "${target}-gcc-posix") + local gcc_win32_path=$("${target}-cpp-win32" -print-libgcc-file-name) + rm -rf $(dirname "${gcc_win32_path}") + done + + # TODO: Add msan for the target when cross-compiling. This only installs it + # for amd64. + ./msan_install.sh + + # Build and install qemu user-linux targets. + ./qemu_install.sh + + # Install emscripten SDK. + ./emsdk_install.sh + + # Setup environment for building WASM libraries from sources. + source /opt/emsdk/emsdk_env.sh + + # Install some dependency libraries manually for the different targets. + + install_from_source JPEG_TURBO "${LIST_MINGW_TARGETS[@]}" "${LIST_WASM_TARGETS[@]}" + install_from_source ZLIB "${LIST_MINGW_TARGETS[@]}" "${LIST_WASM_TARGETS[@]}" + install_from_source LIBPNG "${LIST_MINGW_TARGETS[@]}" "${LIST_WASM_TARGETS[@]}" + install_from_source GIFLIB "${LIST_MINGW_TARGETS[@]}" "${LIST_WASM_TARGETS[@]}" + # webp in Ubuntu is relatively old so we install it from source for everybody. + install_from_source WEBP "${LIST_TARGETS[@]}" "${LIST_MINGW_TARGETS[@]}" + + install_from_source BENCHMARK "${LIST_TARGETS[@]}" "${LIST_MINGW_TARGETS[@]}" + + # Install v8. v8 has better WASM SIMD support than NodeJS 14 (LTS). + # First we need the installer to install v8. + npm install jsvu -g + # install specific version; + HOME=/opt jsvu --os=linux64 "v8@${V8_VERSION}" + ln -s "/opt/.jsvu/v8-${V8_VERSION}" "/opt/.jsvu/v8" + + # Cleanup. + find /var/lib/apt/lists/ -mindepth 1 -delete +} + +main "$@" diff --git a/third_party/jpeg-xl/docker/scripts/msan_install.sh b/third_party/jpeg-xl/docker/scripts/msan_install.sh new file mode 100755 index 0000000000..0216f62b04 --- /dev/null +++ b/third_party/jpeg-xl/docker/scripts/msan_install.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -eu + +MYDIR=$(dirname $(realpath "$0")) + +# Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS +CMAKE_FLAGS=${CMAKE_FLAGS:-} +CMAKE_C_FLAGS=${CMAKE_C_FLAGS:-${CMAKE_FLAGS}} +CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS:-${CMAKE_FLAGS}} +CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-} + +CLANG_VERSION="${CLANG_VERSION:-}" +# Detect the clang version suffix and store it in CLANG_VERSION. For example, +# "6.0" for clang 6 or "7" for clang 7. +detect_clang_version() { + if [[ -n "${CLANG_VERSION}" ]]; then + return 0 + fi + local clang_version=$("${CC:-clang}" --version | head -n1) + local llvm_tag + case "${clang_version}" in + "clang version 6."*) + CLANG_VERSION="6.0" + ;; + "clang version 7."*) + CLANG_VERSION="7" + ;; + "clang version 8."*) + CLANG_VERSION="8" + ;; + "clang version 9."*) + CLANG_VERSION="9" + ;; + *) + echo "Unknown clang version: ${clang_version}" >&2 + return 1 + esac +} + +# Temporary files cleanup hooks. +CLEANUP_FILES=() +cleanup() { + if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then + rm -fr "${CLEANUP_FILES[@]}" + fi +} +trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT + +# Install libc++ libraries compiled with msan in the msan_prefix for the current +# compiler version. +cmd_msan_install() { + local tmpdir=$(mktemp -d) + CLEANUP_FILES+=("${tmpdir}") + # Detect the llvm to install: + export CC="${CC:-clang}" + export CXX="${CXX:-clang++}" + detect_clang_version + local llvm_tag + case "${CLANG_VERSION}" in + "6.0") + llvm_tag="llvmorg-6.0.1" + ;; + "7") + llvm_tag="llvmorg-7.0.1" + ;; + "8") + llvm_tag="llvmorg-8.0.0" + ;; + *) + echo "Unknown clang version: ${clang_version}" >&2 + return 1 + esac + local llvm_targz="${tmpdir}/${llvm_tag}.tar.gz" + curl -L --show-error -o "${llvm_targz}" \ + "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz" + tar -C "${tmpdir}" -zxf "${llvm_targz}" + local llvm_root="${tmpdir}/llvm-project-${llvm_tag}" + + local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" + rm -rf "${msan_prefix}" + + declare -A CMAKE_EXTRAS + CMAKE_EXTRAS[libcxx]="\ + -DLIBCXX_CXX_ABI=libstdc++ \ + -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON" + + for project in libcxx; do + local proj_build="${tmpdir}/build-${project}" + local proj_dir="${llvm_root}/${project}" + mkdir -p "${proj_build}" + cmake -B"${proj_build}" -H"${proj_dir}" \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=Memory \ + -DLLVM_PATH="${llvm_root}/llvm" \ + -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \ + head -n1)" \ + -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \ + -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \ + -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \ + -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \ + ${CMAKE_EXTRAS[${project}]} + cmake --build "${proj_build}" + ninja -C "${proj_build}" install + done +} + +main() { + set -x + for version in 6.0 7 8; do + if ! which "clang-${version}" >/dev/null; then + echo "Skipping msan install for clang version ${version}" + continue + fi + ( + trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT + export CLANG_VERSION=${version} + export CC=clang-${version} + export CXX=clang++-${version} + cmd_msan_install + ) & + done + wait +} + +main "$@" diff --git a/third_party/jpeg-xl/docker/scripts/qemu_install.sh b/third_party/jpeg-xl/docker/scripts/qemu_install.sh new file mode 100755 index 0000000000..8106c4471d --- /dev/null +++ b/third_party/jpeg-xl/docker/scripts/qemu_install.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright (c) the JPEG XL Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +QEMU_RELEASE="4.1.0" +QEMU_URL="https://download.qemu.org/qemu-${QEMU_RELEASE}.tar.xz" +QEMU_ARCHS=( + aarch64 + arm + i386 + # TODO: Consider adding these: + # aarch64_be + # mips64el + # mips64 + # mips + # ppc64 + # ppc +) + +# Ubuntu packages not installed that are needed to build qemu. +QEMU_BUILD_DEPS=( + libglib2.0-dev + libpixman-1-dev + flex + bison +) + +set -eu -x + +# Temporary files cleanup hooks. +CLEANUP_FILES=() +cleanup() { + if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then + rm -fr "${CLEANUP_FILES[@]}" + fi +} +trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT + +main() { + local workdir=$(mktemp -d --suffix=qemu) + CLEANUP_FILES+=("${workdir}") + + apt install -y "${QEMU_BUILD_DEPS[@]}" + + local qemutar="${workdir}/qemu.tar.gz" + curl --output "${qemutar}" "${QEMU_URL}" + tar -Jxf "${qemutar}" -C "${workdir}" + local srcdir="${workdir}/qemu-${QEMU_RELEASE}" + + local builddir="${workdir}/build" + local prefixdir="${workdir}/prefix" + mkdir -p "${builddir}" + + # List of targets to build. + local targets="" + local make_targets=() + local target + for target in "${QEMU_ARCHS[@]}"; do + targets="${targets} ${target}-linux-user" + # Build just the linux-user targets. + make_targets+=("${target}-linux-user/all") + done + + cd "${builddir}" + "${srcdir}/configure" \ + --prefix="${prefixdir}" \ + --static --disable-system --enable-linux-user \ + --target-list="${targets}" + + make -j $(nproc --all || echo 1) "${make_targets[@]}" + + # Manually install these into the non-standard location. This script runs as + # root anyway. + for target in "${QEMU_ARCHS[@]}"; do + cp "${target}-linux-user/qemu-${target}" "/usr/bin/qemu-${target}-static" + done + + apt autoremove -y --purge "${QEMU_BUILD_DEPS[@]}" +} + +main "$@" |