diff options
Diffstat (limited to 'travis-ci')
| -rw-r--r-- | travis-ci/.dockerignore | 30 | ||||
| -rw-r--r-- | travis-ci/Dockerfile | 37 | ||||
| -rwxr-xr-x | travis-ci/managers/debian.sh | 96 | ||||
| -rwxr-xr-x | travis-ci/managers/fedora.sh | 120 | ||||
| -rwxr-xr-x | travis-ci/managers/fuzzbuzz.sh | 21 | ||||
| -rw-r--r-- | travis-ci/managers/travis_wait.bash | 61 | ||||
| -rw-r--r-- | travis-ci/requirements.txt | 5 | ||||
| -rwxr-xr-x | travis-ci/scripts/build-docker-image.sh | 13 | ||||
| -rwxr-xr-x | travis-ci/tools/get-coverity.sh | 35 |
9 files changed, 418 insertions, 0 deletions
diff --git a/travis-ci/.dockerignore b/travis-ci/.dockerignore new file mode 100644 index 0000000..0392158 --- /dev/null +++ b/travis-ci/.dockerignore @@ -0,0 +1,30 @@ +*.a +*.cache +*.gch +*.log +*.o +*.plist +*.py[co] +*.stamp +*.swp +*.trs +*~ +.config.args +.deps/ +/*.gcda +/*.gcno +/GPATH +/GRTAGS +/GSYMS +/GTAGS +/TAGS +/ID +/build* +/coverage/ +/install-tree +/mkosi.builddir/ +/tags +image.raw +image.raw.cache-pre-dev +image.raw.cache-pre-inst +__pycache__/ diff --git a/travis-ci/Dockerfile b/travis-ci/Dockerfile new file mode 100644 index 0000000..daf0ea9 --- /dev/null +++ b/travis-ci/Dockerfile @@ -0,0 +1,37 @@ +## Create Dockerfile that builds container suitable for systemd build +## This container runs as non-root user by default + +FROM fedora:27 + +# Demand the specification of non-root username +ARG DOCKER_USER +ARG DOCKER_USER_UID +ARG DOCKER_USER_GID + +# Copy the requirements into the container at /tmp +COPY requirements.txt /tmp/ + +# Install the requirements +# RUN dnf -y update FIXME +RUN dnf -y install $(cat '/tmp/requirements.txt') +# clean step to prevent cache and metadata corruption +RUN dnf clean all +RUN dnf -y builddep systemd + +# Add non-root user and chown the project dir +RUN groupadd -g $DOCKER_USER_GID $DOCKER_USER +RUN useradd --create-home --shell /bin/bash -u $DOCKER_USER_UID -g $DOCKER_USER_GID -G wheel $DOCKER_USER +ENV HOME /home/$DOCKER_USER +ENV PROJECTDIR $HOME/systemd + +# Copy content to the project directory +COPY . $PROJECTDIR + +# Greant user all permissions to the project dir +RUN chown -R $DOCKER_USER $PROJECTDIR + +# Switch to noroot user by default +USER $DOCKER_USER + +# Update workdir to user home dir +WORKDIR $PROJECTDIR diff --git a/travis-ci/managers/debian.sh b/travis-ci/managers/debian.sh new file mode 100755 index 0000000..558a55a --- /dev/null +++ b/travis-ci/managers/debian.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Run this script from the root of the systemd's git repository +# or set REPO_ROOT to a correct path. +# +# Example execution on Fedora: +# dnf install docker +# systemctl start docker +# export CONT_NAME="my-fancy-container" +# travis-ci/managers/debian.sh SETUP RUN CLEANUP + +PHASES=(${@:-SETUP RUN RUN_ASAN_UBSAN CLEANUP}) +DEBIAN_RELEASE="${DEBIAN_RELEASE:-testing}" +CONT_NAME="${CONT_NAME:-systemd-debian-$DEBIAN_RELEASE}" +DOCKER_EXEC="${DOCKER_EXEC:-docker exec -it $CONT_NAME}" +DOCKER_RUN="${DOCKER_RUN:-docker run}" +REPO_ROOT="${REPO_ROOT:-$PWD}" +ADDITIONAL_DEPS=( + clang + fdisk + libfdisk-dev + libp11-kit-dev + libpwquality-dev + libssl-dev + libzstd-dev + perl + python3-libevdev + python3-pyparsing + zstd +) + +function info() { + echo -e "\033[33;1m$1\033[0m" +} + +set -e + +source "$(dirname $0)/travis_wait.bash" + +for phase in "${PHASES[@]}"; do + case $phase in + SETUP) + info "Setup phase" + info "Using Debian $DEBIAN_RELEASE" + printf "FROM debian:$DEBIAN_RELEASE\nRUN bash -c 'apt-get -y update && apt-get install -y systemd'\n" | docker build -t debian-with-systemd/latest - + info "Starting container $CONT_NAME" + $DOCKER_RUN -v $REPO_ROOT:/build:rw -e container=docker \ + -w /build --privileged=true --name $CONT_NAME \ + -dit --net=host debian-with-systemd/latest /bin/systemd + $DOCKER_EXEC bash -c "echo deb-src http://deb.debian.org/debian $DEBIAN_RELEASE main >>/etc/apt/sources.list" + # Wait for the container to properly boot up, otherwise we were + # running following apt-get commands during the initializing/starting + # (early/late bootup) phase, which caused nasty race conditions + $DOCKER_EXEC bash -c 'systemctl is-system-running --wait || :' + $DOCKER_EXEC apt-get -y update + $DOCKER_EXEC apt-get -y build-dep systemd + $DOCKER_EXEC apt-get -y install "${ADDITIONAL_DEPS[@]}" + ;; + RUN|RUN_GCC|RUN_CLANG) + if [[ "$phase" = "RUN_CLANG" ]]; then + ENV_VARS="-e CC=clang -e CXX=clang++" + MESON_ARGS="--optimization=1" + fi + docker exec $ENV_VARS -it $CONT_NAME meson --werror -Dtests=unsafe -Dslow-tests=true -Dfuzz-tests=true -Dsplit-usr=true -Dman=true $MESON_ARGS build + $DOCKER_EXEC ninja -v -C build + docker exec -e "TRAVIS=$TRAVIS" -it $CONT_NAME ninja -C build test + ;; + RUN_ASAN_UBSAN|RUN_GCC_ASAN_UBSAN|RUN_CLANG_ASAN_UBSAN) + if [[ "$phase" = "RUN_CLANG_ASAN_UBSAN" ]]; then + ENV_VARS="-e CC=clang -e CXX=clang++" + # Build fuzzer regression tests only with clang (for now), + # see: https://github.com/systemd/systemd/pull/15886#issuecomment-632689604 + # -Db_lundef=false: See https://github.com/mesonbuild/meson/issues/764 + MESON_ARGS="-Db_lundef=false -Dfuzz-tests=true --optimization=1" + fi + docker exec $ENV_VARS -it $CONT_NAME meson --werror -Dtests=unsafe -Db_sanitize=address,undefined -Dsplit-usr=true $MESON_ARGS build + $DOCKER_EXEC ninja -v -C build + + # Never remove halt_on_error from UBSAN_OPTIONS. See https://github.com/systemd/systemd/commit/2614d83aa06592aedb. + travis_wait docker exec --interactive=false \ + -e UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 \ + -e ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 \ + -e "TRAVIS=$TRAVIS" \ + -t $CONT_NAME \ + meson test --timeout-multiplier=3 -C ./build/ --print-errorlogs + ;; + CLEANUP) + info "Cleanup phase" + docker stop $CONT_NAME + docker rm -f $CONT_NAME + ;; + *) + echo >&2 "Unknown phase '$phase'" + exit 1 + esac +done diff --git a/travis-ci/managers/fedora.sh b/travis-ci/managers/fedora.sh new file mode 100755 index 0000000..463696a --- /dev/null +++ b/travis-ci/managers/fedora.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +# Run this script from the root of the systemd's git repository +# or set REPO_ROOT to a correct path. +# +# Example execution on Fedora: +# dnf install docker +# systemctl start docker +# export CONT_NAME="my-fancy-container" +# travis-ci/managers/fedora.sh SETUP RUN CLEANUP + +PHASES=(${@:-SETUP RUN RUN_ASAN_UBSAN CLEANUP}) +FEDORA_RELEASE="${FEDORA_RELEASE:-rawhide}" +CONT_NAME="${CONT_NAME:-systemd-fedora-$FEDORA_RELEASE}" +DOCKER_EXEC="${DOCKER_EXEC:-docker exec -it $CONT_NAME}" +DOCKER_RUN="${DOCKER_RUN:-docker run}" +REPO_ROOT="${REPO_ROOT:-$PWD}" +ADDITIONAL_DEPS=( + clang + dnf-plugins-core + hostname + iputils + jq + libasan + libfdisk-devel + libfido2-devel + libpwquality-devel + libubsan + libzstd-devel + llvm + openssl-devel + p11-kit-devel + perl + python3-evdev + python3-pyparsing +) + +info() { + echo -e "\033[33;1m$1\033[0m" +} + +# Simple wrapper which retries given command up to five times +_retry() { + local EC=1 + + for i in {1..5}; do + if "$@"; then + EC=0 + break + fi + + sleep $((i * 5)) + done + + return $EC +} + +set -e + +source "$(dirname $0)/travis_wait.bash" + +for phase in "${PHASES[@]}"; do + case $phase in + SETUP) + info "Setup phase" + info "Using Fedora $FEDORA_RELEASE" + # Pull a Docker image and start a new container + printf "FROM fedora:$FEDORA_RELEASE\nRUN bash -c 'dnf install -y systemd'\n" | docker build -t fedora-with-systemd/latest - + info "Starting container $CONT_NAME" + $DOCKER_RUN -v $REPO_ROOT:/build:rw \ + -w /build --privileged=true --name $CONT_NAME \ + -dit --net=host fedora-with-systemd/latest /sbin/init + # Wait for the container to properly boot up, otherwise we were + # running following dnf commands during the initializing/starting + # (early/late bootup) phase, which caused nasty race conditions + $DOCKER_EXEC bash -c 'systemctl is-system-running --wait || :' + _retry $DOCKER_EXEC dnf makecache + # Install necessary build/test requirements + _retry $DOCKER_EXEC dnf -y --exclude selinux-policy\* upgrade + _retry $DOCKER_EXEC dnf -y install "${ADDITIONAL_DEPS[@]}" + _retry $DOCKER_EXEC dnf -y builddep systemd + ;; + RUN) + info "Run phase" + # Build systemd + $DOCKER_EXEC meson --werror -Dtests=unsafe -Dslow-tests=true -Dfuzz-tests=true build + $DOCKER_EXEC ninja -v -C build + $DOCKER_EXEC ninja -C build test + ;; + RUN_CLANG) + docker exec -e CC=clang -e CXX=clang++ -it $CONT_NAME meson --werror -Dtests=unsafe -Dslow-tests=true -Dfuzz-tests=true -Dman=true build + $DOCKER_EXEC ninja -v -C build + $DOCKER_EXEC ninja -C build test + ;; + RUN_ASAN|RUN_GCC_ASAN_UBSAN|RUN_CLANG_ASAN_UBSAN) + if [[ "$phase" = "RUN_CLANG_ASAN_UBSAN" ]]; then + ENV_VARS="-e CC=clang -e CXX=clang++" + MESON_ARGS="-Db_lundef=false" # See https://github.com/mesonbuild/meson/issues/764 + fi + docker exec $ENV_VARS -it $CONT_NAME meson --werror -Dtests=unsafe -Db_sanitize=address,undefined $MESON_ARGS build + $DOCKER_EXEC ninja -v -C build + + # Never remove halt_on_error from UBSAN_OPTIONS. See https://github.com/systemd/systemd/commit/2614d83aa06592aedb. + travis_wait docker exec --interactive=false \ + -e UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 \ + -e ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 \ + -e "TRAVIS=$TRAVIS" \ + -t $CONT_NAME \ + meson test --timeout-multiplier=3 -C ./build/ --print-errorlogs + ;; + CLEANUP) + info "Cleanup phase" + docker stop $CONT_NAME + docker rm -f $CONT_NAME + ;; + *) + error "Unknown phase '$phase'" + exit 1 + esac +done diff --git a/travis-ci/managers/fuzzbuzz.sh b/travis-ci/managers/fuzzbuzz.sh new file mode 100755 index 0000000..b69197f --- /dev/null +++ b/travis-ci/managers/fuzzbuzz.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e +set -x +set -u + +REPO_ROOT=${REPO_ROOT:-$(pwd)} + +sudo bash -c "echo 'deb-src http://archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse' >>/etc/apt/sources.list" +sudo apt-get update -y +sudo apt-get build-dep systemd -y +sudo apt-get install -y ninja-build python3-pip python3-setuptools quota +# The following should be dropped when debian packaging has been updated to include them +sudo apt-get install -y libfdisk-dev libp11-kit-dev libssl-dev libpwquality-dev +pip3 install meson + +cd $REPO_ROOT +export PATH="$HOME/.local/bin/:$PATH" +tools/oss-fuzz.sh +./out/fuzz-unit-file -max_total_time=5 +git clean -dxff diff --git a/travis-ci/managers/travis_wait.bash b/travis-ci/managers/travis_wait.bash new file mode 100644 index 0000000..59de663 --- /dev/null +++ b/travis-ci/managers/travis_wait.bash @@ -0,0 +1,61 @@ +# This was borrowed from https://github.com/travis-ci/travis-build/tree/master/lib/travis/build/bash +# to get around https://github.com/travis-ci/travis-ci/issues/9979. It should probably be removed +# as soon as Travis CI has started to provide an easy way to export the functions to bash scripts. + +travis_jigger() { + local cmd_pid="${1}" + shift + local timeout="${1}" + shift + local count=0 + + echo -e "\\n" + + while [[ "${count}" -lt "${timeout}" ]]; do + count="$((count + 1))" + echo -ne "Still running (${count} of ${timeout}): ${*}\\r" + sleep 60 + done + + echo -e "\\n${ANSI_RED}Timeout (${timeout} minutes) reached. Terminating \"${*}\"${ANSI_RESET}\\n" + kill -9 "${cmd_pid}" +} + +travis_wait() { + local timeout="${1}" + + if [[ "${timeout}" =~ ^[0-9]+$ ]]; then + shift + else + timeout=20 + fi + + local cmd=("${@}") + local log_file="travis_wait_${$}.log" + + "${cmd[@]}" &>"${log_file}" & + local cmd_pid="${!}" + + travis_jigger "${!}" "${timeout}" "${cmd[@]}" & + local jigger_pid="${!}" + local result + + { + set +e + wait "${cmd_pid}" 2>/dev/null + result="${?}" + ps -p"${jigger_pid}" &>/dev/null && kill "${jigger_pid}" + set -e + } + + if [[ "${result}" -eq 0 ]]; then + echo -e "\\n${ANSI_GREEN}The command ${cmd[*]} exited with ${result}.${ANSI_RESET}" + else + echo -e "\\n${ANSI_RED}The command ${cmd[*]} exited with ${result}.${ANSI_RESET}" + fi + + echo -e "\\n${ANSI_GREEN}Log:${ANSI_RESET}\\n" + cat "${log_file}" + + return "${result}" +} diff --git a/travis-ci/requirements.txt b/travis-ci/requirements.txt new file mode 100644 index 0000000..5ef30d5 --- /dev/null +++ b/travis-ci/requirements.txt @@ -0,0 +1,5 @@ +dnf-plugins-core +meson +ninja-build +perl +python diff --git a/travis-ci/scripts/build-docker-image.sh b/travis-ci/scripts/build-docker-image.sh new file mode 100755 index 0000000..69f9d69 --- /dev/null +++ b/travis-ci/scripts/build-docker-image.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Check environment +[ -z "$TRAVIS_COMMIT" ] && echo "ERROR: TRAVIS_COMMIT must be set" && exit 1 + +# Build docker image +echo -e "\n\033[33;1mBuilding docker image: coverity-$TRAVIS_COMMIT.\033[0m" + +docker build \ + --build-arg DOCKER_USER=$USER \ + --build-arg DOCKER_USER_UID=`id -u` \ + --build-arg DOCKER_USER_GID=`id -g` \ + --force-rm -t coverity-${TRAVIS_COMMIT} --pull=true . diff --git a/travis-ci/tools/get-coverity.sh b/travis-ci/tools/get-coverity.sh new file mode 100755 index 0000000..3634e57 --- /dev/null +++ b/travis-ci/tools/get-coverity.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Download and extract coverity tool + +# Environment check +[ -z "$COVERITY_SCAN_TOKEN" ] && echo 'ERROR: COVERITY_SCAN_TOKEN must be set' && exit 1 + +# Use default values if not set +PLATFORM=$(uname) + +TOOL_BASE=${TOOL_BASE:="/tmp/coverity-scan-analysis"} +TOOL_ARCHIVE=${TOOL_ARCHIVE:="/tmp/cov-analysis-${PLATFORM}.tgz"} + +TOOL_URL="https://scan.coverity.com/download/${PLATFORM}" + +# Make sure wget is installed +sudo apt-get update && sudo apt-get -y install wget + +# Get coverity tool +if [ ! -d $TOOL_BASE ]; then + # Download Coverity Scan Analysis Tool + if [ ! -e $TOOL_ARCHIVE ]; then + echo -e "\033[33;1mDownloading Coverity Scan Analysis Tool...\033[0m" + wget -nv -O $TOOL_ARCHIVE $TOOL_URL --post-data "project=$COVERITY_SCAN_PROJECT_NAME&token=$COVERITY_SCAN_TOKEN" + fi + + # Extract Coverity Scan Analysis Tool + echo -e "\033[33;1mExtracting Coverity Scan Analysis Tool...\033[0m" + mkdir -p $TOOL_BASE + pushd $TOOL_BASE + tar xzf $TOOL_ARCHIVE + popd +fi + +echo -e "\033[33;1mCoverity Scan Analysis Tool can be found at $TOOL_BASE ...\033[0m" |