diff options
Diffstat (limited to 'src/pmdk/utils/docker')
30 files changed, 1602 insertions, 0 deletions
diff --git a/src/pmdk/utils/docker/0001-travis-fix-travisci_build_coverity_scan.sh.patch b/src/pmdk/utils/docker/0001-travis-fix-travisci_build_coverity_scan.sh.patch new file mode 100644 index 000000000..9738942aa --- /dev/null +++ b/src/pmdk/utils/docker/0001-travis-fix-travisci_build_coverity_scan.sh.patch @@ -0,0 +1,27 @@ +From b5179dc4822eaab192361da05aa95d98f523960f Mon Sep 17 00:00:00 2001 +From: Lukasz Dorau <lukasz.dorau@intel.com> +Date: Mon, 7 May 2018 12:05:40 +0200 +Subject: [PATCH] travis: fix travisci_build_coverity_scan.sh + +--- + travisci_build_coverity_scan.sh | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/travisci_build_coverity_scan.sh b/travisci_build_coverity_scan.sh +index ad9d4afcf..562b08bcc 100644 +--- a/travisci_build_coverity_scan.sh ++++ b/travisci_build_coverity_scan.sh +@@ -92,8 +92,8 @@ response=$(curl \ + --form description="Travis CI build" \ + $UPLOAD_URL) + status_code=$(echo "$response" | sed -n '$p') +-if [ "$status_code" != "201" ]; then ++if [ "$status_code" != "200" ]; then + TEXT=$(echo "$response" | sed '$d') +- echo -e "\033[33;1mCoverity Scan upload failed: $TEXT.\033[0m" ++ echo -e "\033[33;1mCoverity Scan upload failed: $response.\033[0m" + exit 1 + fi +-- +2.13.6 + diff --git a/src/pmdk/utils/docker/README b/src/pmdk/utils/docker/README new file mode 100644 index 000000000..b4a271359 --- /dev/null +++ b/src/pmdk/utils/docker/README @@ -0,0 +1,19 @@ +Persistent Memory Development Kit + +This is utils/docker/README. + +Scripts in this directory let Travis CI run a Docker container with ubuntu- +or fedora-based environment and build PMDK project inside it. + +'build-local.sh' can be used to build PMDK locally. + +'build-CI.sh' is used for building PMDK on Travis and GitHub Actions CIs + +NOTE: +If you commit changes to any Dockerfile or shell script in the 'images' +subdirectory and then do git-rebase before pushing your commits to the +repository, make sure that you do not squash the commit which is the head in +your repository. This will let Travis and GitHub Actions CIs recreate +Docker images used during the build before the build. Otherwise the not-updated +Docker image will be pulled from the Docker Hub and used during the build on +Travis and GitHub Actions CIs. diff --git a/src/pmdk/utils/docker/build-CI.sh b/src/pmdk/utils/docker/build-CI.sh new file mode 100755 index 000000000..1b75d3b94 --- /dev/null +++ b/src/pmdk/utils/docker/build-CI.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# build-CI.sh - runs a Docker container from a Docker image with environment +# prepared for building PMDK project and starts building PMDK. +# +# This script is used for building PMDK on Travis and GitHub Actions CIs. +# + +set -e + +source $(dirname $0)/set-ci-vars.sh +source $(dirname $0)/set-vars.sh +source $(dirname $0)/valid-branches.sh + +if [[ "$CI_EVENT_TYPE" != "cron" && "$CI_BRANCH" != "coverity_scan" \ + && "$COVERITY" -eq 1 ]]; then + echo "INFO: Skip Coverity scan job if build is triggered neither by " \ + "'cron' nor by a push to 'coverity_scan' branch" + exit 0 +fi + +if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\ + && "$COVERITY" -ne 1 ]]; then + echo "INFO: Skip regular jobs if build is triggered either by 'cron'" \ + " or by a push to 'coverity_scan' branch" + exit 0 +fi + +if [[ -z "$OS" || -z "$OS_VER" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set properly " \ + "(eg. OS=ubuntu, OS_VER=16.04)." + exit 1 +fi + +if [[ -z "$HOST_WORKDIR" ]]; then + echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \ + "the root of the PMDK project on the host machine" + exit 1 +fi + +if [[ -z "$TEST_BUILD" ]]; then + TEST_BUILD=all +fi + +imageName=${DOCKERHUB_REPO}:1.10-${OS}-${OS_VER}-${CI_CPU_ARCH} +containerName=pmdk-${OS}-${OS_VER} + +if [[ $MAKE_PKG -eq 0 ]] ; then command="./run-build.sh"; fi +if [[ $MAKE_PKG -eq 1 ]] ; then command="./run-build-package.sh"; fi +if [[ $COVERAGE -eq 1 ]] ; then command="./run-coverage.sh"; ci_env=`bash <(curl -s https://codecov.io/env)`; fi + +if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\ + && "$COVERITY" -eq 1 ]]; then + command="./run-coverity.sh" +fi + +if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi +if [[ -f $CI_FILE_SKIP_BUILD_PKG_CHECK ]]; then BUILD_PACKAGE_CHECK=n; else BUILD_PACKAGE_CHECK=y; fi +if [ -z "$NDCTL_ENABLE" ]; then ndctl_enable=; else ndctl_enable="--env NDCTL_ENABLE=$NDCTL_ENABLE"; fi +if [[ $UBSAN -eq 1 ]]; then for x in C CPP LD; do declare EXTRA_${x}FLAGS=-fsanitize=undefined; done; fi + +# Only run doc update on $GITHUB_REPO master or stable branch +if [[ -z "${CI_BRANCH}" || -z "${TARGET_BRANCHES[${CI_BRANCH}]}" || "$CI_EVENT_TYPE" == "pull_request" || "$CI_REPO_SLUG" != "${GITHUB_REPO}" ]]; then + AUTO_DOC_UPDATE=0 +fi + +# Check if we are running on a CI (Travis or GitHub Actions) +[ -n "$GITHUB_ACTIONS" -o -n "$TRAVIS" ] && CI_RUN="YES" || CI_RUN="NO" + +# We have a blacklist only for ppc64le arch +if [[ "$CI_CPU_ARCH" == ppc64le ]] ; then BLACKLIST_FILE=../../utils/docker/ppc64le.blacklist; fi + +# docker on travis + ppc64le runs inside an LXD container and for security +# limits what can be done inside it, and as such, `docker run` fails with +# > the input device is not a TTY +# when using -t because of limited permissions to /dev imposed by LXD. +if [[ -n "$TRAVIS" && "$CI_CPU_ARCH" == ppc64le ]] || [[ -n "$GITHUB_ACTIONS" ]]; then + TTY='' +else + TTY='-t' +fi + +WORKDIR=/pmdk +SCRIPTSDIR=$WORKDIR/utils/docker + +# Run a container with +# - environment variables set (--env) +# - host directory containing PMDK source mounted (-v) +# - a tmpfs /tmp with the necessary size and permissions (--tmpfs)* +# - working directory set (-w) +# +# * We need a tmpfs /tmp inside docker but we cannot run it with --privileged +# and do it from inside, so we do using this docker-run option. +# By default --tmpfs add nosuid,nodev,noexec to the mount flags, we don't +# want that and just to make sure we add the usually default rw,relatime just +# in case docker change the defaults. +docker run --rm --name=$containerName -i $TTY \ + $DNS_SETTING \ + $ci_env \ + --env http_proxy=$http_proxy \ + --env https_proxy=$https_proxy \ + --env AUTO_DOC_UPDATE=$AUTO_DOC_UPDATE \ + --env CC=$PMDK_CC \ + --env CXX=$PMDK_CXX \ + --env VALGRIND=$VALGRIND \ + --env EXTRA_CFLAGS=$EXTRA_CFLAGS \ + --env EXTRA_CXXFLAGS=$EXTRA_CXXFLAGS \ + --env EXTRA_LDFLAGS=$EXTRA_LDFLAGS \ + --env REMOTE_TESTS=$REMOTE_TESTS \ + --env TEST_BUILD=$TEST_BUILD \ + --env WORKDIR=$WORKDIR \ + --env EXPERIMENTAL=$EXPERIMENTAL \ + --env BUILD_PACKAGE_CHECK=$BUILD_PACKAGE_CHECK \ + --env SCRIPTSDIR=$SCRIPTSDIR \ + --env TRAVIS=$TRAVIS \ + --env CI_COMMIT_RANGE=$CI_COMMIT_RANGE \ + --env CI_COMMIT=$CI_COMMIT \ + --env CI_REPO_SLUG=$CI_REPO_SLUG \ + --env CI_BRANCH=$CI_BRANCH \ + --env CI_EVENT_TYPE=$CI_EVENT_TYPE \ + --env DOC_UPDATE_GITHUB_TOKEN=$DOC_UPDATE_GITHUB_TOKEN \ + --env COVERITY_SCAN_TOKEN=$COVERITY_SCAN_TOKEN \ + --env COVERITY_SCAN_NOTIFICATION_EMAIL=$COVERITY_SCAN_NOTIFICATION_EMAIL \ + --env FAULT_INJECTION=$FAULT_INJECTION \ + --env GITHUB_ACTIONS=$GITHUB_ACTIONS \ + --env GITHUB_HEAD_REF=$GITHUB_HEAD_REF \ + --env GITHUB_REPO=$GITHUB_REPO \ + --env GITHUB_REPOSITORY=$GITHUB_REPOSITORY \ + --env GITHUB_REF=$GITHUB_REF \ + --env GITHUB_RUN_ID=$GITHUB_RUN_ID \ + --env GITHUB_SHA=$GITHUB_SHA \ + --env CI_RUN=$CI_RUN \ + --env SRC_CHECKERS=$SRC_CHECKERS \ + --env BLACKLIST_FILE=$BLACKLIST_FILE \ + $ndctl_enable \ + --tmpfs /tmp:rw,relatime,suid,dev,exec,size=6G \ + -v $HOST_WORKDIR:$WORKDIR \ + -v /etc/localtime:/etc/localtime \ + -w $SCRIPTSDIR \ + $imageName $command diff --git a/src/pmdk/utils/docker/build-local.sh b/src/pmdk/utils/docker/build-local.sh new file mode 100755 index 000000000..0fa5b2107 --- /dev/null +++ b/src/pmdk/utils/docker/build-local.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2020, Intel Corporation + +# +# build-local.sh - runs a Docker container from a Docker image with environment +# prepared for building PMDK project and starts building PMDK. +# +# This script is for building PMDK locally (not on CI). +# +# Notes: +# - run this script from its location or set the variable 'HOST_WORKDIR' to +# where the root of the PMDK project is on the host machine. +# - set variables 'OS' and 'OS_VER' properly to a system you want to build PMDK +# on (for proper values take a look on the list of Dockerfiles at the +# utils/docker/images directory), eg. OS=ubuntu, OS_VER=16.04. +# - set 'KEEP_TEST_CONFIG' variable to 1 if you do not want the tests to be +# reconfigured (your current test configuration will be preserved and used). +# - tests with Device Dax are not supported by pcheck yet, so do not provide +# these devices in your configuration. +# + +set -e + +# Environment variables that can be customized (default values are after dash): +export KEEP_CONTAINER=${KEEP_CONTAINER:-0} +export KEEP_TEST_CONFIG=${KEEP_TEST_CONFIG:-0} +export TEST_BUILD=${TEST_BUILD:-all} +export REMOTE_TESTS=${REMOTE_TESTS:-1} +export MAKE_PKG=${MAKE_PKG:-0} +export EXTRA_CFLAGS=${EXTRA_CFLAGS} +export EXTRA_CXXFLAGS=${EXTRA_CXXFLAGS:-} +export PMDK_CC=${PMDK_CC:-gcc} +export PMDK_CXX=${PMDK_CXX:-g++} +export EXPERIMENTAL=${EXPERIMENTAL:-n} +export VALGRIND=${VALGRIND:-1} +export DOCKERHUB_REPO=${DOCKERHUB_REPO:-pmem/pmdk} +export GITHUB_REPO=${GITHUB_REPO:-pmem/pmdk} + +if [[ -z "$OS" || -z "$OS_VER" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set " \ + "(eg. OS=ubuntu, OS_VER=16.04)." + exit 1 +fi + +if [[ -z "$HOST_WORKDIR" ]]; then + HOST_WORKDIR=$(readlink -f ../..) +fi + +if [[ "$KEEP_CONTAINER" != "1" ]]; then + RM_SETTING=" --rm" +fi + +imageName=${DOCKERHUB_REPO}:1.10-${OS}-${OS_VER}-${CI_CPU_ARCH} +containerName=pmdk-${OS}-${OS_VER} + +if [[ $MAKE_PKG -eq 1 ]] ; then + command="./run-build-package.sh" +else + command="./run-build.sh" +fi + +if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi +if [ -z "$NDCTL_ENABLE" ]; then ndctl_enable=; else ndctl_enable="--env NDCTL_ENABLE=$NDCTL_ENABLE"; fi + +WORKDIR=/pmdk +SCRIPTSDIR=$WORKDIR/utils/docker + +# Check if we are running on a CI (Travis or GitHub Actions) +[ -n "$GITHUB_ACTIONS" -o -n "$TRAVIS" ] && CI_RUN="YES" || CI_RUN="NO" + +echo Building ${OS}-${OS_VER} + +# Run a container with +# - environment variables set (--env) +# - host directory containing PMDK source mounted (-v) +# - a tmpfs /tmp with the necessary size and permissions (--tmpfs)* +# - working directory set (-w) +# +# * We need a tmpfs /tmp inside docker but we cannot run it with --privileged +# and do it from inside, so we do using this docker-run option. +# By default --tmpfs add nosuid,nodev,noexec to the mount flags, we don't +# want that and just to make sure we add the usually default rw,relatime just +# in case docker change the defaults. +docker run --name=$containerName -ti \ + $RM_SETTING \ + $DNS_SETTING \ + --env http_proxy=$http_proxy \ + --env https_proxy=$https_proxy \ + --env CC=$PMDK_CC \ + --env CXX=$PMDK_CXX \ + --env VALGRIND=$VALGRIND \ + --env EXTRA_CFLAGS=$EXTRA_CFLAGS \ + --env EXTRA_CXXFLAGS=$EXTRA_CXXFLAGS \ + --env EXTRA_LDFLAGS=$EXTRA_LDFLAGS \ + --env REMOTE_TESTS=$REMOTE_TESTS \ + --env CONFIGURE_TESTS=$CONFIGURE_TESTS \ + --env TEST_BUILD=$TEST_BUILD \ + --env WORKDIR=$WORKDIR \ + --env EXPERIMENTAL=$EXPERIMENTAL \ + --env SCRIPTSDIR=$SCRIPTSDIR \ + --env KEEP_TEST_CONFIG=$KEEP_TEST_CONFIG \ + --env CI_RUN=$CI_RUN \ + --env BLACKLIST_FILE=$BLACKLIST_FILE \ + $ndctl_enable \ + --tmpfs /tmp:rw,relatime,suid,dev,exec,size=6G \ + -v $HOST_WORKDIR:$WORKDIR \ + -v /etc/localtime:/etc/localtime \ + $DAX_SETTING \ + -w $SCRIPTSDIR \ + $imageName $command diff --git a/src/pmdk/utils/docker/configure-tests.sh b/src/pmdk/utils/docker/configure-tests.sh new file mode 100755 index 000000000..23148ed4f --- /dev/null +++ b/src/pmdk/utils/docker/configure-tests.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# configure-tests.sh - is called inside a Docker container; configures tests +# and ssh server for use during build of PMDK project. +# + +set -e + +# Configure tests +cat << EOF > $WORKDIR/src/test/testconfig.sh +LONGDIR=LoremipsumdolorsitametconsecteturadipiscingelitVivamuslacinianibhattortordictumsollicitudinNullamvariusvestibulumligulaetegestaselitsemperidMaurisultriciesligulaeuipsumtinciduntluctusMorbimaximusvariusdolorid +# this path is ~3000 characters long +DIRSUFFIX="$LONGDIR/$LONGDIR/$LONGDIR/$LONGDIR/$LONGDIR" +NON_PMEM_FS_DIR=/tmp +PMEM_FS_DIR=/tmp +PMEM_FS_DIR_FORCE_PMEM=1 +TEST_BUILD="debug nondebug" +ENABLE_SUDO_TESTS=y +TM=1 +EOF + +# Configure remote tests +if [[ $REMOTE_TESTS -eq 1 ]]; then + echo "Configuring remote tests" + cat << EOF >> $WORKDIR/src/test/testconfig.sh +NODE[0]=127.0.0.1 +NODE_WORKING_DIR[0]=/tmp/node0 +NODE_ADDR[0]=127.0.0.1 +NODE_ENV[0]="PMEM_IS_PMEM_FORCE=1" +NODE[1]=127.0.0.1 +NODE_WORKING_DIR[1]=/tmp/node1 +NODE_ADDR[1]=127.0.0.1 +NODE_ENV[1]="PMEM_IS_PMEM_FORCE=1" +NODE[2]=127.0.0.1 +NODE_WORKING_DIR[2]=/tmp/node2 +NODE_ADDR[2]=127.0.0.1 +NODE_ENV[2]="PMEM_IS_PMEM_FORCE=1" +NODE[3]=127.0.0.1 +NODE_WORKING_DIR[3]=/tmp/node3 +NODE_ADDR[3]=127.0.0.1 +NODE_ENV[3]="PMEM_IS_PMEM_FORCE=1" +TEST_BUILD="debug nondebug" +TEST_PROVIDERS=sockets +EOF + + mkdir -p ~/.ssh/cm + + cat << EOF >> ~/.ssh/config +Host 127.0.0.1 + StrictHostKeyChecking no + ControlPath ~/.ssh/cm/%r@%h:%p + ControlMaster auto + ControlPersist 10m +EOF + + if [ ! -f /etc/ssh/ssh_host_rsa_key ] + then + (echo $USERPASS | sudo -S ssh-keygen -t rsa -C $USER@$HOSTNAME -P '' -f /etc/ssh/ssh_host_rsa_key) + fi + echo $USERPASS | sudo -S sh -c 'cat /etc/ssh/ssh_host_rsa_key.pub >> /etc/ssh/authorized_keys' + ssh-keygen -t rsa -C $USER@$HOSTNAME -P '' -f ~/.ssh/id_rsa + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys + chmod -R 700 ~/.ssh + chmod 640 ~/.ssh/authorized_keys + chmod 600 ~/.ssh/config + + # Start ssh service + echo $USERPASS | sudo -S $START_SSH_COMMAND + + ssh 127.0.0.1 exit 0 +else + echo "Skipping remote tests" + echo + echo "Removing all libfabric.pc files in order to simulate that libfabric is not installed:" + find /usr -name "libfabric.pc" 2>/dev/null || true + echo $USERPASS | sudo -S sh -c 'find /usr -name "libfabric.pc" -exec rm -f {} + 2>/dev/null' +fi + +# Configure python tests + cat << EOF >> $WORKDIR/src/test/testconfig.py +config = { + 'unittest_log_level': 1, + 'cacheline_fs_dir': '/tmp', + 'force_cacheline': True, + 'page_fs_dir': '/tmp', + 'force_page': False, + 'byte_fs_dir': '/tmp', + 'force_byte': True, + 'tm': True, + 'test_type': 'check', + 'granularity': 'all', + 'fs_dir_force_pmem': 0, + 'keep_going': False, + 'timeout': '3m', + 'build': ['debug', 'release'], + 'force_enable': None, + 'device_dax_path': [], + 'fail_on_skip': False, + 'enable_admin_tests': True + } +EOF + diff --git a/src/pmdk/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch b/src/pmdk/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch new file mode 100644 index 000000000..7377d07f6 --- /dev/null +++ b/src/pmdk/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch @@ -0,0 +1,37 @@ +From d633d3b0a5f03be280efb80a69b9d5ed4e9c4d56 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= <lukasz.stolarczuk@intel.com> +Date: Tue, 14 Jul 2020 13:58:34 +0200 +Subject: [PATCH] fix generating gcov files and turn-off verbose log + +--- + codecov | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/codecov b/codecov +index e702ecd..0a2f4d8 100755 +--- a/codecov ++++ b/codecov +@@ -1108,9 +1108,9 @@ then + if [ "$ft_gcovout" = "0" ]; + then + # suppress gcov output +- bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -exec $gcov_exe -pb $gcov_arg {} +" >/dev/null 2>&1 || true ++ bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -execdir $gcov_exe -pb $gcov_arg {} \;" >/dev/null 2>&1 || true + else +- bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -exec $gcov_exe -pb $gcov_arg {} +" || true ++ bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -execdir $gcov_exe -pb $gcov_arg {} \;" || true + fi + else + say "${e}==>${x} gcov disabled" +@@ -1425,7 +1425,7 @@ do + report_len=$(wc -c < "$file") + if [ "$report_len" -ne 0 ]; + then +- say " ${g}+${x} $file ${e}bytes=$(echo "$report_len" | tr -d ' ')${x}" ++ #say " ${g}+${x} $file ${e}bytes=$(echo "$report_len" | tr -d ' ')${x}" + # append to to upload + _filename=$(basename "$file") + if [ "${_filename##*.}" = 'gcov' ]; +-- +2.25.1 + diff --git a/src/pmdk/utils/docker/images/Dockerfile.fedora-31 b/src/pmdk/utils/docker/images/Dockerfile.fedora-31 new file mode 100644 index 000000000..a699fee85 --- /dev/null +++ b/src/pmdk/utils/docker/images/Dockerfile.fedora-31 @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# Dockerfile - a 'recipe' for Docker to build an image of fedora-based +# environment for building the PMDK project. +# + +# Pull base image +FROM fedora:31 +MAINTAINER piotr.balcer@intel.com + +# libfabric (optional if libfabric-dev >= 1.4.2 is installed) +ENV FABRIC_DEPS "\ + autoconf \ + automake \ + libtool \ + wget" + +ENV VALGRIND_DEPS "\ + autoconf \ + automake \ + file \ + findutils \ + git" + +# pmdk base +ENV BASE_DEPS "\ + git \ + daxctl-devel \ + make \ + ndctl-devel \ + pkgconfig" + +# benchmarks (optional) +ENV BENCH_DEPS "\ + glib2-devel" + +# examples (optional) +ENV EXAMPLES_DEPS "\ + fuse \ + fuse-devel \ + ncurses-devel \ + libuv-devel" + +# documentation (optional) +ENV DOC_DEPS "\ + pandoc" + +# tests +ENV TESTS_DEPS "\ + bc \ + gdb \ + libunwind-devel \ + ndctl \ + openssh-server \ + strace" + +# packaging +ENV PACKAGING_DEPS "\ + rpm-build \ + rpm-build-libs \ + rpmdevtools" + +# Coverity +ENV COVERITY_DEPS "\ + gcc \ + wget" + +# misc +ENV MISC_DEPS "\ + clang \ + hub \ + lbzip2 \ + man \ + python3-flake8 \ + rsync \ + shadow-utils \ + sudo \ + tar \ + which \ + xmlto" + +# Copy install valgrind script +COPY install-valgrind.sh install-valgrind.sh + +# Copy install libfabric script +COPY install-libfabric.sh install-libfabric.sh + +RUN dnf update -y && dnf install -y \ + $FABRIC_DEPS \ + $VALGRIND_DEPS \ + $BASE_DEPS \ + $BENCH_DEPS \ + $EXAMPLES_DEPS \ + $DOC_DEPS \ + $TESTS_DEPS \ + $PACKAGING_DEPS \ + $COVERITY_DEPS \ + $MISC_DEPS \ + $TESTS_DEPS \ + && ./install-valgrind.sh fedora \ + && ./install-libfabric.sh fedora \ + && dnf clean all + +# Add user +ENV USER pmdkuser +ENV USERPASS pmdkpass +RUN useradd -m $USER +RUN echo "$USER:$USERPASS" | chpasswd +RUN gpasswd wheel -a $USER +RUN echo "%wheel ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +USER $USER + +# Set required environment variables +ENV OS fedora +ENV OS_VER 31 +ENV START_SSH_COMMAND /usr/sbin/sshd +ENV PACKAGE_MANAGER rpm +ENV NOTTY 1 diff --git a/src/pmdk/utils/docker/images/Dockerfile.ubuntu-19.10 b/src/pmdk/utils/docker/images/Dockerfile.ubuntu-19.10 new file mode 100644 index 000000000..9b61f8953 --- /dev/null +++ b/src/pmdk/utils/docker/images/Dockerfile.ubuntu-19.10 @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based +# environment for building the PMDK project. +# + +# Pull base image +FROM ubuntu:19.10 +MAINTAINER piotr.balcer@intel.com + +ENV DEBIAN_FRONTEND noninteractive + +# Additional parameters to build docker without building components +ARG SKIP_SCRIPTS_DOWNLOAD + +# libfabric (optional if libfabric-dev >= 1.4.2 is installed) +ENV FABRIC_DEPS "autoconf \ + automake \ + build-essential \ + libtool \ + unzip \ + wget" + +ENV VALGRIND_DEPS "autoconf \ + automake \ + build-essential \ + git" + +# pmdk base +ENV BASE_DEPS "build-essential \ + git \ + libdaxctl-dev \ + libndctl-dev \ + pkg-config" + +# benchmarks (optional) +ENV BENCH_DEPS libglib2.0-dev + +# examples (optional) +ENV EXAMPLES_DEPS "libfuse-dev \ + libncurses5-dev \ + libuv1-dev" + +# documentation (optional) +ENV DOC_DEPS pandoc + +# tests +ENV TESTS_DEPS "bc \ + gdb \ + libc6-dbg \ + libunwind-dev \ + ndctl \ + python3 \ + ssh \ + strace" + +# packaging +ENV PACKAGING_DEPS "debhelper \ + devscripts \ + fakeroot" + +# CodeCov +ENV CODECOV_DEPS curl + +# Coverity +ENV COVERITY_DEPS ruby gcc g++ wget + +# misc +ENV MISC_DEPS "clang \ + clang-format \ + flake8 \ + sudo \ + whois" + +# Copy install valgrind script +COPY install-valgrind.sh install-valgrind.sh + +# Copy install libfabric script +COPY install-libfabric.sh install-libfabric.sh + +# Copy codecov patch and script to download scripts required in run-*.sh +COPY download-scripts.sh download-scripts.sh +COPY 0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch \ + 0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch + +# Update the Apt cache and install basic tools +RUN apt-get update && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + $FABRIC_DEPS \ + $VALGRIND_DEPS \ + $BASE_DEPS \ + $BENCH_DEPS \ + $EXAMPLES_DEPS \ + $DOC_DEPS \ + $TESTS_DEPS \ + $PACKAGING_DEPS \ + $CODECOV_DEPS \ + $COVERITY_DEPS \ + $MISC_DEPS \ + && ./install-valgrind.sh ubuntu \ + && ./install-libfabric.sh \ + && ./download-scripts.sh \ + && rm -rf /var/lib/apt/lists/* + +# Add user +ENV USER pmdkuser +ENV USERPASS pmdkpass +RUN useradd -m $USER -g sudo -p `mkpasswd $USERPASS` +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +# switch user +USER $USER + +# Set required environment variables +ENV OS ubuntu +ENV OS_VER 19.10 +ENV START_SSH_COMMAND service ssh start +ENV PACKAGE_MANAGER dpkg +ENV NOTTY 1 diff --git a/src/pmdk/utils/docker/images/README b/src/pmdk/utils/docker/images/README new file mode 100644 index 000000000..0b8c551e4 --- /dev/null +++ b/src/pmdk/utils/docker/images/README @@ -0,0 +1,6 @@ +Persistent Memory Development Kit + +This is utils/docker/images/README. + +Scripts in this directory let you prepare Docker images for building +PMDK project under specified OS (ubuntu, fedora). diff --git a/src/pmdk/utils/docker/images/build-image.sh b/src/pmdk/utils/docker/images/build-image.sh new file mode 100755 index 000000000..b9e7a2a4b --- /dev/null +++ b/src/pmdk/utils/docker/images/build-image.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# build-image.sh <OS-VER> <ARCH> - prepares a Docker image with <OS>-based +# environment intended for the <ARCH> CPU architecture +# designed for building PMDK project, according to +# the Dockerfile.<OS-VER> file located in the same directory. +# +# The script can be run locally. +# + +set -e + +OS_VER=$1 +CPU_ARCH=$2 + +function usage { + echo "Usage:" + echo " build-image.sh <OS-VER> <ARCH>" + echo "where:" + echo " <OS-VER> - can be for example 'ubuntu-19.10' provided "\ + "a Dockerfile named 'Dockerfile.ubuntu-19.10' "\ + "exists in the current directory and" + echo " <ARCH> - is a CPU architecture, for example 'x86_64'" +} + +# Check if two first arguments are not empty +if [[ -z "$2" ]]; then + usage + exit 1 +fi + +# Check if the file Dockerfile.OS-VER exists +if [[ ! -f "Dockerfile.$OS_VER" ]]; then + echo "Error: Dockerfile.$OS_VER does not exist." + echo + usage + exit 1 +fi + +if [[ -z "${DOCKERHUB_REPO}" ]]; then + echo "Error: DOCKERHUB_REPO environment variable is not set" + exit 1 +fi + +# Build a Docker image tagged with ${DOCKERHUB_REPO}:OS-VER-ARCH +tag=${DOCKERHUB_REPO}:1.10-${OS_VER}-${CPU_ARCH} +docker build -t $tag \ + --build-arg http_proxy=$http_proxy \ + --build-arg https_proxy=$https_proxy \ + -f Dockerfile.$OS_VER . diff --git a/src/pmdk/utils/docker/images/download-scripts.sh b/src/pmdk/utils/docker/images/download-scripts.sh new file mode 100755 index 000000000..fae37b464 --- /dev/null +++ b/src/pmdk/utils/docker/images/download-scripts.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020, Intel Corporation + +# +# download-scripts.sh - downloads specific version of codecov's bash +# script to generate and upload reports. It's useful, +# since unverified version may break coverage results. +# + +set -e + +# master: Merge pull request #342 from codecov/revert-proj-name-..., 18.08.2020 +CODECOV_VERSION="e877c1280cc6e902101fb5df2981ed1c962da7f0" + +if [ "${SKIP_SCRIPTS_DOWNLOAD}" ]; then + echo "Variable 'SKIP_SCRIPTS_DOWNLOAD' is set; skipping scripts' download" + exit +fi + +mkdir -p /opt/scripts + +# Download codecov's bash script +git clone https://github.com/codecov/codecov-bash +cd codecov-bash +git checkout $CODECOV_VERSION + +git apply ../0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch +mv -v codecov /opt/scripts/codecov + +cd .. +rm -rf codecov-bash diff --git a/src/pmdk/utils/docker/images/install-libfabric.sh b/src/pmdk/utils/docker/images/install-libfabric.sh new file mode 100755 index 000000000..355f24525 --- /dev/null +++ b/src/pmdk/utils/docker/images/install-libfabric.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# install-libfabric.sh - installs a customized version of libfabric +# + +set -e + +OS=$1 + +# Keep in sync with requirements in src/common.inc. +libfabric_ver=1.4.2 +libfabric_url=https://github.com/ofiwg/libfabric/archive +libfabric_dir=libfabric-$libfabric_ver +libfabric_tarball=v${libfabric_ver}.zip +wget "${libfabric_url}/${libfabric_tarball}" +unzip $libfabric_tarball + +cd $libfabric_dir + +# XXX HACK HACK HACK +# Disable use of spin locks in libfabric. +# +# spinlocks do not play well (IOW at all) with cpu-constrained environments, +# like GitHub Actions, and this leads to timeouts of some PMDK's tests. +# This change speeds up pmempool_sync_remote/TEST28-31 by a factor of 20-30. +# +perl -pi -e 's/have_spinlock=1/have_spinlock=0/' configure.ac +# XXX HACK HACK HACK + +./autogen.sh +./configure --prefix=/usr --enable-sockets +make -j$(nproc) +make -j$(nproc) install + +cd .. +rm -f ${libfabric_tarball} +rm -rf ${libfabric_dir} diff --git a/src/pmdk/utils/docker/images/install-libndctl.sh b/src/pmdk/utils/docker/images/install-libndctl.sh new file mode 100755 index 000000000..fc8cee3fb --- /dev/null +++ b/src/pmdk/utils/docker/images/install-libndctl.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2019, Intel Corporation + +# +# install-libndctl.sh - installs libndctl +# + +set -e + +OS=$2 + +echo "==== clone ndctl repo ====" +git clone https://github.com/pmem/ndctl.git +cd ndctl +git checkout $1 + +if [ "$OS" = "fedora" ]; then + +echo "==== setup rpmbuild tree ====" +rpmdev-setuptree + +RPMDIR=$HOME/rpmbuild/ +VERSION=$(./git-version) +SPEC=./rhel/ndctl.spec + +echo "==== create source tarball =====" +git archive --format=tar --prefix="ndctl-${VERSION}/" HEAD | gzip > "$RPMDIR/SOURCES/ndctl-${VERSION}.tar.gz" + +echo "==== build ndctl ====" +./autogen.sh +./configure --disable-docs +make -j$(nproc) + +echo "==== build ndctl packages ====" +rpmbuild -ba $SPEC + +echo "==== install ndctl packages ====" +RPM_ARCH=$(uname -m) +rpm -i $RPMDIR/RPMS/$RPM_ARCH/*.rpm + +echo "==== cleanup ====" +rm -rf $RPMDIR + +else + +echo "==== build ndctl ====" +./autogen.sh +./configure --disable-docs +make -j$(nproc) + +echo "==== install ndctl ====" +make -j$(nproc) install + +echo "==== cleanup ====" + +fi + +cd .. +rm -rf ndctl diff --git a/src/pmdk/utils/docker/images/install-valgrind.sh b/src/pmdk/utils/docker/images/install-valgrind.sh new file mode 100755 index 000000000..97babcb67 --- /dev/null +++ b/src/pmdk/utils/docker/images/install-valgrind.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# install-valgrind.sh - installs valgrind for persistent memory +# + +set -e + +OS=$1 + +install_upstream_from_distro() { + case "$OS" in + fedora) dnf install -y valgrind ;; + ubuntu) apt-get install -y --no-install-recommends valgrind ;; + *) return 1 ;; + esac +} + +install_upstream_3_16_1() { + git clone git://sourceware.org/git/valgrind.git + cd valgrind + # valgrind v3.16.1 upstream + git checkout VALGRIND_3_16_BRANCH + ./autogen.sh + ./configure + make -j$(nproc) + make -j$(nproc) install + cd .. + rm -rf valgrind +} + +install_custom-pmem_from_source() { + git clone https://github.com/pmem/valgrind.git + cd valgrind + # valgrind v3.15 with pmemcheck + # 2020.04.01 Merge pull request #78 from marcinslusarz/opt3 + git checkout 759686fd66cc0105df8311cfe676b0b2f9e89196 + ./autogen.sh + ./configure + make -j$(nproc) + make -j$(nproc) install + cd .. + rm -rf valgrind +} + +ARCH=$(uname -m) +case "$ARCH" in + ppc64le) install_upstream_3_16_1 ;; + *) install_custom-pmem_from_source ;; +esac diff --git a/src/pmdk/utils/docker/images/push-image.sh b/src/pmdk/utils/docker/images/push-image.sh new file mode 100755 index 000000000..03b935213 --- /dev/null +++ b/src/pmdk/utils/docker/images/push-image.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# push-image.sh - pushes the Docker image to the Docker Hub. +# +# The script utilizes $DOCKERHUB_USER and $DOCKERHUB_PASSWORD variables +# to log in to Docker Hub. The variables can be set in the Travis project's +# configuration for automated builds. +# + +set -e + +source $(dirname $0)/../set-ci-vars.sh + +if [[ -z "$OS" ]]; then + echo "OS environment variable is not set" + exit 1 +fi + +if [[ -z "$OS_VER" ]]; then + echo "OS_VER environment variable is not set" + exit 1 +fi + +if [[ -z "$CI_CPU_ARCH" ]]; then + echo "CI_CPU_ARCH environment variable is not set" + exit 1 +fi + +if [[ -z "${DOCKERHUB_REPO}" ]]; then + echo "DOCKERHUB_REPO environment variable is not set" + exit 1 +fi + +TAG="1.10-${OS}-${OS_VER}-${CI_CPU_ARCH}" + +# Check if the image tagged with pmdk/OS-VER exists locally +if [[ ! $(docker images -a | awk -v pattern="^${DOCKERHUB_REPO}:${TAG}\$" \ + '$1":"$2 ~ pattern') ]] +then + echo "ERROR: Docker image tagged ${DOCKERHUB_REPO}:${TAG} does not exists locally." + exit 1 +fi + +# Log in to the Docker Hub +docker login -u="$DOCKERHUB_USER" -p="$DOCKERHUB_PASSWORD" + +# Push the image to the repository +docker push ${DOCKERHUB_REPO}:${TAG} diff --git a/src/pmdk/utils/docker/ppc64le.blacklist b/src/pmdk/utils/docker/ppc64le.blacklist new file mode 100644 index 000000000..6002ad91f --- /dev/null +++ b/src/pmdk/utils/docker/ppc64le.blacklist @@ -0,0 +1,19 @@ +ex_librpmem_basic +ex_librpmem_hello +ex_librpmem_manpage +libpmempool_rm_remote +obj_basic_integration +obj_check_remote +obj_ctl_debug +obj_mem +obj_memcheck_register +obj_pmalloc_mt +obj_rpmem_basic_integration +obj_rpmem_heap_interrupt +obj_rpmem_heap_state +obj_ulog_size +pmempool_create +pmempool_sync_remote +pmempool_transform_remote +rpmem_basic +rpmem_fip diff --git a/src/pmdk/utils/docker/prepare-for-build.sh b/src/pmdk/utils/docker/prepare-for-build.sh new file mode 100755 index 000000000..f18c18c68 --- /dev/null +++ b/src/pmdk/utils/docker/prepare-for-build.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# prepare-for-build.sh - is called inside a Docker container; prepares +# the environment inside a Docker container for +# running build of PMDK project. +# + +set -e + +# This should be run only on CIs +if [ "$CI_RUN" == "YES" ]; then + # Make sure $WORKDIR has correct access rights + # - set them to the current UID and GID + echo $USERPASS | sudo -S chown -R $(id -u).$(id -g) $WORKDIR +fi + +# Configure tests (e.g. ssh for remote tests) unless the current configuration +# should be preserved +KEEP_TEST_CONFIG=${KEEP_TEST_CONFIG:-0} +if [[ "$KEEP_TEST_CONFIG" == 0 ]]; then + ./configure-tests.sh +fi diff --git a/src/pmdk/utils/docker/pull-or-rebuild-image.sh b/src/pmdk/utils/docker/pull-or-rebuild-image.sh new file mode 100755 index 000000000..a3a23cdd1 --- /dev/null +++ b/src/pmdk/utils/docker/pull-or-rebuild-image.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# pull-or-rebuild-image.sh - rebuilds the Docker image used in the +# current Travis build if necessary. +# +# The script rebuilds the Docker image if the Dockerfile for the current +# OS version (Dockerfile.${OS}-${OS_VER}) or any .sh script from the directory +# with Dockerfiles were modified and committed. +# +# If the Travis build is not of the "pull_request" type (i.e. in case of +# merge after pull_request) and it succeed, the Docker image should be pushed +# to the Docker Hub repository. An empty file is created to signal that to +# further scripts. +# +# If the Docker image does not have to be rebuilt, it will be pulled from +# Docker Hub. +# + +set -e + +source $(dirname $0)/set-ci-vars.sh +source $(dirname $0)/set-vars.sh + +if [[ "$CI_EVENT_TYPE" != "cron" && "$CI_BRANCH" != "coverity_scan" \ + && "$COVERITY" -eq 1 ]]; then + echo "INFO: Skip Coverity scan job if build is triggered neither by " \ + "'cron' nor by a push to 'coverity_scan' branch" + exit 0 +fi + +if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\ + && "$COVERITY" -ne 1 ]]; then + echo "INFO: Skip regular jobs if build is triggered either by 'cron'" \ + " or by a push to 'coverity_scan' branch" + exit 0 +fi + +if [[ -z "$OS" || -z "$OS_VER" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set properly " \ + "(eg. OS=ubuntu, OS_VER=16.04)." + exit 1 +fi + +if [[ -z "$HOST_WORKDIR" ]]; then + echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \ + "the root of the PMDK project on the host machine" + exit 1 +fi + +# Find all the commits for the current build +if [ -n "$CI_COMMIT_RANGE" ]; then + commits=$(git rev-list $CI_COMMIT_RANGE) +else + commits=$CI_COMMIT +fi + +echo "Commits in the commit range:" +for commit in $commits; do echo $commit; done + +# Get the list of files modified by the commits +files=$(for commit in $commits; do git diff-tree --no-commit-id --name-only \ + -r $commit; done | sort -u) +echo "Files modified within the commit range:" +for file in $files; do echo $file; done + +# Path to directory with Dockerfiles and image building scripts +images_dir_name=images +base_dir=utils/docker/$images_dir_name + +# Check if committed file modifications require the Docker image to be rebuilt +for file in $files; do + # Check if modified files are relevant to the current build + if [[ $file =~ ^($base_dir)\/Dockerfile\.($OS)-($OS_VER)$ ]] \ + || [[ $file =~ ^($base_dir)\/.*\.sh$ ]] + then + # Rebuild Docker image for the current OS version + echo "Rebuilding the Docker image for the Dockerfile.$OS-$OS_VER" + pushd $images_dir_name + ./build-image.sh ${OS}-${OS_VER} ${CI_CPU_ARCH} + popd + + # Check if the image has to be pushed to Docker Hub + # (i.e. the build is triggered by commits to the $GITHUB_REPO + # repository's stable-* or master branch, and the Travis build is not + # of the "pull_request" type). In that case, create the empty + # file. + if [[ "$CI_REPO_SLUG" == "$GITHUB_REPO" \ + && ($CI_BRANCH == stable-* || $CI_BRANCH == devel-* || $CI_BRANCH == master) \ + && $CI_EVENT_TYPE != "pull_request" \ + && $PUSH_IMAGE == "1" ]] + then + echo "The image will be pushed to Docker Hub" + touch $CI_FILE_PUSH_IMAGE_TO_REPO + else + echo "Skip pushing the image to Docker Hub" + fi + + if [[ $PUSH_IMAGE == "1" ]] + then + echo "Skip build package check if image has to be pushed" + touch $CI_FILE_SKIP_BUILD_PKG_CHECK + fi + exit 0 + fi +done + +# Getting here means rebuilding the Docker image is not required. +# Pull the image from Docker Hub. +docker pull ${DOCKERHUB_REPO}:1.10-${OS}-${OS_VER}-${CI_CPU_ARCH} diff --git a/src/pmdk/utils/docker/run-build-package.sh b/src/pmdk/utils/docker/run-build-package.sh new file mode 100755 index 000000000..5141904aa --- /dev/null +++ b/src/pmdk/utils/docker/run-build-package.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2019, Intel Corporation + +# +# run-build-package.sh - is called inside a Docker container; prepares +# the environment and starts a build of PMDK project. +# + +set -e + +# Prepare build enviromnent +./prepare-for-build.sh + +# Create fake tag, so that package has proper 'version' field +git config user.email "test@package.com" +git config user.name "test package" +git tag -a 1.4.99 -m "1.4" HEAD~1 || true + +# Build all and run tests +cd $WORKDIR +export PCHECK_OPTS="-j2 BLACKLIST_FILE=${BLACKLIST_FILE}" +make -j$(nproc) $PACKAGE_MANAGER + +# Install packages +if [[ "$PACKAGE_MANAGER" == "dpkg" ]]; then + cd $PACKAGE_MANAGER + echo $USERPASS | sudo -S dpkg --install *.deb +else + RPM_ARCH=$(uname -m) + cd $PACKAGE_MANAGER/$RPM_ARCH + echo $USERPASS | sudo -S rpm --install *.rpm +fi + +# Compile and run standalone test +cd $WORKDIR/utils/docker/test_package +make -j$(nproc) LIBPMEMOBJ_MIN_VERSION=1.4 +./test_package testfile1 + +# Use pmreorder installed in the system +pmreorder_version="$(pmreorder -v)" +pmreorder_pattern="pmreorder\.py .+$" +(echo "$pmreorder_version" | grep -Ev "$pmreorder_pattern") && echo "pmreorder version failed" && exit 1 + +touch testfile2 +touch logfile1 +pmreorder -p testfile2 -l logfile1 diff --git a/src/pmdk/utils/docker/run-build.sh b/src/pmdk/utils/docker/run-build.sh new file mode 100755 index 000000000..78edf09ec --- /dev/null +++ b/src/pmdk/utils/docker/run-build.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# run-build.sh - is called inside a Docker container; prepares the environment +# and starts a build of PMDK project. +# + +set -e + +# Prepare build environment +./prepare-for-build.sh + +# Build all and run tests +cd $WORKDIR +if [ "$SRC_CHECKERS" != "0" ]; then + make -j$(nproc) check-license + make -j$(nproc) cstyle +fi + +make -j$(nproc) +make -j$(nproc) test +# do not change -j2 to -j$(nproc) in case of tests (make check/pycheck) +make -j2 pcheck TEST_BUILD=$TEST_BUILD +# do not change -j2 to -j$(nproc) in case of tests (make check/pycheck) +make -j2 pycheck +make -j$(nproc) DESTDIR=/tmp source + +# Create PR with generated docs +if [[ "$AUTO_DOC_UPDATE" == "1" ]]; then + echo "Running auto doc update" + ./utils/docker/run-doc-update.sh +fi diff --git a/src/pmdk/utils/docker/run-coverage.sh b/src/pmdk/utils/docker/run-coverage.sh new file mode 100755 index 000000000..b6f7ed0a8 --- /dev/null +++ b/src/pmdk/utils/docker/run-coverage.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2020, Intel Corporation + +# +# run-coverage.sh - is called inside a Docker container; runs tests +# to measure code coverage and sends report to codecov.io +# + +set -e + +# Get and prepare PMDK source +./prepare-for-build.sh + +# Hush error messages, mainly from Valgrind +export UT_DUMP_LINES=0 + +# Skip printing mismatched files for tests with Valgrind +export UT_VALGRIND_SKIP_PRINT_MISMATCHED=1 + +# Build all and run tests +cd $WORKDIR +make -j$(nproc) COVERAGE=1 +make -j$(nproc) test COVERAGE=1 + +# XXX: unfortunately valgrind raports issues in coverage instrumentation +# which we have to ignore (-k flag), also there is dependency between +# local and remote tests (which cannot be easily removed) we have to +# run local and remote tests separately +cd src/test +# do not change -j2 to -j$(nproc) in case of tests (make check/pycheck) +make -kj2 pcheck-local-quiet TEST_BUILD=debug || true +make check-remote-quiet TEST_BUILD=debug || true +# do not change -j2 to -j$(nproc) in case of tests (make check/pycheck) +make -j2 pycheck TEST_BUILD=debug || true +cd ../.. + +# prepare flag for codecov report to differentiate builds +flag=tests +[ -n "$GITHUB_ACTIONS" ] && flag=GHA +[ -n "$TRAVIS" ] && flag=Travis + +# run gcov exe, using codecov's bash (remove parsed coverage files, set flag and exit 1 if not successful) +/opt/scripts/codecov -c -F ${flag} -Z + +printf "check for any leftover gcov files\n" +leftover_files=$(find . -name "*.gcov" | wc -l) +if [[ $leftover_files > 0 ]]; then + # display found files and exit with error (they all should be parsed) + find . -name "*.gcov" + return 1 +fi diff --git a/src/pmdk/utils/docker/run-coverity.sh b/src/pmdk/utils/docker/run-coverity.sh new file mode 100755 index 000000000..140fa753d --- /dev/null +++ b/src/pmdk/utils/docker/run-coverity.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2020, Intel Corporation + +# +# run-coverity.sh - runs the Coverity scan build +# + +set -e + +if [[ "$CI_REPO_SLUG" != "$GITHUB_REPO" \ + && ( "$COVERITY_SCAN_NOTIFICATION_EMAIL" == "" \ + || "$COVERITY_SCAN_TOKEN" == "" ) ]]; then + echo + echo "Skipping Coverity build:"\ + "COVERITY_SCAN_TOKEN=\"$COVERITY_SCAN_TOKEN\" or"\ + "COVERITY_SCAN_NOTIFICATION_EMAIL="\ + "\"$COVERITY_SCAN_NOTIFICATION_EMAIL\" is not set" + exit 0 +fi + +# Prepare build environment +./prepare-for-build.sh + +CERT_FILE=/etc/ssl/certs/ca-certificates.crt +TEMP_CF=$(mktemp) +cp $CERT_FILE $TEMP_CF + +# Download Coverity certificate +echo -n | openssl s_client -connect scan.coverity.com:443 | \ + sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | \ + tee -a $TEMP_CF + +echo $USERPASS | sudo -S mv $TEMP_CF $CERT_FILE + +export COVERITY_SCAN_PROJECT_NAME="$CI_REPO_SLUG" +[[ "$CI_EVENT_TYPE" == "cron" ]] \ + && export COVERITY_SCAN_BRANCH_PATTERN="master" \ + || export COVERITY_SCAN_BRANCH_PATTERN="coverity_scan" +export COVERITY_SCAN_BUILD_COMMAND="make -j$(nproc) all" + +cd $WORKDIR + +# +# Run the Coverity scan +# + +# The 'travisci_build_coverity_scan.sh' script requires the following +# environment variables to be set: +# - TRAVIS_BRANCH - has to contain the name of the current branch +# - TRAVIS_PULL_REQUEST - has to be set to 'true' in case of pull requests +# +export TRAVIS_BRANCH=${CI_BRANCH} +[ "${CI_EVENT_TYPE}" == "pull_request" ] && export TRAVIS_PULL_REQUEST="true" + +# XXX: Patch the Coverity script. +# Recently, this script regularly exits with an error, even though +# the build is successfully submitted. Probably because the status code +# is missing in response, or it's not 201. +# Changes: +# 1) change the expected status code to 200 and +# 2) print the full response string. +# +# This change should be reverted when the Coverity script is fixed. +# +# The previous version was: +# curl -s https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh | bash + +wget https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh +patch < utils/docker/0001-travis-fix-travisci_build_coverity_scan.sh.patch +bash ./travisci_build_coverity_scan.sh diff --git a/src/pmdk/utils/docker/run-doc-update.sh b/src/pmdk/utils/docker/run-doc-update.sh new file mode 100755 index 000000000..132315eb8 --- /dev/null +++ b/src/pmdk/utils/docker/run-doc-update.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2019-2020, Intel Corporation + +set -e + +source `dirname $0`/valid-branches.sh + +BOT_NAME="pmem-bot" +USER_NAME="pmem" +REPO_NAME="pmdk" + +ORIGIN="https://${DOC_UPDATE_GITHUB_TOKEN}@github.com/${BOT_NAME}/${REPO_NAME}" +UPSTREAM="https://github.com/${USER_NAME}/${REPO_NAME}" +# master or stable-* branch +TARGET_BRANCH=${CI_BRANCH} +VERSION=${TARGET_BRANCHES[$TARGET_BRANCH]} + +if [ -z $VERSION ]; then + echo "Target location for branch $TARGET_BRANCH is not defined." + exit 1 +fi + +# Clone bot repo +git clone ${ORIGIN} +cd ${REPO_NAME} +git remote add upstream ${UPSTREAM} + +git config --local user.name ${BOT_NAME} +git config --local user.email "pmem-bot@intel.com" + +git remote update +git checkout -B ${TARGET_BRANCH} upstream/${TARGET_BRANCH} + +# Copy man & PR web md +cd ./doc +make -j$(nproc) web +cd .. + +mv ./doc/web_linux ../ +mv ./doc/web_windows ../ +mv ./doc/generated/libs_map.yml ../ + +# Checkout gh-pages and copy docs +GH_PAGES_NAME="gh-pages-for-${TARGET_BRANCH}" +git checkout -B $GH_PAGES_NAME upstream/gh-pages +git clean -dfx + +rsync -a ../web_linux/ ./manpages/linux/${VERSION}/ +rsync -a ../web_windows/ ./manpages/windows/${VERSION}/ \ + --exclude='librpmem' \ + --exclude='rpmemd' --exclude='pmreorder' \ + --exclude='daxio' + +rm -r ../web_linux +rm -r ../web_windows + +if [ $TARGET_BRANCH = "master" ]; then + [ ! -d _data ] && mkdir _data + cp ../libs_map.yml _data +fi + +# Add and push changes. +# git commit command may fail if there is nothing to commit. +# In that case we want to force push anyway (there might be open pull request +# with changes which were reverted). +git add -A +git commit -m "doc: automatic gh-pages docs update" && true +git push -f ${ORIGIN} $GH_PAGES_NAME + +GITHUB_TOKEN=${DOC_UPDATE_GITHUB_TOKEN} hub pull-request -f \ + -b ${USER_NAME}:gh-pages \ + -h ${BOT_NAME}:${GH_PAGES_NAME} \ + -m "doc: automatic gh-pages docs update" && true + +exit 0 diff --git a/src/pmdk/utils/docker/set-ci-vars.sh b/src/pmdk/utils/docker/set-ci-vars.sh new file mode 100755 index 000000000..af41a753e --- /dev/null +++ b/src/pmdk/utils/docker/set-ci-vars.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020, Intel Corporation + +# +# set-ci-vars.sh -- set CI variables common for both: +# Travis and GitHub Actions CIs +# + +set -e + +function get_commit_range_from_last_merge { + # get commit id of the last merge + LAST_MERGE=$(git log --merges --pretty=%H -1) + LAST_COMMIT=$(git log --pretty=%H -1) + if [ "$LAST_MERGE" == "$LAST_COMMIT" ]; then + # GitHub Actions commits its own merge in case of pull requests + # so the first merge commit has to be skipped. + LAST_MERGE=$(git log --merges --pretty=%H -2 | tail -n1) + fi + if [ "$LAST_MERGE" == "" ]; then + # possible in case of shallow clones + # or new repos with no merge commits yet + # - pick up the first commit + LAST_MERGE=$(git log --pretty=%H | tail -n1) + fi + COMMIT_RANGE="$LAST_MERGE..HEAD" + # make sure it works now + if ! git rev-list $COMMIT_RANGE >/dev/null; then + COMMIT_RANGE="" + fi + echo $COMMIT_RANGE +} + +COMMIT_RANGE_FROM_LAST_MERGE=$(get_commit_range_from_last_merge) + +if [ -n "$TRAVIS" ]; then + CI_COMMIT=$TRAVIS_COMMIT + CI_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE/.../..}" + CI_BRANCH=$TRAVIS_BRANCH + CI_EVENT_TYPE=$TRAVIS_EVENT_TYPE + CI_REPO_SLUG=$TRAVIS_REPO_SLUG + + # CI_COMMIT_RANGE is usually invalid for force pushes - fix it when used + # with non-upstream repository + if [ -n "$CI_COMMIT_RANGE" -a "$CI_REPO_SLUG" != "$GITHUB_REPO" ]; then + if ! git rev-list $CI_COMMIT_RANGE; then + CI_COMMIT_RANGE=$COMMIT_RANGE_FROM_LAST_MERGE + fi + fi + + case "$TRAVIS_CPU_ARCH" in + "amd64") + CI_CPU_ARCH="x86_64" + ;; + *) + CI_CPU_ARCH=$TRAVIS_CPU_ARCH + ;; + esac + +elif [ -n "$GITHUB_ACTIONS" ]; then + CI_COMMIT=$GITHUB_SHA + CI_COMMIT_RANGE=$COMMIT_RANGE_FROM_LAST_MERGE + CI_BRANCH=$(echo $GITHUB_REF | cut -d'/' -f3) + CI_REPO_SLUG=$GITHUB_REPOSITORY + CI_CPU_ARCH="x86_64" # GitHub Actions supports only x86_64 + + case "$GITHUB_EVENT_NAME" in + "schedule") + CI_EVENT_TYPE="cron" + ;; + *) + CI_EVENT_TYPE=$GITHUB_EVENT_NAME + ;; + esac + +else + CI_COMMIT=$(git log --pretty=%H -1) + CI_COMMIT_RANGE=$COMMIT_RANGE_FROM_LAST_MERGE + CI_CPU_ARCH="x86_64" +fi + +export CI_COMMIT=$CI_COMMIT +export CI_COMMIT_RANGE=$CI_COMMIT_RANGE +export CI_BRANCH=$CI_BRANCH +export CI_EVENT_TYPE=$CI_EVENT_TYPE +export CI_REPO_SLUG=$CI_REPO_SLUG +export CI_CPU_ARCH=$CI_CPU_ARCH + +echo CI_COMMIT=$CI_COMMIT +echo CI_COMMIT_RANGE=$CI_COMMIT_RANGE +echo CI_BRANCH=$CI_BRANCH +echo CI_EVENT_TYPE=$CI_EVENT_TYPE +echo CI_REPO_SLUG=$CI_REPO_SLUG +echo CI_CPU_ARCH=$CI_CPU_ARCH diff --git a/src/pmdk/utils/docker/set-vars.sh b/src/pmdk/utils/docker/set-vars.sh new file mode 100755 index 000000000..9a531a840 --- /dev/null +++ b/src/pmdk/utils/docker/set-vars.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2019, Intel Corporation + +# +# set-vars.sh - set required environment variables +# + +set -e + +export CI_FILE_PUSH_IMAGE_TO_REPO=/tmp/push_image_to_repo_flag +export CI_FILE_SKIP_BUILD_PKG_CHECK=/tmp/skip_build_package_check diff --git a/src/pmdk/utils/docker/test_package/.gitignore b/src/pmdk/utils/docker/test_package/.gitignore new file mode 100644 index 000000000..7e564dc4e --- /dev/null +++ b/src/pmdk/utils/docker/test_package/.gitignore @@ -0,0 +1 @@ +test_package diff --git a/src/pmdk/utils/docker/test_package/Makefile b/src/pmdk/utils/docker/test_package/Makefile new file mode 100644 index 000000000..f5970aac3 --- /dev/null +++ b/src/pmdk/utils/docker/test_package/Makefile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018, Intel Corporation + +check_package = $(shell pkg-config $(1) && echo y || echo n) + +HAS_LIBPMEMOBJ := $(call check_package, libpmemobj --atleast-version $(LIBPMEMOBJ_MIN_VERSION) --print-errors) + +LIBS += $(shell pkg-config --libs libpmemobj) +CFLAGS += $(shell pkg-config --cflags libpmemobj) + +ifeq ($(HAS_LIBPMEMOBJ),n) +$(error libpmemobj(version >= $(LIBPMEMOBJ_MIN_VERSION)) is missing) +endif + +test_package: test_package.c + $(CC) test_package.c $(LIBS) $(CFLAGS) -o $@ + +clean: + rm -f test_package + +.PHONY: clean diff --git a/src/pmdk/utils/docker/test_package/README b/src/pmdk/utils/docker/test_package/README new file mode 100644 index 000000000..de81a6a5b --- /dev/null +++ b/src/pmdk/utils/docker/test_package/README @@ -0,0 +1,6 @@ +Persistent Memory Development Kit + +This is utils/docker/test_package/README. + +This directory contains simple application which uses libpmemobj. +It can be used to test whether libpmemobj was installed properly. diff --git a/src/pmdk/utils/docker/test_package/test_package.c b/src/pmdk/utils/docker/test_package/test_package.c new file mode 100644 index 000000000..cf1e7eb8b --- /dev/null +++ b/src/pmdk/utils/docker/test_package/test_package.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2018, Intel Corporation */ + +#include <libpmemobj.h> +#include <stdio.h> +#include <sys/stat.h> + +#define LAYOUT_NAME "test" + +struct my_root { + int foo; +}; + +int +main(int argc, char *argv[]) +{ + if (argc < 2) { + printf("usage: %s file-name\n", argv[0]); + return 1; + } + + const char *path = argv[1]; + + PMEMobjpool *pop = pmemobj_create(path, LAYOUT_NAME, + PMEMOBJ_MIN_POOL, S_IWUSR | S_IRUSR); + + if (pop == NULL) { + printf("failed to create pool\n"); + return 1; + } + + PMEMoid root = pmemobj_root(pop, sizeof(struct my_root)); + struct my_root *rootp = pmemobj_direct(root); + + rootp->foo = 10; + pmemobj_persist(pop, &rootp->foo, sizeof(rootp->foo)); + + pmemobj_close(pop); + + return 0; +} diff --git a/src/pmdk/utils/docker/valid-branches.sh b/src/pmdk/utils/docker/valid-branches.sh new file mode 100755 index 000000000..b111c2a3d --- /dev/null +++ b/src/pmdk/utils/docker/valid-branches.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018-2020, Intel Corporation + +declare -A TARGET_BRANCHES=( \ + ["master"]="master" \ + ["stable-1.5"]="v1.5" \ + ["stable-1.6"]="v1.6" \ + ["stable-1.7"]="v1.7" \ + ["stable-1.8"]="v1.8" \ + ["stable-1.9"]="v1.9" \ + ) |