10 files changed, 572 insertions, 0 deletions
diff --git a/src/pmdk/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch b/src/pmdk/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch
new file mode 100644
index 000000000..7377d07f6
--- /dev/null
+++ b/src/pmdk/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch
@@ -0,0 +1,37 @@
+From d633d3b0a5f03be280efb80a69b9d5ed4e9c4d56 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= <lukasz.stolarczuk@intel.com>
+Date: Tue, 14 Jul 2020 13:58:34 +0200
+Subject: [PATCH] fix generating gcov files and turn-off verbose log
+
+---
+ codecov | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/codecov b/codecov
+index e702ecd..0a2f4d8 100755
+--- a/codecov
++++ b/codecov
+@@ -1108,9 +1108,9 @@ then
+     if [ "$ft_gcovout" = "0" ];
+     then
+       # suppress gcov output
+-      bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -exec $gcov_exe -pb $gcov_arg {} +" >/dev/null 2>&1 || true
++      bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -execdir $gcov_exe -pb $gcov_arg {} \;" >/dev/null 2>&1 || true
+     else
+-      bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -exec $gcov_exe -pb $gcov_arg {} +" || true
++      bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -execdir $gcov_exe -pb $gcov_arg {} \;" || true
+     fi
+   else
+     say "${e}==>${x} gcov disabled"
+@@ -1425,7 +1425,7 @@ do
+       report_len=$(wc -c < "$file")
+       if [ "$report_len" -ne 0 ];
+       then
+-        say "    ${g}+${x} $file ${e}bytes=$(echo "$report_len" | tr -d ' ')${x}"
++        #say "    ${g}+${x} $file ${e}bytes=$(echo "$report_len" | tr -d ' ')${x}"
+         # append to to upload
+         _filename=$(basename "$file")
+         if [ "${_filename##*.}" = 'gcov' ];
+-- 
+2.25.1
+
diff --git a/src/pmdk/utils/docker/images/Dockerfile.fedora-31 b/src/pmdk/utils/docker/images/Dockerfile.fedora-31
new file mode 100644
index 000000000..a699fee85
--- /dev/null
+++ b/src/pmdk/utils/docker/images/Dockerfile.fedora-31
@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2016-2020, Intel Corporation
+
+#
+# Dockerfile - a 'recipe' for Docker to build an image of fedora-based
+#              environment for building the PMDK project.
+#
+
+# Pull base image
+FROM fedora:31
+MAINTAINER piotr.balcer@intel.com
+
+# libfabric (optional if libfabric-dev >= 1.4.2 is installed)
+ENV FABRIC_DEPS "\
+	autoconf \
+	automake \
+	libtool \
+	wget"
+
+ENV VALGRIND_DEPS "\
+	autoconf \
+	automake \
+	file \
+	findutils \
+	git"
+
+# pmdk base
+ENV BASE_DEPS "\
+	git \
+	daxctl-devel \
+	make \
+	ndctl-devel \
+	pkgconfig"
+
+# benchmarks (optional)
+ENV BENCH_DEPS "\
+	glib2-devel"
+
+# examples (optional)
+ENV EXAMPLES_DEPS "\
+	fuse \
+	fuse-devel \
+	ncurses-devel \
+	libuv-devel"
+
+# documentation (optional)
+ENV DOC_DEPS "\
+	pandoc"
+
+# tests
+ENV TESTS_DEPS "\
+	bc \
+	gdb \
+	libunwind-devel \
+	ndctl \
+	openssh-server \
+	strace"
+
+# packaging
+ENV PACKAGING_DEPS "\
+	rpm-build \
+	rpm-build-libs \
+	rpmdevtools"
+
+# Coverity
+ENV COVERITY_DEPS "\
+	gcc \
+	wget"
+
+# misc
+ENV MISC_DEPS "\
+	clang \
+	hub \
+	lbzip2 \
+	man \
+	python3-flake8 \
+	rsync \
+	shadow-utils \
+	sudo \
+	tar \
+	which \
+	xmlto"
+
+# Copy install valgrind script
+COPY install-valgrind.sh install-valgrind.sh
+
+# Copy install libfabric script
+COPY install-libfabric.sh install-libfabric.sh
+
+RUN dnf update -y && dnf install -y \
+	$FABRIC_DEPS \
+	$VALGRIND_DEPS \
+	$BASE_DEPS \
+	$BENCH_DEPS \
+	$EXAMPLES_DEPS \
+	$DOC_DEPS \
+	$TESTS_DEPS \
+	$PACKAGING_DEPS \
+	$COVERITY_DEPS \
+	$MISC_DEPS \
+	$TESTS_DEPS \
+	&& ./install-valgrind.sh fedora \
+	&& ./install-libfabric.sh fedora \
+	&& dnf clean all
+
+# Add user
+ENV USER pmdkuser
+ENV USERPASS pmdkpass
+RUN useradd -m $USER
+RUN echo "$USER:$USERPASS" | chpasswd
+RUN gpasswd wheel -a $USER
+RUN echo "%wheel ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+USER $USER
+
+# Set required environment variables
+ENV OS fedora
+ENV OS_VER 31
+ENV START_SSH_COMMAND /usr/sbin/sshd
+ENV PACKAGE_MANAGER rpm
+ENV NOTTY 1
diff --git a/src/pmdk/utils/docker/images/Dockerfile.ubuntu-19.10 b/src/pmdk/utils/docker/images/Dockerfile.ubuntu-19.10
new file mode 100644
index 000000000..9b61f8953
--- /dev/null
+++ b/src/pmdk/utils/docker/images/Dockerfile.ubuntu-19.10
@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2016-2020, Intel Corporation
+
+#
+# Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based
+#              environment for building the PMDK project.
+#
+
+# Pull base image
+FROM ubuntu:19.10
+MAINTAINER piotr.balcer@intel.com
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Additional parameters to build docker without building components
+ARG SKIP_SCRIPTS_DOWNLOAD
+
+# libfabric (optional if libfabric-dev >= 1.4.2 is installed)
+ENV FABRIC_DEPS "autoconf \
+	automake \
+	build-essential \
+	libtool \
+	unzip \
+	wget"
+
+ENV VALGRIND_DEPS "autoconf \
+	automake \
+	build-essential \
+	git"
+
+# pmdk base
+ENV BASE_DEPS "build-essential \
+	git \
+	libdaxctl-dev \
+	libndctl-dev \
+	pkg-config"
+
+# benchmarks (optional)
+ENV BENCH_DEPS libglib2.0-dev
+
+# examples (optional)
+ENV EXAMPLES_DEPS "libfuse-dev \
+	libncurses5-dev \
+	libuv1-dev"
+
+# documentation (optional)
+ENV DOC_DEPS pandoc
+
+# tests
+ENV TESTS_DEPS "bc \
+	gdb \
+	libc6-dbg \
+	libunwind-dev \
+	ndctl \
+	python3 \
+	ssh \
+	strace"
+
+# packaging
+ENV PACKAGING_DEPS "debhelper \
+	devscripts \
+	fakeroot"
+
+# CodeCov
+ENV CODECOV_DEPS curl
+
+# Coverity
+ENV COVERITY_DEPS ruby gcc g++ wget
+
+# misc
+ENV MISC_DEPS "clang \
+	clang-format \
+	flake8 \
+	sudo \
+	whois"
+
+# Copy install valgrind script
+COPY install-valgrind.sh install-valgrind.sh
+
+# Copy install libfabric script
+COPY install-libfabric.sh install-libfabric.sh
+
+# Copy codecov patch and script to download scripts required in run-*.sh
+COPY download-scripts.sh download-scripts.sh
+COPY 0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch \
+	0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch
+
+# Update the Apt cache and install basic tools
+RUN apt-get update && apt-get dist-upgrade -y \
+	&& apt-get install -y --no-install-recommends \
+	$FABRIC_DEPS \
+	$VALGRIND_DEPS \
+	$BASE_DEPS \
+	$BENCH_DEPS \
+	$EXAMPLES_DEPS \
+	$DOC_DEPS \
+	$TESTS_DEPS \
+	$PACKAGING_DEPS \
+	$CODECOV_DEPS \
+	$COVERITY_DEPS \
+	$MISC_DEPS \
+	&& ./install-valgrind.sh ubuntu \
+	&& ./install-libfabric.sh \
+	&& ./download-scripts.sh \
+	&& rm -rf /var/lib/apt/lists/*
+
+# Add user
+ENV USER pmdkuser
+ENV USERPASS pmdkpass
+RUN useradd -m $USER -g sudo -p `mkpasswd $USERPASS`
+RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+# switch user
+USER $USER
+
+# Set required environment variables
+ENV OS ubuntu
+ENV OS_VER 19.10
+ENV START_SSH_COMMAND service ssh start
+ENV PACKAGE_MANAGER dpkg
+ENV NOTTY 1
diff --git a/src/pmdk/utils/docker/images/README b/src/pmdk/utils/docker/images/README
new file mode 100644
index 000000000..0b8c551e4
--- /dev/null
+++ b/src/pmdk/utils/docker/images/README
@@ -0,0 +1,6 @@
+Persistent Memory Development Kit
+
+This is utils/docker/images/README.
+
+Scripts in this directory let you prepare Docker images for building
+PMDK project under specified OS (ubuntu, fedora).
diff --git a/src/pmdk/utils/docker/images/build-image.sh b/src/pmdk/utils/docker/images/build-image.sh
new file mode 100755
index 000000000..b9e7a2a4b
--- /dev/null
+++ b/src/pmdk/utils/docker/images/build-image.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2016-2020, Intel Corporation
+
+#
+# build-image.sh <OS-VER> <ARCH> - prepares a Docker image with <OS>-based
+#                environment intended for the <ARCH> CPU architecture
+#                designed for building PMDK project, according to
+#                the Dockerfile.<OS-VER> file located in the same directory.
+#
+# The script can be run locally.
+#
+
+set -e
+
+OS_VER=$1
+CPU_ARCH=$2
+
+function usage {
+	echo "Usage:"
+	echo "    build-image.sh <OS-VER> <ARCH>"
+	echo "where:"
+	echo "  <OS-VER> - can be for example 'ubuntu-19.10' provided "\
+		"a Dockerfile named 'Dockerfile.ubuntu-19.10' "\
+		"exists in the current directory and"
+	echo "  <ARCH> - is a CPU architecture, for example 'x86_64'"
+}
+
+# Check if two first arguments are not empty
+if [[ -z "$2" ]]; then
+	usage
+	exit 1
+fi
+
+# Check if the file Dockerfile.OS-VER exists
+if [[ ! -f "Dockerfile.$OS_VER" ]]; then
+	echo "Error: Dockerfile.$OS_VER does not exist."
+	echo
+	usage
+	exit 1
+fi
+
+if [[ -z "${DOCKERHUB_REPO}" ]]; then
+	echo "Error: DOCKERHUB_REPO environment variable is not set"
+	exit 1
+fi
+
+# Build a Docker image tagged with ${DOCKERHUB_REPO}:OS-VER-ARCH
+tag=${DOCKERHUB_REPO}:1.10-${OS_VER}-${CPU_ARCH}
+docker build -t $tag \
+	--build-arg http_proxy=$http_proxy \
+	--build-arg https_proxy=$https_proxy \
+	-f Dockerfile.$OS_VER .
diff --git a/src/pmdk/utils/docker/images/download-scripts.sh b/src/pmdk/utils/docker/images/download-scripts.sh
new file mode 100755
index 000000000..fae37b464
--- /dev/null
+++ b/src/pmdk/utils/docker/images/download-scripts.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2020, Intel Corporation
+
+#
+# download-scripts.sh - downloads specific version of codecov's bash
+#		script to generate and upload reports. It's useful,
+#		since unverified version may break coverage results.
+#
+
+set -e
+
+# master: Merge pull request #342 from codecov/revert-proj-name-..., 18.08.2020
+CODECOV_VERSION="e877c1280cc6e902101fb5df2981ed1c962da7f0"
+
+if [ "${SKIP_SCRIPTS_DOWNLOAD}" ]; then
+	echo "Variable 'SKIP_SCRIPTS_DOWNLOAD' is set; skipping scripts' download"
+	exit
+fi
+
+mkdir -p /opt/scripts
+
+# Download codecov's bash script
+git clone https://github.com/codecov/codecov-bash
+cd codecov-bash
+git checkout $CODECOV_VERSION
+
+git apply ../0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch
+mv -v codecov /opt/scripts/codecov
+
+cd ..
+rm -rf codecov-bash
diff --git a/src/pmdk/utils/docker/images/install-libfabric.sh b/src/pmdk/utils/docker/images/install-libfabric.sh
new file mode 100755
index 000000000..355f24525
--- /dev/null
+++ b/src/pmdk/utils/docker/images/install-libfabric.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2016-2020, Intel Corporation
+
+#
+# install-libfabric.sh - installs a customized version of libfabric
+#
+
+set -e
+
+OS=$1
+
+# Keep in sync with requirements in src/common.inc.
+libfabric_ver=1.4.2
+libfabric_url=https://github.com/ofiwg/libfabric/archive
+libfabric_dir=libfabric-$libfabric_ver
+libfabric_tarball=v${libfabric_ver}.zip
+wget "${libfabric_url}/${libfabric_tarball}"
+unzip $libfabric_tarball
+
+cd $libfabric_dir
+
+# XXX HACK HACK HACK
+# Disable use of spin locks in libfabric.
+#
+# spinlocks do not play well (IOW at all) with cpu-constrained environments,
+# like GitHub Actions, and this leads to timeouts of some PMDK's tests.
+# This change speeds up pmempool_sync_remote/TEST28-31 by a factor of 20-30.
+#
+perl -pi -e 's/have_spinlock=1/have_spinlock=0/' configure.ac
+# XXX HACK HACK HACK
+
+./autogen.sh
+./configure --prefix=/usr --enable-sockets
+make -j$(nproc)
+make -j$(nproc) install
+
+cd ..
+rm -f ${libfabric_tarball}
+rm -rf ${libfabric_dir}
diff --git a/src/pmdk/utils/docker/images/install-libndctl.sh b/src/pmdk/utils/docker/images/install-libndctl.sh
new file mode 100755
index 000000000..fc8cee3fb
--- /dev/null
+++ b/src/pmdk/utils/docker/images/install-libndctl.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2017-2019, Intel Corporation
+
+#
+# install-libndctl.sh - installs libndctl
+#
+
+set -e
+
+OS=$2
+
+echo "==== clone ndctl repo ===="
+git clone https://github.com/pmem/ndctl.git
+cd ndctl
+git checkout $1
+
+if [ "$OS" = "fedora" ]; then
+
+echo "==== setup rpmbuild tree ===="
+rpmdev-setuptree
+
+RPMDIR=$HOME/rpmbuild/
+VERSION=$(./git-version)
+SPEC=./rhel/ndctl.spec
+
+echo "==== create source tarball ====="
+git archive --format=tar --prefix="ndctl-${VERSION}/" HEAD | gzip > "$RPMDIR/SOURCES/ndctl-${VERSION}.tar.gz"
+
+echo "==== build ndctl ===="
+./autogen.sh
+./configure --disable-docs
+make -j$(nproc)
+
+echo "==== build ndctl packages ===="
+rpmbuild -ba $SPEC
+
+echo "==== install ndctl packages ===="
+RPM_ARCH=$(uname -m)
+rpm -i $RPMDIR/RPMS/$RPM_ARCH/*.rpm
+
+echo "==== cleanup ===="
+rm -rf $RPMDIR
+
+else
+
+echo "==== build ndctl ===="
+./autogen.sh
+./configure --disable-docs
+make -j$(nproc)
+
+echo "==== install ndctl ===="
+make -j$(nproc) install
+
+echo "==== cleanup ===="
+
+fi
+
+cd ..
+rm -rf ndctl
diff --git a/src/pmdk/utils/docker/images/install-valgrind.sh b/src/pmdk/utils/docker/images/install-valgrind.sh
new file mode 100755
index 000000000..97babcb67
--- /dev/null
+++ b/src/pmdk/utils/docker/images/install-valgrind.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2016-2020, Intel Corporation
+
+#
+# install-valgrind.sh - installs valgrind for persistent memory
+#
+
+set -e
+
+OS=$1
+
+install_upstream_from_distro() {
+  case "$OS" in
+    fedora) dnf install -y valgrind ;;
+    ubuntu) apt-get install -y --no-install-recommends valgrind ;;
+    *) return 1 ;;
+  esac
+}
+
+install_upstream_3_16_1() {
+  git clone git://sourceware.org/git/valgrind.git
+  cd valgrind
+  # valgrind v3.16.1 upstream
+  git checkout VALGRIND_3_16_BRANCH
+  ./autogen.sh
+  ./configure
+  make -j$(nproc)
+  make -j$(nproc) install
+  cd ..
+  rm -rf valgrind
+}
+
+install_custom-pmem_from_source() {
+  git clone https://github.com/pmem/valgrind.git
+  cd valgrind
+  # valgrind v3.15 with pmemcheck
+  # 2020.04.01 Merge pull request #78 from marcinslusarz/opt3
+  git checkout 759686fd66cc0105df8311cfe676b0b2f9e89196
+  ./autogen.sh
+  ./configure
+  make -j$(nproc)
+  make -j$(nproc) install
+  cd ..
+  rm -rf valgrind
+}
+
+ARCH=$(uname -m)
+case "$ARCH" in
+  ppc64le) install_upstream_3_16_1 ;;
+  *) install_custom-pmem_from_source ;;
+esac
diff --git a/src/pmdk/utils/docker/images/push-image.sh b/src/pmdk/utils/docker/images/push-image.sh
new file mode 100755
index 000000000..03b935213
--- /dev/null
+++ b/src/pmdk/utils/docker/images/push-image.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2016-2020, Intel Corporation
+
+#
+# push-image.sh - pushes the Docker image to the Docker Hub.
+#
+# The script utilizes $DOCKERHUB_USER and $DOCKERHUB_PASSWORD variables
+# to log in to Docker Hub. The variables can be set in the Travis project's
+# configuration for automated builds.
+#
+
+set -e
+
+source $(dirname $0)/../set-ci-vars.sh
+
+if [[ -z "$OS" ]]; then
+	echo "OS environment variable is not set"
+	exit 1
+fi
+
+if [[ -z "$OS_VER" ]]; then
+	echo "OS_VER environment variable is not set"
+	exit 1
+fi
+
+if [[ -z "$CI_CPU_ARCH" ]]; then
+	echo "CI_CPU_ARCH environment variable is not set"
+	exit 1
+fi
+
+if [[ -z "${DOCKERHUB_REPO}" ]]; then
+	echo "DOCKERHUB_REPO environment variable is not set"
+	exit 1
+fi
+
+TAG="1.10-${OS}-${OS_VER}-${CI_CPU_ARCH}"
+
+# Check if the image tagged with pmdk/OS-VER exists locally
+if [[ ! $(docker images -a | awk -v pattern="^${DOCKERHUB_REPO}:${TAG}\$" \
+	'$1":"$2 ~ pattern') ]]
+then
+	echo "ERROR: Docker image tagged ${DOCKERHUB_REPO}:${TAG} does not exists locally."
+	exit 1
+fi
+
+# Log in to the Docker Hub
+docker login -u="$DOCKERHUB_USER" -p="$DOCKERHUB_PASSWORD"
+
+# Push the image to the repository
+docker push ${DOCKERHUB_REPO}:${TAG}