Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

10 files changed, 455 insertions, 0 deletions

diff --git a/src/spdk/scripts/pkgdep.sh b/src/spdk/scripts/pkgdep.sh
new file mode 100755
index 000000000..7d748c437
--- /dev/null
+++ b/src/spdk/scripts/pkgdep.sh
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# Please run this script as root.
+
+set -e
+
+function usage() {
+	echo ""
+	echo "This script is intended to automate the installation of package dependencies to build SPDK."
+	echo "Please run this script as root user or with sudo -E."
+	echo ""
+	echo "$0"
+	echo "  -h --help"
+	echo "  -a --all"
+	echo "  -d --developer-tools        Install tools for developers (code styling, code coverage, etc.)"
+	echo "  -p --pmem                   Additional dependencies for reduce and pmdk"
+	echo "  -f --fuse                   Additional dependencies for FUSE and NVMe-CUSE"
+	echo "  -r --rdma                   Additional dependencies for RDMA transport in NVMe over Fabrics"
+	echo "  -b --docs                   Additional dependencies for building docs"
+	echo "  -u --uring                  Additional dependencies for io_uring"
+	echo ""
+	exit 0
+}
+
+function install_all_dependencies() {
+	INSTALL_DEV_TOOLS=true
+	INSTALL_PMEM=true
+	INSTALL_FUSE=true
+	INSTALL_RDMA=true
+	INSTALL_DOCS=true
+	INSTALL_LIBURING=true
+}
+
+function install_liburing() {
+	local GIT_REPO_LIBURING=https://github.com/axboe/liburing.git
+	local liburing_dir=/usr/local/src/liburing
+
+	if [[ -e /usr/lib64/liburing.so ]]; then
+		echo "liburing is already installed. skipping"
+	else
+		if [[ -d $liburing_dir ]]; then
+			echo "liburing source already present, not cloning"
+		else
+			mkdir $liburing_dir
+			git clone "${GIT_REPO_LIBURING}" "$liburing_dir"
+		fi
+		(cd "$liburing_dir" && ./configure --libdir=/usr/lib64 && make install)
+	fi
+}
+
+function install_shfmt() {
+	# Fetch version that has been tested
+	local shfmt_version=3.1.0
+	local shfmt=shfmt-$shfmt_version
+	local shfmt_dir=${SHFMT_DIR:-/opt/shfmt}
+	local shfmt_dir_out=${SHFMT_DIR_OUT:-/usr/bin}
+	local shfmt_url
+	local os
+
+	if hash "$shfmt" && [[ $("$shfmt" --version) == "v$shfmt_version" ]]; then
+		echo "$shfmt already installed"
+		return 0
+	fi 2> /dev/null
+
+	os=$(uname -s)
+
+	case "$os" in
+		Linux) shfmt_url=https://github.com/mvdan/sh/releases/download/v$shfmt_version/shfmt_v${shfmt_version}_linux_amd64 ;;
+		FreeBSD) shfmt_url=https://github.com/mvdan/sh/releases/download/v$shfmt_version/shfmt_v${shfmt_version}_freebsd_amd64 ;;
+		*)
+			echo "Not supported OS (${os:-Unknown}), skipping"
+			return 0
+			;;
+	esac
+
+	mkdir -p "$shfmt_dir"
+	mkdir -p "$shfmt_dir_out"
+
+	echo "Fetching ${shfmt_url##*/}"...
+	local err
+	if err=$(curl -f -Lo"$shfmt_dir/$shfmt" "$shfmt_url" 2>&1); then
+		chmod +x "$shfmt_dir/$shfmt"
+		ln -sf "$shfmt_dir/$shfmt" "$shfmt_dir_out"
+	else
+		cat <<- CURL_ERR
+
+			* Fetching $shfmt_url failed, $shfmt will not be available for format check.
+			* Error:
+
+			$err
+
+		CURL_ERR
+		return 0
+	fi
+	echo "$shfmt installed"
+}
+
+INSTALL_CRYPTO=false
+INSTALL_DEV_TOOLS=false
+INSTALL_PMEM=false
+INSTALL_FUSE=false
+INSTALL_RDMA=false
+INSTALL_DOCS=false
+INSTALL_LIBURING=false
+
+while getopts 'abdfhipru-:' optchar; do
+	case "$optchar" in
+		-)
+			case "$OPTARG" in
+				help) usage ;;
+				all) install_all_dependencies ;;
+				developer-tools) INSTALL_DEV_TOOLS=true ;;
+				pmem) INSTALL_PMEM=true ;;
+				fuse) INSTALL_FUSE=true ;;
+				rdma) INSTALL_RDMA=true ;;
+				docs) INSTALL_DOCS=true ;;
+				uring) INSTALL_LIBURING=true ;;
+				*)
+					echo "Invalid argument '$OPTARG'"
+					usage
+					;;
+			esac
+			;;
+		h) usage ;;
+		a) install_all_dependencies ;;
+		d) INSTALL_DEV_TOOLS=true ;;
+		p) INSTALL_PMEM=true ;;
+		f) INSTALL_FUSE=true ;;
+		r) INSTALL_RDMA=true ;;
+		b) INSTALL_DOCS=true ;;
+		u) INSTALL_LIBURING=true ;;
+		*)
+			echo "Invalid argument '$OPTARG'"
+			usage
+			;;
+	esac
+done
+
+trap 'set +e; trap - ERR; echo "Error!"; exit 1;' ERR
+
+scriptsdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $scriptsdir/..)
+
+OS=$(uname -s)
+
+if [[ -e /etc/os-release ]]; then
+	source /etc/os-release
+fi
+
+ID=${ID:-$OS} ID=${ID,,}
+
+#Link suse related OS to sles
+if [[ ${ID,,} == *"suse"* ]]; then
+	ID="sles"
+fi
+
+if [[ -e $scriptsdir/pkgdep/$ID.sh ]]; then
+	source "$scriptsdir/pkgdep/$ID.sh"
+else
+	printf 'Not supported platform detected (%s), aborting\n' "$ID" >&2
+fi
diff --git a/src/spdk/scripts/pkgdep/arch.sh b/src/spdk/scripts/pkgdep/arch.sh
new file mode 100755
index 000000000..53e5d8d7d
--- /dev/null
+++ b/src/spdk/scripts/pkgdep/arch.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# Install main dependencies
+pacman -Sy --needed --noconfirm gcc make cunit libaio openssl \
+	libutil-linux libiscsi python ncurses ninja meson
+# Additional dependencies for SPDK CLI
+pacman -Sy --needed --noconfirm python-pexpect python-pip libffi
+pip install configshell_fb
+# Additional dependencies for DPDK
+pacman -Sy --needed --noconfirm numactl nasm
+# Additional dependencies for ISA-L used in compression
+pacman -Sy --needed --noconfirm autoconf automake libtool help2man
+if [[ $INSTALL_DEV_TOOLS == "true" ]]; then
+	# Tools for developers
+	pacman -Sy --needed --noconfirm git astyle autopep8 \
+		clang sg3_utils pciutils shellcheck
+	#fakeroot needed to instal via makepkg
+	pacman -Sy --needed --noconfirm fakeroot
+	su - $SUDO_USER -c "pushd /tmp;
+		git clone https://aur.archlinux.org/perl-perlio-gzip.git;
+		cd perl-perlio-gzip;
+		yes y | makepkg -si --needed;
+		cd ..; rm -rf perl-perlio-gzip
+		popd"
+	# sed is to modify sources section in PKGBUILD
+	# By default it uses git:// which will fail behind proxy, so
+	# redirect it to http:// source instead
+	su - $SUDO_USER -c "pushd /tmp;
+		git clone https://aur.archlinux.org/lcov-git.git;
+		cd lcov-git;
+		sed -i 's/git:/git+http:/' PKGBUILD;
+		makepkg -si --needed --noconfirm;
+		cd .. && rm -rf lcov-git;
+		popd"
+	install_shfmt
+fi
+if [[ $INSTALL_PMEM == "true" ]]; then
+	# Additional dependencies for building pmem based backends
+	pacman -Sy --needed --noconfirm ndctl pkg-config
+	git clone https://github.com/pmem/pmdk.git /tmp/pmdk -b 1.6.1
+	make -C /tmp/pmdk -j$(nproc)
+	make install prefix=/usr -C /tmp/pmdk
+	echo "/usr/local/lib" > /etc/ld.so.conf.d/pmdk.conf
+	ldconfig
+	rm -rf /tmp/pmdk
+fi
+if [[ $INSTALL_FUSE == "true" ]]; then
+	# Additional dependencies for FUSE and NVMe-CUSE
+	pacman -Sy --needed --noconfirm fuse3
+fi
+if [[ $INSTALL_RDMA == "true" ]]; then
+	# Additional dependencies for RDMA transport in NVMe over Fabrics
+	if [[ -n "$http_proxy" ]]; then
+		gpg_options=" --keyserver hkp://pgp.mit.edu:11371 --keyserver-options \"http-proxy=$http_proxy\""
+	fi
+	su - $SUDO_USER -c "gpg $gpg_options --recv-keys 29F0D86B9C1019B1"
+	su - $SUDO_USER -c "pushd /tmp;
+		git clone https://aur.archlinux.org/rdma-core.git;
+		cd rdma-core;
+		makepkg -si --needed --noconfirm;
+		cd .. && rm -rf rdma-core;
+		popd"
+fi
+if [[ $INSTALL_DOCS == "true" ]]; then
+	# Additional dependencies for building docs
+	pacman -Sy --needed --noconfirm doxygen graphviz
+	pacman -S --noconfirm --needed gd ttf-font
+	su - $SUDO_USER -c "pushd /tmp;
+		git clone https://aur.archlinux.org/mscgen.git;
+		cd mscgen;
+		makepkg -si --needed --noconfirm;
+		cd .. && rm -rf mscgen;
+		popd"
+fi
+if [[ $INSTALL_LIBURING == "true" ]]; then
+	install_liburing
+fi
diff --git a/src/spdk/scripts/pkgdep/centos.sh b/src/spdk/scripts/pkgdep/centos.sh
new file mode 120000
index 000000000..0f75e52a1
--- /dev/null
+++ b/src/spdk/scripts/pkgdep/centos.sh
@@ -0,0 +1 @@
+rhel.sh
\ No newline at end of file
diff --git a/src/spdk/scripts/pkgdep/clear-linux-os.sh b/src/spdk/scripts/pkgdep/clear-linux-os.sh
new file mode 100755
index 000000000..a79a67e79
--- /dev/null
+++ b/src/spdk/scripts/pkgdep/clear-linux-os.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Install main dependencies
+swupd bundle-add -y c-basic make dev-utils openssl devpkg-libiscsi \
+	devpkg-ncurses python3-basic python-extras devpkg-open-iscsi \
+	storage-utils
+# Additional dependencies for ISA-L used in compression
+swupd bundle-add -y dev-utils-dev
+# Additional dependencies for DPDK
+swupd bundle-add -y nasm sysadmin-basic
+# Additional dependencies for SPDK CLI
+pip3 install pexpect
+pip3 install configshell_fb
+if [[ $INSTALL_DEV_TOOLS == "true" ]]; then
+	swupd bundle-add -y git os-testsuite-0day
+	install_shfmt
+fi
+if [[ $INSTALL_PMEM == "true" ]]; then
+	# Additional dependencies for building pmem based backends
+	swupd bundle-add -y devpkg-pmdk
+fi
+if [[ $INSTALL_FUSE == "true" ]]; then
+	# Additional dependencies for FUSE and NVMe-CUSE
+	swupd bundle-add -y devpkg-fuse
+fi
+if [[ $INSTALL_RDMA == "true" ]]; then
+	# Additional dependencies for RDMA transport in NVMe over Fabrics
+	swupd bundle-add -y devpkg-rdma-core network-basic-dev
+fi
+if [[ $INSTALL_DOCS == "true" ]]; then
+	# Additional dependencies for building docs
+	swupd bundle-add -y doxygen graphviz
+fi
diff --git a/src/spdk/scripts/pkgdep/debian.sh b/src/spdk/scripts/pkgdep/debian.sh
new file mode 100755
index 000000000..2513a0160
--- /dev/null
+++ b/src/spdk/scripts/pkgdep/debian.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+VERSION_ID_NUM=$(sed 's/\.//g' <<< $VERSION_ID)
+# Includes Ubuntu, Debian
+# Minimal install
+apt-get install -y gcc g++ make libcunit1-dev libaio-dev libssl-dev \
+	uuid-dev libiscsi-dev python libncurses5-dev libncursesw5-dev python3-pip
+pip3 install ninja
+pip3 install meson
+# Additional dependencies for SPDK CLI - not available on older Ubuntus
+apt-get install -y python3-configshell-fb python3-pexpect || echo \
+	"Note: Some SPDK CLI dependencies could not be installed."
+
+# Additional dependencies for DPDK
+if [[ $NAME == "Ubuntu" ]] && [[ $VERSION_ID_NUM -lt 1900 ]]; then
+	echo "Ubuntu $VERSION_ID needs NASM version 2.13.03 for DPDK but is not in the mainline repository."
+	echo "You can install it manually"
+else
+	apt-get install -y nasm
+fi
+apt-get install -y libnuma-dev
+# Additional dependencies for ISA-L used in compression
+apt-get install -y autoconf automake libtool help2man
+if [[ $INSTALL_DEV_TOOLS == "true" ]]; then
+	# Tools for developers
+	apt-get install -y git astyle pep8 lcov clang sg3-utils pciutils shellcheck
+	# Additional python style checker not available on ubuntu 16.04 or earlier.
+	apt-get install -y pycodestyle || true
+	# Additional dependecies for nvmf performance test script
+	apt-get install -y python3-paramiko
+	install_shfmt
+fi
+if [[ $INSTALL_PMEM == "true" ]]; then
+	# Additional dependencies for building pmem based backends
+	if [[ $NAME == "Ubuntu" ]] && [[ $VERSION_ID_NUM -gt 1800 ]]; then
+		apt-get install -y libpmem-dev
+	fi
+fi
+if [[ $INSTALL_FUSE == "true" ]]; then
+	# Additional dependencies for FUSE and NVMe-CUSE
+	if [[ $NAME == "Ubuntu" ]] && ((VERSION_ID_NUM > 1400 && VERSION_ID_NUM < 1900)); then
+		echo "Ubuntu $VERSION_ID does not have libfuse3-dev in mainline repository."
+		echo "You can install it manually"
+	else
+		apt-get install -y libfuse3-dev
+	fi
+fi
+if [[ $INSTALL_RDMA == "true" ]]; then
+	# Additional dependencies for RDMA transport in NVMe over Fabrics
+	apt-get install -y libibverbs-dev librdmacm-dev
+fi
+if [[ $INSTALL_DOCS == "true" ]]; then
+	# Additional dependencies for building docs
+	apt-get install -y doxygen mscgen graphviz
+fi
+if [[ $INSTALL_LIBURING == "true" ]]; then
+	install_liburing
+fi
diff --git a/src/spdk/scripts/pkgdep/fedora.sh b/src/spdk/scripts/pkgdep/fedora.sh
new file mode 120000
index 000000000..0f75e52a1
--- /dev/null
+++ b/src/spdk/scripts/pkgdep/fedora.sh
@@ -0,0 +1 @@
+rhel.sh
\ No newline at end of file
diff --git a/src/spdk/scripts/pkgdep/freebsd.sh b/src/spdk/scripts/pkgdep/freebsd.sh
new file mode 100755
index 000000000..032170b6d
--- /dev/null
+++ b/src/spdk/scripts/pkgdep/freebsd.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# Minimal install
+pkg install -y gmake cunit openssl git bash misc/e2fsprogs-libuuid python \
+	ncurses ninja meson
+# Additional dependencies for ISA-L used in compression
+pkg install -y autoconf automake libtool help2man
+if [[ $INSTALL_DEV_TOOLS == "true" ]]; then
+	# Tools for developers
+	pkg install -y devel/astyle bash py27-pycodestyle \
+		misc/e2fsprogs-libuuid sysutils/sg3_utils nasm
+	install_shfmt
+fi
+if [[ $INSTALL_DOCS == "true" ]]; then
+	# Additional dependencies for building docs
+	pkg install -y doxygen mscgen graphviz
+fi
diff --git a/src/spdk/scripts/pkgdep/rhel.sh b/src/spdk/scripts/pkgdep/rhel.sh
new file mode 100755
index 000000000..af5d4c0b3
--- /dev/null
+++ b/src/spdk/scripts/pkgdep/rhel.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+# Minimal install
+if echo "$ID $VERSION_ID" | grep -E -q 'centos 8'; then
+	# Add PowerTools needed for install CUnit-devel in Centos8
+	yum install -y yum-utils
+	yum config-manager --set-enabled PowerTools
+fi
+yum install -y gcc gcc-c++ make CUnit-devel libaio-devel openssl-devel \
+	libuuid-devel libiscsi-devel ncurses-devel
+if echo "$ID $VERSION_ID" | grep -E -q 'centos 8'; then
+	yum install -y python36
+	#Create hard link to use in SPDK as python
+	ln /etc/alternatives/python3 /usr/bin/python || true
+else
+	yum install -y python
+fi
+yum install -y python3-pip
+pip-3 install ninja
+pip-3 install meson
+
+# Additional dependencies for SPDK CLI - not available in rhel and centos
+if ! echo "$ID $VERSION_ID" | grep -E -q 'rhel 7|centos 7'; then
+	yum install -y python3-configshell python3-pexpect
+fi
+# Additional dependencies for ISA-L used in compression
+yum install -y autoconf automake libtool help2man
+# Additional dependencies for DPDK
+yum install -y numactl-devel nasm
+if [[ $INSTALL_DEV_TOOLS == "true" ]]; then
+	# Tools for developers
+	# Includes Fedora, CentOS 7, RHEL 7
+	# Add EPEL repository for CUnit-devel
+	if echo "$ID $VERSION_ID" | grep -E -q 'rhel 7|centos 7|centos 8'; then
+		if ! rpm --quiet -q epel-release; then
+			yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+		fi
+
+		if [[ $ID = 'rhel' ]]; then
+			subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms"
+		elif [[ $ID = 'centos' ]]; then
+			yum --enablerepo=extras install -y epel-release
+		fi
+	fi
+	if echo "$ID $VERSION_ID" | grep -E -q 'centos 8'; then
+		yum install -y python3-pycodestyle
+		echo "Centos 8 does not have lcov and ShellCheck dependencies"
+	else
+		yum install -y python-pycodestyle lcov ShellCheck
+	fi
+	yum install -y git astyle sg3_utils pciutils
+	install_shfmt
+fi
+if [[ $INSTALL_PMEM == "true" ]]; then
+	# Additional dependencies for building pmem based backends
+	yum install -y libpmemblk-devel || true
+fi
+if [[ $INSTALL_FUSE == "true" ]]; then
+	# Additional dependencies for FUSE and NVMe-CUSE
+	yum install -y fuse3-devel
+fi
+if [[ $INSTALL_RDMA == "true" ]]; then
+	# Additional dependencies for RDMA transport in NVMe over Fabrics
+	yum install -y libibverbs-devel librdmacm-devel
+fi
+if [[ $INSTALL_DOCS == "true" ]]; then
+	# Additional dependencies for building docs
+	yum install -y mscgen || echo "Warning: couldn't install mscgen via yum. Please install mscgen manually."
+	yum install -y doxygen graphviz
+fi
+if [[ $INSTALL_LIBURING == "true" ]]; then
+	install_liburing
+fi
diff --git a/src/spdk/scripts/pkgdep/sles.sh b/src/spdk/scripts/pkgdep/sles.sh
new file mode 100755
index 000000000..dacf6d1b5
--- /dev/null
+++ b/src/spdk/scripts/pkgdep/sles.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Minimal install
+zypper install -y gcc gcc-c++ make cunit-devel libaio-devel libopenssl-devel \
+	libuuid-devel python-base ncurses-devel ninja meson
+# Additional dependencies for DPDK
+zypper install -y libnuma-devel nasm
+# Additional dependencies for ISA-L used in compression
+zypper install -y autoconf automake libtool help2man
+if [[ $INSTALL_DEV_TOOLS == "true" ]]; then
+	# Tools for developers
+	zypper install -y git-core lcov python-pycodestyle sg3_utils \
+		pciutils ShellCheck
+	install_shfmt
+fi
+if [[ $INSTALL_PMEM == "true" ]]; then
+	# Additional dependencies for building pmem based backends
+	zypper install -y libpmemblk-devel
+fi
+if [[ $INSTALL_FUSE == "true" ]]; then
+	# Additional dependencies for FUSE and NVMe-CUSE
+	zypper install -y fuse3-devel
+fi
+if [[ $INSTALL_RDMA == "true" ]]; then
+	# Additional dependencies for RDMA transport in NVMe over Fabrics
+	zypper install -y rdma-core-devel
+fi
+if [[ $INSTALL_DOCS == "true" ]]; then
+	# Additional dependencies for building docs
+	zypper install -y doxygen mscgen graphviz
+fi
+if [[ $INSTALL_LIBURING == "true" ]]; then
+	install_liburing
+fi
diff --git a/src/spdk/scripts/pkgdep/ubuntu.sh b/src/spdk/scripts/pkgdep/ubuntu.sh
new file mode 120000
index 000000000..0edcb8b83
--- /dev/null
+++ b/src/spdk/scripts/pkgdep/ubuntu.sh
@@ -0,0 +1 @@
+debian.sh
\ No newline at end of file