diff options
Diffstat (limited to 'src/spdk/test/common/config/pkgdep/git')
| -rw-r--r-- | src/spdk/test/common/config/pkgdep/git | 325 |
1 files changed, 325 insertions, 0 deletions
diff --git a/src/spdk/test/common/config/pkgdep/git b/src/spdk/test/common/config/pkgdep/git new file mode 100644 index 000000000..f46183ac8 --- /dev/null +++ b/src/spdk/test/common/config/pkgdep/git @@ -0,0 +1,325 @@ +function install_spdk() { + mkdir -p "$GIT_REPOS/spdk_repo/output" || echo "Can not create spdk_repo/output directory." + + if [[ -d $GIT_REPOS/spdk_repo/spdk ]]; then + echo "spdk source already present, not cloning" + else + git -C "$GIT_REPOS/spdk_repo" clone "${GIT_REPO_SPDK}" + fi + git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.dpdk.url "${GIT_REPO_DPDK}" + git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}" + git -C "$GIT_REPOS/spdk_repo/spdk" submodule update --init --recursive +} + +function install_refspdk() { + local last_release + local output_dir + local config_params + local rootdir + + # Create a reference SPDK build for ABI tests + git -C "$GIT_REPOS/spdk_repo/spdk" fetch --tags + last_release=$(git -C "$GIT_REPOS/spdk_repo/spdk" tag | sort --version-sort | grep -v rc | tail -n1) + output_dir="$GIT_REPOS/spdk_$(tr . _ < <(tr -d '[:alpha:]' <<< $last_release))" + + if [[ ! -d $output_dir ]]; then + cp -r "$GIT_REPOS/spdk_repo/spdk" "$output_dir" + fi + + git -C "$output_dir" checkout "$last_release" + git -C "$output_dir" submodule update --init + + cat > $HOME/autorun-spdk.conf <<- EOF + SPDK_BUILD_SHARED_OBJECT=1 + SPDK_TEST_AUTOBUILD=1 + SPDK_TEST_UNITTEST=1 + SPDK_TEST_BLOCKDEV=1 + SPDK_TEST_PMDK=1 + SPDK_TEST_ISAL=1 + SPDK_TEST_REDUCE=1 + SPDK_TEST_CRYPTO=1 + SPDK_TEST_FTL=1 + SPDK_TEST_OCF=1 + SPDK_TEST_RAID5=1 + SPDK_TEST_RBD=1 + SPDK_RUN_ASAN=1 + SPDK_RUN_UBSAN=1 + EOF + + mkdir -p $HOME/output + + ( + rootdir="$output_dir" + source $HOME/autorun-spdk.conf + source $output_dir/test/common/autotest_common.sh + + # Prepare separate, fixed, cmdline for the FreeBSD, Issue #1397. + if [[ $OSID == freebsd ]]; then + config_params="--enable-debug --enable-werror" + config_params+=" --with-idxd --with-fio=/usr/src/fio" + config_params+=" --disable-unit-tests --without-isal" + MAKE=gmake + else + config_params="$(get_config_params)" + fi + $output_dir/configure $(echo $config_params | sed 's/--enable-coverage//g') + if [[ $OSID != freebsd ]]; then + $MAKE -C $output_dir $MAKEFLAGS include/spdk/config.h + CONFIG_OCF_PATH="$output_dir/ocf" $MAKE -C $output_dir/lib/env_ocf $MAKEFLAGS exportlib O=$output_dir/build/ocf.a + $output_dir/configure $config_params --with-ocf=$output_dir/build/ocf.a --with-shared + fi + $MAKE -C $output_dir $MAKEFLAGS + ) +} + +function install_qat() { + + kernel_maj=$(uname -r | cut -d'.' -f1) + kernel_min=$(uname -r | cut -d'.' -f2) + + if [[ -e /sys/module/qat_c62x ]]; then + sudo modprobe -r qat_c62x || : + fi + if [[ -d $GIT_REPOS/QAT ]]; then + sudo rm -rf "$GIT_REPOS/QAT" + fi + + mkdir "$GIT_REPOS/QAT" + + tar -C "$GIT_REPOS/QAT" -xzof - < <(wget -O- "$DRIVER_LOCATION_QAT") + + #The driver version 1.7.l.4.3.0-00033 contains a reference to a deprecated function. Remove it so the build won't fail. + if [ $kernel_maj -le 4 ]; then + if [ $kernel_min -le 17 ]; then + sudo sed -i 's/rdtscll(timestamp);/timestamp = rdtsc_ordered();/g' \ + "$GIT_REPOS/QAT/quickassist/utilities/osal/src/linux/kernel_space/OsalServices.c" || true + fi + fi + + (cd "$GIT_REPOS/QAT" && sudo ./configure --enable-icp-sriov=host && sudo make install) + + if sudo service qat_service start; then + echo "failed to start the qat service. Something may be wrong with your device or package." + fi +} + +function install_rocksdb() { + # Rocksdb is installed for use with the blobfs tests. + if [ ! -d /usr/src/rocksdb ]; then + git clone "${GIT_REPO_ROCKSDB}" "$GIT_REPOS/rocksdb" + git -C "$GIT_REPOS/rocksdb" checkout spdk-v5.6.1 + sudo mv "$GIT_REPOS/rocksdb" /usr/src/ + else + sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1 + echo "rocksdb already in /usr/src. Not checking out again" + fi +} + +function install_fio() { + # This version of fio is installed in /usr/src/fio to enable + # building the spdk fio plugin. + local fio_version="fio-3.19" + + if [ ! -d /usr/src/fio ]; then + if [ ! -d fio ]; then + git clone "${GIT_REPO_FIO}" "$GIT_REPOS/fio" + sudo mv "$GIT_REPOS/fio" /usr/src/ + else + sudo mv "$GIT_REPOS/fio" /usr/src/ + fi + ( + git -C /usr/src/fio checkout master \ + && git -C /usr/src/fio pull \ + && git -C /usr/src/fio checkout $fio_version \ + && if [ $OSID == 'freebsd' ]; then + gmake -C /usr/src/fio -j${jobs} \ + && sudo gmake -C /usr/src/fio install + else + make -C /usr/src/fio -j${jobs} \ + && sudo make -C /usr/src/fio install + fi + ) + else + echo "fio already in /usr/src/fio. Not installing" + fi +} + +function install_flamegraph() { + # Flamegraph is used when printing out timing graphs for the tests. + if [ ! -d /usr/local/FlameGraph ]; then + git clone "${GIT_REPO_FLAMEGRAPH}" "$GIT_REPOS/FlameGraph" + mkdir -p /usr/local + sudo mv "$GIT_REPOS/FlameGraph" /usr/local/FlameGraph + else + echo "flamegraph already installed. Skipping" + fi +} + +function install_qemu() { + # Two versions of QEMU are used in the tests. + # Stock QEMU is used for vhost. A special fork + # is used to test OCSSDs. Install both. + + # Forked QEMU + SPDK_QEMU_BRANCH=spdk-5.0.0 + mkdir -p "$GIT_REPOS/qemu" + if [[ ! -d $GIT_REPOS/qemu/$SPDK_QEMU_BRANCH ]]; then + git clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" + else + echo "qemu already checked out. Skipping" + fi + + declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH") + if ((gcc_version >= 9)); then + # GCC 9 fails to compile Qemu due to some old warnings which were not detected by older versions. + opt_params+=("--extra-cflags=-Wno-error=stringop-truncation -Wno-error=deprecated-declarations -Wno-error=incompatible-pointer-types -Wno-error=format-truncation") + opt_params+=("--disable-glusterfs") + fi + + # Most tsocks proxies rely on a configuration file in /etc/tsocks.conf. + # If using tsocks, please make sure to complete this config before trying to build qemu. + if [[ $INSTALL_TSOCKS == true && $NO_TSOCKS != true ]]; then + if hash tsocks 2> /dev/null; then + opt_params+=("--with-git='tsocks git'") + fi + fi + + sed -i s@git://git.qemu.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.gitmodules" + sed -i s@git://git.qemu.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.git/config" + sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.gitmodules" + sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.git/config" + # The qemu configure script places several output files in the CWD. + (cd "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa) + + make -C "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" -j${jobs} + sudo make -C "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" install +} + +function install_nvmecli() { + SPDK_NVME_CLI_BRANCH=spdk-1.6 + if [[ ! -d $GIT_REPOS/nvme-cli ]]; then + git clone "${GIT_REPO_SPDK_NVME_CLI}" -b "$SPDK_NVME_CLI_BRANCH" "$GIT_REPOS/nvme-cli" + else + echo "nvme-cli already checked out. Skipping" + fi + if [ ! -d "/usr/local/src/nvme-cli" ]; then + # Changes required for SPDK are already merged on top of + # nvme-cli, however not released yet. + # Support for SPDK should be released in nvme-cli >1.11.1 + if [[ ! -d $GIT_REPOS/nvme-cli-cuse ]]; then + git clone "https://github.com/linux-nvme/nvme-cli.git" "$GIT_REPOS/nvme-cli-cuse" + fi + git -C "$GIT_REPOS/nvme-cli-cuse" checkout "e770466615096a6d41f038a28819b00bc3078e1d" + make -C "$GIT_REPOS/nvme-cli-cuse" + sudo mv "$GIT_REPOS/nvme-cli-cuse" /usr/local/src/nvme-cli + fi +} + +function install_libiscsi() { + # We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need + # to later. Cloning from git is just future proofing the machines. + if [[ ! -d $GIT_REPOS/libiscsi ]]; then + git clone "${GIT_REPO_LIBISCSI}" "$GIT_REPOS/libiscsi" + else + echo "libiscsi already checked out. Skipping" + fi + (cd "$GIT_REPOS/libiscsi" && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi) + make -C "$GIT_REPOS/libiscsi" -j${jobs} + sudo make -C "$GIT_REPOS/libiscsi" install +} + +function install_git() { + install zlib-devel curl-devel + tar -C "$GIT_REPOS" -xzof <(wget -qO- "$GIT_REPO_GIT") + (cd "$GIT_REPOS/git-$GIT_VERSION" \ + && make configure \ + && ./configure --prefix=/usr/local/git \ + && sudo make -j${jobs} install) + sudo sh -c "echo 'export PATH=/usr/local/git/bin:$PATH' >> /etc/bashrc" + export "PATH=/usr/local/git/bin:$PATH" +} + +function install_extra_pkgs() { + if [[ $INSTALL_QAT == true ]]; then + install libudev-devel || install libudev-dev + fi + + if [[ $INSTALL_QEMU == true ]]; then + install qemu-system-x86 qemu-img \ + || install qemu-system-x86 qemu-utils \ + || install qemu + fi +} + +GIT_VERSION=2.25.1 +: ${GIT_REPO_SPDK=https://github.com/spdk/spdk.git} +export GIT_REPO_SPDK +: ${GIT_REPO_DPDK=https://github.com/spdk/dpdk.git} +export GIT_REPO_DPDK +: ${GIT_REPO_ROCKSDB=https://review.spdk.io/spdk/rocksdb} +export GIT_REPO_ROCKSDB +: ${GIT_REPO_FIO=http://git.kernel.dk/fio.git} +export GIT_REPO_FIO +: ${GIT_REPO_FLAMEGRAPH=https://github.com/brendangregg/FlameGraph.git} +export GIT_REPO_FLAMEGRAPH +: ${GIT_REPO_QEMU=https://github.com/spdk/qemu} +export GIT_REPO_QEMU +: ${GIT_REPO_LIBISCSI=https://github.com/sahlberg/libiscsi} +export GIT_REPO_LIBISCSI +: ${GIT_REPO_SPDK_NVME_CLI=https://github.com/spdk/nvme-cli} +export GIT_REPO_SPDK_NVME_CLI +: ${GIT_REPO_INTEL_IPSEC_MB=https://github.com/spdk/intel-ipsec-mb.git} +export GIT_REPO_INTEL_IPSEC_MB +: ${DRIVER_LOCATION_QAT=https://01.org/sites/default/files/downloads//qat1.7.l.4.9.0-00008.tar.gz} +export DRIVER_LOCATION_QAT +: ${GIT_REPO_GIT=https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz} +export GIT_REPO_GIT +GIT_REPOS=${GIT_REPOS:-$HOME} + +gcc_version=$(gcc -dumpversion) gcc_version=${gcc_version%%.*} +if [[ $ID == centos ]] && (( VERSION_ID == 7 )); then + # install proper version of the git first + install_git +fi + +IFS="," read -ra conf_env <<< "$CONF" +for conf in "${conf_env[@]}"; do + export "INSTALL_${conf^^}=true" +done +sources=(install_refspdk) + +if [[ $OS == FreeBSD ]]; then + jobs=$(($(sysctl -n hw.ncpu) * 2)) +else + jobs=$(($(nproc) * 2)) + sources+=( + install_libiscsi + install_nvmecli + install_qat + install_rocksdb + install_flamegraph + install_qemu + ) +fi +sources+=(install_fio) + +sudo mkdir -p /usr/{,local}/src +sudo mkdir -p "$GIT_REPOS" + +install_extra_pkgs + +if [[ $INSTALL_REFSPDK == true ]]; then + # Serialize builds as refspdk depends on spdk + install_spdk + install_refspdk +else + sources+=(install_spdk) +fi + +for source in "${sources[@]}"; do + source_conf=${source^^} + if [[ ${!source_conf} == true ]]; then + "$source" & + fi +done +wait |