diff options
Diffstat (limited to 'src/pmdk/utils')
75 files changed, 6448 insertions, 0 deletions
diff --git a/src/pmdk/utils/.gitignore b/src/pmdk/utils/.gitignore new file mode 100644 index 000000000..c4c4ffc6a --- /dev/null +++ b/src/pmdk/utils/.gitignore @@ -0,0 +1 @@ +*.zip diff --git a/src/pmdk/utils/CHECK_WHITESPACE.PS1 b/src/pmdk/utils/CHECK_WHITESPACE.PS1 new file mode 100644 index 000000000..5ffeedc4a --- /dev/null +++ b/src/pmdk/utils/CHECK_WHITESPACE.PS1 @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2017, Intel Corporation +# +# CHECK_WHITESPACE.PS1 -- script to check coding style +# +# XXX - integrate with VS projects and execute for each build +# + +$scriptdir = Split-Path -Parent $PSCommandPath +$rootdir = $scriptdir + "\.." +$whitepace = $rootdir + "\utils\check_whitespace" + +If ( Get-Command -Name perl -ErrorAction SilentlyContinue ) { + &perl $whitepace -g + if ($LASTEXITCODE -ne 0) { + Exit $LASTEXITCODE + } +} else { + Write-Output "Cannot execute check_whitespace - perl is missing" +} diff --git a/src/pmdk/utils/CREATE-ZIP.PS1 b/src/pmdk/utils/CREATE-ZIP.PS1 new file mode 100644 index 000000000..902cb07a9 --- /dev/null +++ b/src/pmdk/utils/CREATE-ZIP.PS1 @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2019, Intel Corporation +# +# CREATE-ZIP.PS1 -- script to create release zip package +# + +# +# parameter handling +# +[CmdletBinding(PositionalBinding=$false)] +Param( + [alias("b")] + $build = "debug", + [alias("v")] + $version = "0", + [alias("e")] + $extended = "0" + ) + +$scriptdir = Split-Path -Parent $PSCommandPath +$rootdir = $scriptdir + "\..\" + +$builddir = $rootdir + "\src\x64\" +$zipdir = $builddir + "\pmdk\" + +if ($version -eq "0") { + $git = Get-Command -Name git -ErrorAction SilentlyContinue + if ($git) { + $version = $(git describe) + } else { + $version = "0" + } +} + +$zipfile = $builddir + "\pmdk-" + $version + "-win-x64-" + $build + ".zip" + +Remove-Item $zipdir -Force -Recurse -ea si +Get-ChildItem | Where-Object {$_.Name -Match "pmdk-.*-win-x64.zip"} | Remove-Item -Force -ea si +New-Item -ItemType directory -Path ( $zipdir) -Force | Out-Null +New-Item -ItemType directory -Path ( $zipdir + "\bin\") -Force | Out-Null +New-Item -ItemType directory -Path ( $zipdir + "\lib\") -Force | Out-Null + +$libs = @("libpmem", "libpmemblk", "libpmemlog", "libpmemobj", "libpmempool") +$apps = @("pmempool") +$apps_extended = @("pmempool", "pmemalloc", "pmemdetect", "pmemspoil", "pmemwrite") + +if ($extended -eq "1") { + $apps = $apps_extended +} + +foreach ($lib in $libs) { + Copy-Item ($builddir + $build + "\libs\" + $lib + ".dll") ($zipdir + "\bin\") + foreach ($ex in @(".lib", ".pdb")) { + Copy-Item ($builddir + $build + "\libs\" + $lib + $ex) ($zipdir + "\lib\") + } +} +foreach ($app in $apps) { + if ($app -eq "pmempool") { + Copy-Item ($builddir + $build + "\libs\" + $app + ".exe") ($zipdir + "\bin\") + Copy-Item ($builddir + $build + "\libs\" + $app + ".pdb") ($zipdir + "\lib\") + } else { + Copy-Item ($builddir + $build + "\tests\" + $app + ".exe") ($zipdir + "\bin\") + Copy-Item ($builddir + $build + "\tests\" + $app + ".pdb") ($zipdir + "\lib\") + } +} + +Copy-Item -Recurse ($rootdir + "src\include") ($zipdir) +Remove-Item -Force ($zipdir + "include\.cstyleignore") +Remove-Item -Force ($zipdir + "include\README") +Remove-Item -Force ($zipdir + "include\librpmem.h") +Copy-Item ($rootdir + "README.md") ($zipdir) +Copy-Item ($rootdir + "LICENSE") ($zipdir) +Copy-Item ($rootdir + "ChangeLog") ($zipdir) + +Add-Type -Assembly System.IO.Compression.FileSystem +$comprlevel = [System.IO.Compression.CompressionLevel]::Optimal + +if (Test-Path ($zipdir)) { + [System.IO.Compression.ZipFile]::CreateFromDirectory($zipdir, $zipfile, + $comprlevel, $true) +} + +Remove-Item $zipdir -Force -Recurse -ea si diff --git a/src/pmdk/utils/CSTYLE.ps1 b/src/pmdk/utils/CSTYLE.ps1 new file mode 100644 index 000000000..42b22b34c --- /dev/null +++ b/src/pmdk/utils/CSTYLE.ps1 @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2017, Intel Corporation +# +# CSTYLE.ps1 -- script to check coding style +# +# XXX - integrate with VS projects and execute for each build +# + +$scriptdir = Split-Path -Parent $PSCommandPath +$rootdir = $scriptdir + "\.." +$cstyle = $rootdir + "\utils\cstyle" +$checkdir = $rootdir + +# XXX - *.cpp/*.hpp files not supported yet +$include = @( "*.c", "*.h" ) + +If ( Get-Command -Name perl -ErrorAction SilentlyContinue ) { + Get-ChildItem -Path $checkdir -Recurse -Include $include | ` + Where-Object { $_.FullName -notlike "*jemalloc*" } | ` + ForEach-Object { + $IGNORE = $_.DirectoryName + "\.cstyleignore" + if(Test-Path $IGNORE) { + if((Select-String $_.Name $IGNORE)) { + return + } + } + $_ + } | ForEach-Object { + Write-Output $_.FullName + & perl $cstyle $_.FullName + if ($LASTEXITCODE -ne 0) { + Exit $LASTEXITCODE + } + } +} else { + Write-Output "Cannot execute cstyle - perl is missing" +} diff --git a/src/pmdk/utils/Makefile b/src/pmdk/utils/Makefile new file mode 100644 index 000000000..14352886f --- /dev/null +++ b/src/pmdk/utils/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +rwildcard=$(strip $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2)\ + $(filter $(subst *,%,$2),$d))) + +SCRIPTS = $(call rwildcard,,*.sh) + +cstyle: + ./check-shebang.sh $(SCRIPTS) + +.PHONY: cstyle diff --git a/src/pmdk/utils/README b/src/pmdk/utils/README new file mode 100644 index 000000000..07a762ca2 --- /dev/null +++ b/src/pmdk/utils/README @@ -0,0 +1,5 @@ +Persistent Memory Development Kit + +This is utils/README. + +The scripts found here are used during library development. diff --git a/src/pmdk/utils/SRCVERSION.ps1 b/src/pmdk/utils/SRCVERSION.ps1 new file mode 100644 index 000000000..8e6e43639 --- /dev/null +++ b/src/pmdk/utils/SRCVERSION.ps1 @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# SRCVERSION.PS1 -- script to create SCRVERSION macro and generate srcversion.h +# + +# +# Windows dll versioning supports only fixed number of fields. The most +# important are MAJOR, MINOR and REVISION. We have 3-compoment releases +# (e.g. 1.5.1) with release candidates, so we have to encode this information +# into this fixed number of fields. That's why we abuse REVISION to encode both +# 3rd component and rc status. +# REVISION = 3RDCOMP * 1000 + (!is_rc) * 100 + rc. +# +# Examples: +# +---------------------+-----+-----+--------+-----+------+-------+----------+ +# |git describe --long |MAJOR|MINOR|REVISION|BUILD|BUGFIX|PRIVATE|PRERELEASE| +# +---------------------+-----+-----+--------+-----+------+-------+----------+ +# |1.5-rc2-0-12345678 | 1| 5| 2| 0| false| false| true| +# |1.5-rc3-6-12345678 | 1| 5| 3| 6| false| true| true| +# |1.5-0-12345678 | 1| 5| 100| 0| false| false| false| +# |1.5-6-123345678 | 1| 5| 100| 6| false| true| false| +# |1.5.2-rc1-0-12345678 | 1| 5| 2001| 0| true| false| true| +# |1.5.2-rc4-6-12345678 | 1| 5| 2004| 6| true| true| true| +# |1.5.2-0-12345678 | 1| 5| 2100| 0| true| false| false| +# |1.5.2-6-12345678 | 1| 5| 2100| 6| true| true| false| +# +---------------------+-----+-----+--------+-----+------+-------+----------+ +# + +$scriptPath = Split-Path -parent $MyInvocation.MyCommand.Definition +$file_path = $scriptPath + "\..\src\windows\include\srcversion.h" +$git_version_file = $scriptPath + "\..\GIT_VERSION" +$version_file = $scriptPath + "\..\VERSION" +$git = Get-Command -Name git -ErrorAction SilentlyContinue + +if (Test-Path $file_path) { + $old_src_version = Get-Content $file_path | ` + Where-Object { $_ -like '#define SRCVERSION*' } +} else { + $old_src_version = "" +} + +$git_version = "" +$git_version_hash = "" + +if (Test-Path $git_version_file) { + $git_version = Get-Content $git_version_file + if ($git_version -eq "`$Format:%h`$") { + $git_version = "" + } else { + $git_version_hash = $git_version + } +} + +$PRERELEASE = $false +$BUGFIX = $false +$PRIVATE = $true +$CUSTOM = $false + +if ($null -ne $args[0]) { + $version = $args[0] + $ver_array = $version.split("-+") +} elseif (Test-Path $version_file) { + $version = Get-Content $version_file + $ver_array = $version.split("-+") +} elseif ($git_version_hash -ne "") { + $MAJOR = 0 + $MINOR = 0 + $REVISION = 0 + $BUILD = 0 + + $version = $git_version_hash + $CUSTOM = $true + $version_custom_msg = "#define VERSION_CUSTOM_MSG `"$git_version_hash`"" +} elseif ($null -ne $git) { + $version = $(git describe) + $ver_array = $(git describe --long).split("-+") +} else { + $MAJOR = 0 + $MINOR = 0 + $REVISION = 0 + $BUILD = 0 + + $version = "UNKNOWN_VERSION" + $CUSTOM = $true + $version_custom_msg = "#define VERSION_CUSTOM_MSG `"UNKNOWN_VERSION`"" +} + +if ($null -ne $ver_array) { + $ver_dots = $ver_array[0].split(".") + $MAJOR = $ver_dots[0] + $MINOR = $ver_dots[1] + if ($ver_dots.length -ge 3) { + $REV = $ver_dots[2] + $BUGFIX = $true + } else { + $REV = 0 + } + + $REVISION = 1000 * $REV + $BUILD = $ver_array[$ver_array.length - 2] + + if ($ver_array.length -eq 4) { + # <MAJOR>.<MINOR>[.<BUGFIX>]-<SUFFIX><REVISION>-<BUILD>-<HASH> + + if ($ver_array[1].StartsWith("rc")) { + # <MAJOR>.<MINOR>[.<BUGFIX>]-rc<REVISION>-<BUILD>-<HASH> + $REVISION += $ver_array[1].Substring("rc".Length) + $PRERELEASE = $true + $version = "$($ver_array[0])-$($ver_array[1])+git$($ver_array[2]).$($ver_array[3])" + } else { + # <MAJOR>.<MINOR>[.<BUGFIX>]-<SOMETHING>-<BUILD>-<HASH> + throw "Unknown version format" + } + } else { + # <MAJOR>.<MINOR>[.<BUGFIX>]-<BUILD>-<HASH> + $REVISION += 100 + $version = "$($ver_array[0])+git$($ver_array[1]).$($ver_array[2])" + } + + if ($BUILD -eq 0) { + # it is not a (pre)release build + $PRIVATE = $false + } +} + +$src_version = "#define SRCVERSION `"$version`"" + +if ($old_src_version -eq $src_version) { + exit 0 +} + +Write-Output "updating source version: $version" +Write-Output $src_version > $file_path + +Write-Output "#ifdef RC_INVOKED" >> $file_path + +Write-Output "#define MAJOR $MAJOR" >> $file_path +Write-Output "#define MINOR $MINOR" >> $file_path +Write-Output "#define REVISION $REVISION" >> $file_path +Write-Output "#define BUILD $BUILD" >> $file_path + +if ($PRERELEASE) { + Write-Output "#define PRERELEASE 1" >> $file_path +} +if ($BUGFIX) { + Write-Output "#define BUGFIX 1" >> $file_path +} +if ($PRIVATE) { + Write-Output "#define PRIVATE 1" >> $file_path +} +if ($CUSTOM) { + Write-Output "#define CUSTOM 1" >> $file_path + Write-Output $version_custom_msg >> $file_path +} + +Write-Output "#endif" >> $file_path diff --git a/src/pmdk/utils/build-dpkg.sh b/src/pmdk/utils/build-dpkg.sh new file mode 100755 index 000000000..cb96f4185 --- /dev/null +++ b/src/pmdk/utils/build-dpkg.sh @@ -0,0 +1,836 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2014-2020, Intel Corporation + +# +# build-dpkg.sh - Script for building deb packages +# + +set -e + +SCRIPT_DIR=$(dirname $0) +source $SCRIPT_DIR/pkg-common.sh + +# +# usage -- print usage message and exit +# +usage() +{ + [ "$1" ] && echo Error: $1 + cat >&2 <<EOF +Usage: $0 [ -h ] -t version-tag -s source-dir -w working-dir -o output-dir + [ -e build-experimental ] [ -c run-check ] + [ -n with-ndctl ] [ -f testconfig-file ] + +-h print this help message +-t version-tag source version tag +-s source-dir source directory +-w working-dir working directory +-o output-dir output directory +-e build-experimental build experimental packages +-c run-check run package check +-n with-ndctl build with libndctl +-f testconfig-file custom testconfig.sh +EOF + exit 1 +} + +# +# command-line argument processing... +# +args=`getopt he:c:r:n:t:d:s:w:o:f: $*` +[ $? != 0 ] && usage +set -- $args +for arg +do + receivetype=auto + case "$arg" + in + -e) + EXPERIMENTAL="$2" + shift 2 + ;; + -c) + BUILD_PACKAGE_CHECK="$2" + shift 2 + ;; + -f) + TEST_CONFIG_FILE="$2" + shift 2 + ;; + -r) + BUILD_RPMEM="$2" + shift 2 + ;; + -n) + NDCTL_ENABLE="$2" + shift 2 + ;; + -t) + PACKAGE_VERSION_TAG="$2" + shift 2 + ;; + -s) + SOURCE="$2" + shift 2 + ;; + -w) + WORKING_DIR="$2" + shift 2 + ;; + -o) + OUT_DIR="$2" + shift 2 + ;; + --) + shift + break + ;; + esac +done + +# check for mandatory arguments +if [ -z "$PACKAGE_VERSION_TAG" -o -z "$SOURCE" -o -z "$WORKING_DIR" -o -z "$OUT_DIR" ] +then + error "Mandatory arguments missing" + usage +fi + +PMEM2_INSTALL="y" + +PREFIX=usr +LIB_DIR=$PREFIX/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH) +INC_DIR=$PREFIX/include +MAN1_DIR=$PREFIX/share/man/man1 +MAN3_DIR=$PREFIX/share/man/man3 +MAN5_DIR=$PREFIX/share/man/man5 +MAN7_DIR=$PREFIX/share/man/man7 + +DOC_DIR=$PREFIX/share/doc +if [ "$EXTRA_CFLAGS_RELEASE" = "" ]; then + export EXTRA_CFLAGS_RELEASE="-ggdb -fno-omit-frame-pointer" +fi + +LIBFABRIC_MIN_VERSION=1.4.2 +NDCTL_MIN_VERSION=60.1 + +function convert_changelog() { + while read line + do + if [[ $line =~ $REGEX_DATE_AUTHOR ]] + then + DATE="${BASH_REMATCH[1]}" + AUTHOR="${BASH_REMATCH[2]}" + echo " * ${DATE} ${AUTHOR}" + elif [[ $line =~ $REGEX_MESSAGE_START ]] + then + MESSAGE="${BASH_REMATCH[1]}" + echo " - ${MESSAGE}" + elif [[ $line =~ $REGEX_MESSAGE ]] + then + MESSAGE="${BASH_REMATCH[1]}" + echo " ${MESSAGE}" + fi + done < $1 +} + +function rpmem_install_triggers_overrides() { +cat << EOF > debian/librpmem.install +$LIB_DIR/librpmem.so.* +EOF + +cat << EOF > debian/librpmem.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +librpmem: package-name-doesnt-match-sonames +EOF + +cat << EOF > debian/librpmem-dev.install +$LIB_DIR/pmdk_debug/librpmem.a $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/librpmem.so $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/librpmem.so.* $LIB_DIR/pmdk_dbg/ +$LIB_DIR/librpmem.so +$LIB_DIR/pkgconfig/librpmem.pc +$INC_DIR/librpmem.h +$MAN7_DIR/librpmem.7 +$MAN3_DIR/rpmem_*.3 +EOF + +cat << EOF > debian/librpmem-dev.triggers +interest man-db +EOF + +cat << EOF > debian/librpmem-dev.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +# The following warnings are triggered by a bug in debhelper: +# https://bugs.debian.org/204975 +postinst-has-useless-call-to-ldconfig +postrm-has-useless-call-to-ldconfig +# We do not want to compile with -O2 for debug version +hardening-no-fortify-functions $LIB_DIR/pmdk_dbg/* +EOF + +cat << EOF > debian/rpmemd.install +usr/bin/rpmemd +$MAN1_DIR/rpmemd.1 +EOF + +cat << EOF > debian/rpmemd.triggers +interest man-db +EOF + +cat << EOF > debian/rpmemd.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +EOF +} + +function append_rpmem_control() { +cat << EOF >> $CONTROL_FILE + +Package: librpmem +Architecture: any +Depends: \${shlibs:Depends}, \${misc:Depends} +Description: Persistent Memory remote access support library + librpmem provides low-level support for remote access to persistent memory + (pmem) utilizing RDMA-capable RNICs. The library can be used to replicate + remotely a memory region over RDMA protocol. It utilizes appropriate + persistency mechanism based on remote node’s platform capabilities. The + librpmem utilizes the ssh client to authenticate a user on remote node and for + encryption of connection’s out-of-band configuration data. + . + This library is for applications that use remote persistent memory directly, + without the help of any library-supplied transactions or memory allocation. + Higher-level libraries that build on libpmem are available and are recommended + for most applications. + +Package: librpmem-dev +Section: libdevel +Architecture: any +Depends: librpmem (=\${binary:Version}), libpmem-dev, \${shlibs:Depends}, \${misc:Depends} +Description: Development files for librpmem + librpmem provides low-level support for remote access to persistent memory + (pmem) utilizing RDMA-capable RNICs. + . + This package contains libraries and header files used for linking programs + against librpmem. + +Package: rpmemd +Section: misc +Architecture: any +Priority: optional +Depends: \${shlibs:Depends}, \${misc:Depends} +Description: rpmem daemon + Daemon for Remote Persistent Memory support. +EOF +} + +function libpmem2_install_triggers_overrides() { +cat << EOF > debian/libpmem2.install +$LIB_DIR/libpmem2.so.* +EOF + +cat << EOF > debian/libpmem2.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +libpmem2: package-name-doesnt-match-sonames +EOF + +cat << EOF > debian/libpmem2-dev.install +$LIB_DIR/pmdk_debug/libpmem2.a $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmem2.so $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmem2.so.* $LIB_DIR/pmdk_dbg/ +$LIB_DIR/libpmem2.so +$LIB_DIR/pkgconfig/libpmem2.pc +$INC_DIR/libpmem2.h +$MAN7_DIR/libpmem2.7 +$MAN3_DIR/pmem2_*.3 +EOF + +cat << EOF > debian/libpmem2-dev.triggers +interest man-db +EOF + +cat << EOF > debian/libpmem2-dev.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +# The following warnings are triggered by a bug in debhelper: +# https://bugs.debian.org/204975 +postinst-has-useless-call-to-ldconfig +postrm-has-useless-call-to-ldconfig +# We do not want to compile with -O2 for debug version +hardening-no-fortify-functions $LIB_DIR/pmdk_dbg/* +EOF +} + +function append_libpmem2_control() { +cat << EOF >> $CONTROL_FILE + +Package: libpmem2 +Architecture: any +Depends: \${shlibs:Depends}, \${misc:Depends} +Description: Persistent Memory low level support library + libpmem2 provides low level persistent memory support. In particular, support + for the persistent memory instructions for flushing changes to pmem is + provided. + +Package: libpmem2-dev +Section: libdevel +Architecture: any +Depends: libpmem2 (=\${binary:Version}), \${shlibs:Depends}, \${misc:Depends} +Description: Development files for libpmem2 + libpmem2 provides low level persistent memory support. In particular, support + for the persistent memory instructions for flushing changes to pmem is + provided. +EOF +} + +function daxio_install_triggers_overrides() { +cat << EOF > debian/daxio.install +usr/bin/daxio +$MAN1_DIR/daxio.1 +EOF + +cat << EOF > debian/daxio.triggers +interest man-db +EOF + +cat << EOF > debian/daxio.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +EOF +} + +function append_daxio_control() { +cat << EOF >> $CONTROL_FILE + +Package: daxio +Section: misc +Architecture: any +Priority: optional +Depends: libpmem (=\${binary:Version}), \${shlibs:Depends}, \${misc:Depends} +Description: dd-like tool to read/write to a devdax device + The daxio utility performs I/O on Device DAX devices or zeroes a Device + DAX device. Since the standard I/O APIs (read/write) cannot be used + with Device DAX, data transfer is performed on a memory-mapped device. + The daxio may be used to dump Device DAX data to a file, restore data from + a backup copy, move/copy data to another device or to erase data from + a device. +EOF +} + +if [ "${BUILD_PACKAGE_CHECK}" == "y" ] +then +CHECK_CMD=" +override_dh_auto_test: + dh_auto_test + if [ -f $TEST_CONFIG_FILE ]; then\ + cp $TEST_CONFIG_FILE src/test/testconfig.sh;\ + else\ + echo 'PMEM_FS_DIR=/tmp' > src/test/testconfig.sh; \ + echo 'PMEM_FS_DIR_FORCE_PMEM=1' >> src/test/testconfig.sh; \ + echo 'TEST_BUILD=\"debug nondebug\"' >> src/test/testconfig.sh; \ + echo 'TEST_FS=\"pmem any none\"' >> src/test/testconfig.sh; \ + fi + make pcheck ${PCHECK_OPTS} +" +else +CHECK_CMD=" +override_dh_auto_test: + +" +fi + +check_tool debuild +check_tool dch +check_file $SCRIPT_DIR/pkg-config.sh + +source $SCRIPT_DIR/pkg-config.sh + +PACKAGE_VERSION=$(get_version $PACKAGE_VERSION_TAG) +PACKAGE_RELEASE=1 +PACKAGE_SOURCE=${PACKAGE_NAME}-${PACKAGE_VERSION} +PACKAGE_TARBALL_ORIG=${PACKAGE_NAME}_${PACKAGE_VERSION}.orig.tar.gz +MAGIC_INSTALL=utils/magic-install.sh +MAGIC_UNINSTALL=utils/magic-uninstall.sh +CONTROL_FILE=debian/control + +[ -d $WORKING_DIR ] || mkdir $WORKING_DIR +[ -d $OUT_DIR ] || mkdir $OUT_DIR + +OLD_DIR=$PWD + +cd $WORKING_DIR + +check_dir $SOURCE + +mv $SOURCE $PACKAGE_SOURCE +tar zcf $PACKAGE_TARBALL_ORIG $PACKAGE_SOURCE + +cd $PACKAGE_SOURCE + +rm -rf debian +mkdir debian + +# Generate compat file +cat << EOF > debian/compat +9 +EOF + +# Generate control file +cat << EOF > $CONTROL_FILE +Source: $PACKAGE_NAME +Maintainer: $PACKAGE_MAINTAINER +Section: libs +Priority: optional +Standards-version: 4.1.4 +Build-Depends: debhelper (>= 9) +Homepage: https://pmem.io/pmdk/ + +Package: libpmem +Architecture: any +Depends: \${shlibs:Depends}, \${misc:Depends} +Description: Persistent Memory low level support library + libpmem provides low level persistent memory support. In particular, support + for the persistent memory instructions for flushing changes to pmem is + provided. + +Package: libpmem-dev +Section: libdevel +Architecture: any +Depends: libpmem (=\${binary:Version}), \${shlibs:Depends}, \${misc:Depends} +Description: Development files for libpmem + libpmem provides low level persistent memory support. In particular, support + for the persistent memory instructions for flushing changes to pmem is + provided. + +Package: libpmemblk +Architecture: any +Depends: libpmem (=\${binary:Version}), \${shlibs:Depends}, \${misc:Depends} +Description: Persistent Memory block array support library + libpmemblk implements a pmem-resident array of blocks, all the same size, where + a block is updated atomically with respect to power failure or program + interruption (no torn blocks). + +Package: libpmemblk-dev +Section: libdevel +Architecture: any +Depends: libpmemblk (=\${binary:Version}), libpmem-dev, \${shlibs:Depends}, \${misc:Depends} +Description: Development files for libpmemblk + libpmemblk implements a pmem-resident array of blocks, all the same size, where + a block is updated atomically with respect to power failure or program + interruption (no torn blocks). + +Package: libpmemlog +Architecture: any +Depends: libpmem (=\${binary:Version}), \${shlibs:Depends}, \${misc:Depends} +Description: Persistent Memory log file support library + libpmemlog implements a pmem-resident log file. + +Package: libpmemlog-dev +Section: libdevel +Architecture: any +Depends: libpmemlog (=\${binary:Version}), libpmem-dev, \${shlibs:Depends}, \${misc:Depends} +Description: Development files for libpmemlog + libpmemlog implements a pmem-resident log file. + +Package: libpmemobj +Architecture: any +Depends: libpmem (=\${binary:Version}), \${shlibs:Depends}, \${misc:Depends} +Description: Persistent Memory object store support library + libpmemobj turns a persistent memory file into a flexible object store, + supporting transactions, memory management, locking, lists, and a number of + other features. + +Package: libpmemobj-dev +Section: libdevel +Architecture: any +Depends: libpmemobj (=\${binary:Version}), libpmem-dev, \${shlibs:Depends}, \${misc:Depends} +Description: Development files for libpmemobj + libpmemobj turns a persistent memory file into a flexible object store, + supporting transactions, memory management, locking, lists, and a number of + other features. + . + This package contains libraries and header files used for linking programs + against libpmemobj. + +Package: libpmempool +Architecture: any +Depends: libpmem (=\${binary:Version}), \${shlibs:Depends}, \${misc:Depends} +Description: Persistent Memory pool management support library + libpmempool provides a set of utilities for management, diagnostics and repair + of persistent memory pools. A pool in this context means a pmemobj pool, + pmemblk pool, pmemlog pool or BTT layout, independent of the underlying + storage. The libpmempool is for applications that need high reliability or + built-in troubleshooting. It may be useful for testing and debugging purposes + also. + +Package: libpmempool-dev +Section: libdevel +Architecture: any +Depends: libpmempool (=\${binary:Version}), libpmem-dev, \${shlibs:Depends}, \${misc:Depends} +Description: Development files for libpmempool + libpmempool provides a set of utilities for management, diagnostics and repair + of persistent memory pools. + . + This package contains libraries and header files used for linking programs + against libpmempool. + +Package: $PACKAGE_NAME-dbg +Section: debug +Priority: optional +Architecture: any +Depends: libpmem (=\${binary:Version}), libpmemblk (=\${binary:Version}), libpmemlog (=\${binary:Version}), libpmemobj (=\${binary:Version}), libpmempool (=\${binary:Version}), \${misc:Depends} +Description: Debug symbols for PMDK libraries + Debug symbols for all PMDK libraries. + +Package: pmempool +Section: misc +Architecture: any +Priority: optional +Depends: \${shlibs:Depends}, \${misc:Depends} +Description: utility for management and off-line analysis of PMDK memory pools + This utility is a standalone tool that manages Persistent Memory pools + created by PMDK libraries. It provides a set of utilities for + administration and diagnostics of Persistent Memory pools. Pmempool may be + useful for troubleshooting by system administrators and users of the + applications based on PMDK libraries. + +Package: pmreorder +Section: misc +Architecture: any +Priority: optional +Depends: \${shlibs:Depends}, \${misc:Depends} +Description: tool to parse and replay pmemcheck logs + Pmreorder is tool that parses and replays log of operations collected by + pmemcheck -- a atandalone tool which is a collection of python scripts designed + to parse and replay operations logged by pmemcheck - a persistent memory + checking tool. Pmreorder performs the store reordering between persistent + memory barriers - a sequence of flush-fence operations. It uses a + consistency checking routine provided in the command line options to check + whether files are in a consistent state. +EOF + +cp LICENSE debian/copyright + +if [ -n "$NDCTL_ENABLE" ]; then + pass_ndctl_enable="NDCTL_ENABLE=$NDCTL_ENABLE" +else + pass_ndctl_enable="" +fi + +cat << EOF > debian/rules +#!/usr/bin/make -f +#export DH_VERBOSE=1 +%: + dh \$@ + +override_dh_strip: + dh_strip --dbg-package=$PACKAGE_NAME-dbg + +override_dh_auto_build: + dh_auto_build -- EXPERIMENTAL=${EXPERIMENTAL} prefix=/$PREFIX libdir=/$LIB_DIR includedir=/$INC_DIR docdir=/$DOC_DIR man1dir=/$MAN1_DIR man3dir=/$MAN3_DIR man5dir=/$MAN5_DIR man7dir=/$MAN7_DIR sysconfdir=/etc bashcompdir=/usr/share/bash-completion/completions NORPATH=1 ${pass_ndctl_enable} SRCVERSION=$SRCVERSION PMEM2_INSTALL=${PMEM2_INSTALL} + +override_dh_auto_install: + dh_auto_install -- EXPERIMENTAL=${EXPERIMENTAL} prefix=/$PREFIX libdir=/$LIB_DIR includedir=/$INC_DIR docdir=/$DOC_DIR man1dir=/$MAN1_DIR man3dir=/$MAN3_DIR man5dir=/$MAN5_DIR man7dir=/$MAN7_DIR sysconfdir=/etc bashcompdir=/usr/share/bash-completion/completions NORPATH=1 ${pass_ndctl_enable} SRCVERSION=$SRCVERSION PMEM2_INSTALL=${PMEM2_INSTALL} + find -path './debian/*usr/share/man/man*/*.gz' -exec gunzip {} \; + +override_dh_install: + mkdir -p debian/tmp/usr/share/pmdk/ + cp utils/pmdk.magic debian/tmp/usr/share/pmdk/ + dh_install + +${CHECK_CMD} +EOF + +chmod +x debian/rules + +mkdir debian/source + +ITP_BUG_EXCUSE="# This is our first package but we do not want to upload it yet. +# Please refer to Debian Developer's Reference section 5.1 (New packages) for details: +# https://www.debian.org/doc/manuals/developers-reference/pkgs.html#newpackage" + +cat << EOF > debian/source/format +3.0 (quilt) +EOF + +cat << EOF > debian/libpmem.install +$LIB_DIR/libpmem.so.* +usr/share/pmdk/pmdk.magic +$MAN5_DIR/poolset.5 +EOF + +cat $MAGIC_INSTALL > debian/libpmem.postinst +sed -i '1s/.*/\#\!\/bin\/bash/' debian/libpmem.postinst +echo $'\n#DEBHELPER#\n' >> debian/libpmem.postinst +cat $MAGIC_UNINSTALL > debian/libpmem.prerm +sed -i '1s/.*/\#\!\/bin\/bash/' debian/libpmem.prerm +echo $'\n#DEBHELPER#\n' >> debian/libpmem.prerm + +cat << EOF > debian/libpmem.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +libpmem: package-name-doesnt-match-sonames +EOF + +cat << EOF > debian/libpmem-dev.install +$LIB_DIR/pmdk_debug/libpmem.a $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmem.so $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmem.so.* $LIB_DIR/pmdk_dbg/ +$LIB_DIR/libpmem.so +$LIB_DIR/pkgconfig/libpmem.pc +$INC_DIR/libpmem.h +$MAN7_DIR/libpmem.7 +$MAN3_DIR/pmem_*.3 +EOF + +cat << EOF > debian/libpmem-dev.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +# The following warnings are triggered by a bug in debhelper: +# https://bugs.debian.org/204975 +postinst-has-useless-call-to-ldconfig +postrm-has-useless-call-to-ldconfig +# We do not want to compile with -O2 for debug version +hardening-no-fortify-functions $LIB_DIR/pmdk_dbg/* +# pmdk provides second set of libraries for debugging. +# These are in /usr/lib/$arch/pmdk_dbg/, but still trigger ldconfig. +# Related issue: https://github.com/pmem/issues/issues/841 +libpmem-dev: package-has-unnecessary-activation-of-ldconfig-trigger + +EOF + +cat << EOF > debian/libpmemblk.install +$LIB_DIR/libpmemblk.so.* +EOF + +cat << EOF > debian/libpmemblk.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +libpmemblk: package-name-doesnt-match-sonames +EOF + +cat << EOF > debian/libpmemblk-dev.install +$LIB_DIR/pmdk_debug/libpmemblk.a $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmemblk.so $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmemblk.so.* $LIB_DIR/pmdk_dbg/ +$LIB_DIR/libpmemblk.so +$LIB_DIR/pkgconfig/libpmemblk.pc +$INC_DIR/libpmemblk.h +$MAN7_DIR/libpmemblk.7 +$MAN3_DIR/pmemblk_*.3 +EOF + +cat << EOF > debian/libpmemblk-dev.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +# The following warnings are triggered by a bug in debhelper: +# https://bugs.debian.org/204975 +postinst-has-useless-call-to-ldconfig +postrm-has-useless-call-to-ldconfig +# We do not want to compile with -O2 for debug version +hardening-no-fortify-functions $LIB_DIR/pmdk_dbg/* +# pmdk provides second set of libraries for debugging. +# These are in /usr/lib/$arch/pmdk_dbg/, but still trigger ldconfig. +# Related issue: https://github.com/pmem/issues/issues/841 +libpmemblk-dev: package-has-unnecessary-activation-of-ldconfig-trigger +EOF + +cat << EOF > debian/libpmemlog.install +$LIB_DIR/libpmemlog.so.* +EOF + +cat << EOF > debian/libpmemlog.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +libpmemlog: package-name-doesnt-match-sonames +EOF + +cat << EOF > debian/libpmemlog-dev.install +$LIB_DIR/pmdk_debug/libpmemlog.a $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmemlog.so $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmemlog.so.* $LIB_DIR/pmdk_dbg/ +$LIB_DIR/libpmemlog.so +$LIB_DIR/pkgconfig/libpmemlog.pc +$INC_DIR/libpmemlog.h +$MAN7_DIR/libpmemlog.7 +$MAN3_DIR/pmemlog_*.3 +EOF + +cat << EOF > debian/libpmemlog-dev.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +# The following warnings are triggered by a bug in debhelper: +# https://bugs.debian.org/204975 +postinst-has-useless-call-to-ldconfig +postrm-has-useless-call-to-ldconfig +# We do not want to compile with -O2 for debug version +hardening-no-fortify-functions $LIB_DIR/pmdk_dbg/* +# pmdk provides second set of libraries for debugging. +# These are in /usr/lib/$arch/pmdk_dbg/, but still trigger ldconfig. +# Related issue: https://github.com/pmem/issues/issues/841 +libpmemlog-dev: package-has-unnecessary-activation-of-ldconfig-trigger +EOF + +cat << EOF > debian/libpmemobj.install +$LIB_DIR/libpmemobj.so.* +EOF + +cat << EOF > debian/libpmemobj.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +libpmemobj: package-name-doesnt-match-sonames +EOF + +cat << EOF > debian/libpmemobj-dev.install +$LIB_DIR/pmdk_debug/libpmemobj.a $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmemobj.so $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmemobj.so.* $LIB_DIR/pmdk_dbg/ +$LIB_DIR/libpmemobj.so +$LIB_DIR/pkgconfig/libpmemobj.pc +$INC_DIR/libpmemobj.h +$INC_DIR/libpmemobj/*.h +$MAN7_DIR/libpmemobj.7 +$MAN3_DIR/pmemobj_*.3 +$MAN3_DIR/pobj_*.3 +$MAN3_DIR/oid_*.3 +$MAN3_DIR/toid*.3 +$MAN3_DIR/direct_*.3 +$MAN3_DIR/d_r*.3 +$MAN3_DIR/tx_*.3 +EOF + +cat << EOF > debian/libpmemobj-dev.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +# The following warnings are triggered by a bug in debhelper: +# https://bugs.debian.org/204975 +postinst-has-useless-call-to-ldconfig +postrm-has-useless-call-to-ldconfig +# We do not want to compile with -O2 for debug version +hardening-no-fortify-functions $LIB_DIR/pmdk_dbg/* +# pmdk provides second set of libraries for debugging. +# These are in /usr/lib/$arch/pmdk_dbg/, but still trigger ldconfig. +# Related issue: https://github.com/pmem/issues/issues/841 +libpmemobj-dev: package-has-unnecessary-activation-of-ldconfig-trigger +EOF + +cat << EOF > debian/libpmempool.install +$LIB_DIR/libpmempool.so.* +EOF + +cat << EOF > debian/libpmempool.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +libpmempool: package-name-doesnt-match-sonames +EOF + +cat << EOF > debian/libpmempool-dev.install +$LIB_DIR/pmdk_debug/libpmempool.a $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmempool.so $LIB_DIR/pmdk_dbg/ +$LIB_DIR/pmdk_debug/libpmempool.so.* $LIB_DIR/pmdk_dbg/ +$LIB_DIR/libpmempool.so +$LIB_DIR/pkgconfig/libpmempool.pc +$INC_DIR/libpmempool.h +$MAN7_DIR/libpmempool.7 +$MAN3_DIR/pmempool_*.3 +EOF + +cat << EOF > debian/libpmempool-dev.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +# The following warnings are triggered by a bug in debhelper: +# https://bugs.debian.org/204975 +postinst-has-useless-call-to-ldconfig +postrm-has-useless-call-to-ldconfig +# We do not want to compile with -O2 for debug version +hardening-no-fortify-functions $LIB_DIR/pmdk_dbg/* +# pmdk provides second set of libraries for debugging. +# These are in /usr/lib/$arch/pmdk_dbg/, but still trigger ldconfig. +# Related issue: https://github.com/pmem/issues/issues/841 +libpmempool-dev: package-has-unnecessary-activation-of-ldconfig-trigger +EOF + +cat << EOF > debian/$PACKAGE_NAME-dbg.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +EOF + +cat << EOF > debian/pmempool.install +usr/bin/pmempool +$MAN1_DIR/pmempool.1 +$MAN1_DIR/pmempool-*.1 +usr/share/bash-completion/completions/pmempool +EOF + +cat << EOF > debian/pmempool.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +EOF + +cat << EOF > debian/pmreorder.install +usr/bin/pmreorder +usr/share/pmreorder/*.py +$MAN1_DIR/pmreorder.1 +EOF + +cat << EOF > debian/pmreorder.lintian-overrides +$ITP_BUG_EXCUSE +new-package-should-close-itp-bug +EOF + +# librpmem & rpmemd +if [ "${BUILD_RPMEM}" = "y" -a "${RPMEM_DPKG}" = "y" ] +then + append_rpmem_control; + rpmem_install_triggers_overrides; +fi + +# libpmem2 +if [ "${PMEM2_INSTALL}" == "y" ] +then + append_libpmem2_control; + libpmem2_install_triggers_overrides; +fi + +# daxio +if [ "${NDCTL_ENABLE}" != "n" ] +then + append_daxio_control; + daxio_install_triggers_overrides; +fi + +# Convert ChangeLog to debian format +CHANGELOG_TMP=changelog.tmp +dch --create --empty --package $PACKAGE_NAME -v $PACKAGE_VERSION-$PACKAGE_RELEASE -M -c $CHANGELOG_TMP +touch debian/changelog +head -n1 $CHANGELOG_TMP >> debian/changelog +echo "" >> debian/changelog +convert_changelog ChangeLog >> debian/changelog +echo "" >> debian/changelog +tail -n1 $CHANGELOG_TMP >> debian/changelog +rm $CHANGELOG_TMP + +# This is our first release but we do +debuild --preserve-envvar=EXTRA_CFLAGS_RELEASE \ + --preserve-envvar=EXTRA_CFLAGS_DEBUG \ + --preserve-envvar=EXTRA_CFLAGS \ + --preserve-envvar=EXTRA_CXXFLAGS \ + --preserve-envvar=EXTRA_LDFLAGS \ + --preserve-envvar=NDCTL_ENABLE \ + -us -uc -b + +cd $OLD_DIR + +find $WORKING_DIR -name "*.deb"\ + -or -name "*.dsc"\ + -or -name "*.changes"\ + -or -name "*.orig.tar.gz"\ + -or -name "*.debian.tar.gz" | while read FILE +do + mv -v $FILE $OUT_DIR/ +done + +exit 0 diff --git a/src/pmdk/utils/build-rpm.sh b/src/pmdk/utils/build-rpm.sh new file mode 100755 index 000000000..759a1fe1c --- /dev/null +++ b/src/pmdk/utils/build-rpm.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2014-2020, Intel Corporation + +# +# build-rpm.sh - Script for building rpm packages +# + +set -e + +SCRIPT_DIR=$(dirname $0) +source $SCRIPT_DIR/pkg-common.sh + +check_tool rpmbuild +check_file $SCRIPT_DIR/pkg-config.sh +source $SCRIPT_DIR/pkg-config.sh + +# +# usage -- print usage message and exit +# +usage() +{ + [ "$1" ] && echo Error: $1 + cat >&2 <<EOF +Usage: $0 [ -h ] -t version-tag -s source-dir -w working-dir -o output-dir + [ -d distro ] [ -e build-experimental ] [ -c run-check ] + [ -r build-rpmem ] [ -n with-ndctl ] [ -f testconfig-file ] + +-h print this help message +-t version-tag source version tag +-s source-dir source directory +-w working-dir working directory +-o output-dir output directory +-d distro Linux distro name +-e build-experimental build experimental packages +-c run-check run package check +-r build-rpmem build librpmem and rpmemd packages +-n with-ndctl build with libndctl +-f testconfig-file custom testconfig.sh +EOF + exit 1 +} + +# +# command-line argument processing... +# +args=`getopt he:c:r:n:t:d:s:w:o:f: $*` +[ $? != 0 ] && usage +set -- $args +for arg +do + receivetype=auto + case "$arg" + in + -e) + EXPERIMENTAL="$2" + shift 2 + ;; + -c) + BUILD_PACKAGE_CHECK="$2" + shift 2 + ;; + -f) + TEST_CONFIG_FILE="$2" + shift 2 + ;; + -r) + BUILD_RPMEM="$2" + shift 2 + ;; + -n) + NDCTL_ENABLE="$2" + shift 2 + ;; + -t) + PACKAGE_VERSION_TAG="$2" + shift 2 + ;; + -s) + SOURCE="$2" + shift 2 + ;; + -w) + WORKING_DIR="$2" + shift 2 + ;; + -o) + OUT_DIR="$2" + shift 2 + ;; + -d) + DISTRO="$2" + shift 2 + ;; + --) + shift + break + ;; + esac +done + +# check for mandatory arguments +if [ -z "$PACKAGE_VERSION_TAG" -o -z "$SOURCE" -o -z "$WORKING_DIR" -o -z "$OUT_DIR" ] +then + error "Mandatory arguments missing" + usage +fi + +# detected distro or defined in cmd +if [ -z "${DISTRO}" ] +then + OS=$(get_os) + if [ "$OS" != "1" ] + then + echo "Detected OS: $OS" + DISTRO=$OS + else + error "Unknown distribution" + exit 1 + fi +fi + +if [ "$EXTRA_CFLAGS_RELEASE" = "" ]; then + export EXTRA_CFLAGS_RELEASE="-ggdb -fno-omit-frame-pointer" +fi + +PMEM2_INSTALL="y" + +LIBFABRIC_MIN_VERSION=1.4.2 +NDCTL_MIN_VERSION=60.1 + +RPMBUILD_OPTS=( ) +PACKAGE_VERSION=$(get_version $PACKAGE_VERSION_TAG) + +if [ -z "$PACKAGE_VERSION" ] +then + error "Can not parse version from '${PACKAGE_VERSION_TAG}'" + exit 1 +fi + +PACKAGE_SOURCE=${PACKAGE_NAME}-${PACKAGE_VERSION} +SOURCE=$PACKAGE_NAME +PACKAGE_TARBALL=$PACKAGE_SOURCE.tar.gz +RPM_SPEC_FILE=$PACKAGE_SOURCE/$PACKAGE_NAME.spec +MAGIC_INSTALL=$PACKAGE_SOURCE/utils/magic-install.sh +MAGIC_UNINSTALL=$PACKAGE_SOURCE/utils/magic-uninstall.sh +OLDPWD=$PWD + +[ -d $WORKING_DIR ] || mkdir -v $WORKING_DIR +[ -d $OUT_DIR ] || mkdir $OUT_DIR + +cd $WORKING_DIR + +check_dir $SOURCE +mv $SOURCE $PACKAGE_SOURCE + +if [ "$DISTRO" = "SLES_like" ] +then + RPM_LICENSE="BSD-3-Clause" + RPM_GROUP_SYS_BASE="System\/Base" + RPM_GROUP_SYS_LIBS="System\/Libraries" + RPM_GROUP_DEV_LIBS="Development\/Libraries\/C and C++" + RPM_PKG_NAME_SUFFIX="1" + RPM_MAKE_FLAGS="BINDIR=""%_bindir"" NORPATH=1" + RPM_MAKE_INSTALL="%fdupes %{buildroot}\/%{_prefix}" +else + RPM_LICENSE="BSD" + RPM_GROUP_SYS_BASE="System Environment\/Base" + RPM_GROUP_SYS_LIBS="System Environment\/Libraries" + RPM_GROUP_DEV_LIBS="Development\/Libraries" + RPM_PKG_NAME_SUFFIX="" + RPM_MAKE_FLAGS="NORPATH=1" + RPM_MAKE_INSTALL="" +fi + +# +# Create parametrized spec file required by rpmbuild. +# Most of variables are set in pkg-config.sh file in order to +# keep descriptive values separately from this script. +# +sed -e "s/__VERSION__/$PACKAGE_VERSION/g" \ + -e "s/__LICENSE__/$RPM_LICENSE/g" \ + -e "s/__PACKAGE_MAINTAINER__/$PACKAGE_MAINTAINER/g" \ + -e "s/__PACKAGE_SUMMARY__/$PACKAGE_SUMMARY/g" \ + -e "s/__GROUP_SYS_BASE__/$RPM_GROUP_SYS_BASE/g" \ + -e "s/__GROUP_SYS_LIBS__/$RPM_GROUP_SYS_LIBS/g" \ + -e "s/__GROUP_DEV_LIBS__/$RPM_GROUP_DEV_LIBS/g" \ + -e "s/__PKG_NAME_SUFFIX__/$RPM_PKG_NAME_SUFFIX/g" \ + -e "s/__MAKE_FLAGS__/$RPM_MAKE_FLAGS/g" \ + -e "s/__MAKE_INSTALL_FDUPES__/$RPM_MAKE_INSTALL/g" \ + -e "s/__LIBFABRIC_MIN_VER__/$LIBFABRIC_MIN_VERSION/g" \ + -e "s/__NDCTL_MIN_VER__/$NDCTL_MIN_VERSION/g" \ + $OLDPWD/$SCRIPT_DIR/pmdk.spec.in > $RPM_SPEC_FILE + +if [ "$DISTRO" = "SLES_like" ] +then + sed -i '/^#.*bugzilla.redhat/d' $RPM_SPEC_FILE +fi + +# do not split on space +IFS=$'\n' + +# experimental features +if [ "${EXPERIMENTAL}" = "y" ] +then + # no experimental features for now + RPMBUILD_OPTS+=( ) +fi + +# libpmem2 +if [ "${PMEM2_INSTALL}" == "y" ] +then + RPMBUILD_OPTS+=(--define "_pmem2_install 1") +fi + +# librpmem & rpmemd +if [ "${BUILD_RPMEM}" = "y" ] +then + RPMBUILD_OPTS+=(--with fabric) +else + RPMBUILD_OPTS+=(--without fabric) +fi + +# daxio & RAS +if [ "${NDCTL_ENABLE}" = "n" ] +then + RPMBUILD_OPTS+=(--without ndctl) +else + RPMBUILD_OPTS+=(--with ndctl) +fi + +# use specified testconfig file or default +if [[( -n "${TEST_CONFIG_FILE}") && ( -f "$TEST_CONFIG_FILE" ) ]] +then + echo "Test config file: $TEST_CONFIG_FILE" + RPMBUILD_OPTS+=(--define "_testconfig $TEST_CONFIG_FILE") +else + echo -e "Test config file $TEST_CONFIG_FILE does not exist.\n"\ + "Default test config will be used." +fi + +# run make check or not +if [ "${BUILD_PACKAGE_CHECK}" == "n" ] +then + RPMBUILD_OPTS+=(--define "_skip_check 1") +fi + +tar zcf $PACKAGE_TARBALL $PACKAGE_SOURCE + +# Create directory structure for rpmbuild +mkdir -v BUILD SPECS + +echo "opts: ${RPMBUILD_OPTS[@]}" + +rpmbuild --define "_topdir `pwd`"\ + --define "_rpmdir ${OUT_DIR}"\ + --define "_srcrpmdir ${OUT_DIR}"\ + -ta $PACKAGE_TARBALL \ + ${RPMBUILD_OPTS[@]} + +echo "Building rpm packages done" + +exit 0 diff --git a/src/pmdk/utils/check-area.sh b/src/pmdk/utils/check-area.sh new file mode 100755 index 000000000..911556b91 --- /dev/null +++ b/src/pmdk/utils/check-area.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018-2020, Intel Corporation + +# +# Finds applicable area name for specified commit id. +# + +if [ -z "$1" ]; then + echo "Missing commit id argument." + exit 1 +fi + +files=$(git show $1 --format=oneline --name-only | grep -v -e "$1") + +git show -q $1 | cat + +echo +echo "Modified files:" +echo "$files" + +function categorize() { + category=$1 + shift + cat_files=`echo "$files" | grep $*` + + if [ -n "${cat_files}" ]; then + echo "$category" + files=`echo "$files" | grep -v $*` + fi +} + +echo +echo "Areas computed basing on the list of modified files: (see utils/check-area.sh for full algorithm)" + +categorize core -e "^src/core/" +categorize pmem -e "^src/libpmem/" -e "^src/include/libpmem.h" +categorize pmem2 -e "^src/libpmem2/" -e "^src/include/libpmem2.h" +categorize rpmem -e "^src/librpmem/" -e "^src/include/librpmem.h" -e "^src/tools/rpmemd/" -e "^src/rpmem_common/" +categorize log -e "^src/libpmemlog/" -e "^src/include/libpmemlog.h" +categorize blk -e "^src/libpmemblk/" -e "^src/include/libpmemblk.h" +categorize obj -e "^src/libpmemobj/" -e "^src/include/libpmemobj.h" -e "^src/include/libpmemobj/" +categorize pool -e "^src/libpmempool/" -e "^src/include/libpmempool.h" -e "^src/tools/pmempool/" +categorize benchmark -e "^src/benchmarks/" +categorize examples -e "^src/examples/" +categorize daxio -e "^src/tools/daxio/" +categorize pmreorder -e "^src/tools/pmreorder/" +categorize test -e "^src/test/" +categorize doc -e "^doc/" -e ".md\$" -e "^ChangeLog" -e "README" +categorize common -e "^src/common/" \ + -e "^utils/" \ + -e ".inc\$" \ + -e ".yml\$" \ + -e ".gitattributes" \ + -e ".gitignore" \ + -e "^.mailmap\$" \ + -e "^src/PMDK.sln\$" \ + -e "Makefile\$" \ + -e "^src/freebsd/" \ + -e "^src/windows/" \ + -e "^src/include/pmemcompat.h" + +echo +echo "If the above list contains more than 1 entry, please consider splitting" +echo "your change into more commits, unless those changes don't make sense " +echo "individually (they do not build, tests do not pass, etc)." +echo "For example, it's perfectly fine to use 'obj' prefix for one commit that" +echo "changes libpmemobj source code, its tests and documentation." + +if [ -n "$files" ]; then + echo + echo "Uncategorized files:" + echo "$files" +fi diff --git a/src/pmdk/utils/check-commit.sh b/src/pmdk/utils/check-commit.sh new file mode 100755 index 000000000..3985e40c3 --- /dev/null +++ b/src/pmdk/utils/check-commit.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# Used to check whether all the commit messages in a pull request +# follow the GIT/PMDK guidelines. +# +# usage: ./check-commit.sh commit +# + +if [ -z "$1" ]; then + echo "Usage: check-commit.sh commit-id" + exit 1 +fi + +echo "Checking $1" + +subject=$(git log --format="%s" -n 1 $1) + +if [[ $subject =~ ^Merge.* ]]; then + # skip + exit 0 +fi + +if [[ $subject =~ ^Revert.* ]]; then + # skip + exit 0 +fi + +# valid area names +AREAS="pmem\|pmem2\|rpmem\|log\|blk\|obj\|pool\|test\|benchmark\|examples\|doc\|core\|common\|daxio\|pmreorder" + +prefix=$(echo $subject | sed -n "s/^\($AREAS\)\:.*/\1/p") + +if [ "$prefix" = "" ]; then + echo "FAIL: subject line in commit message does not contain valid area name" + echo + `dirname $0`/check-area.sh $1 + exit 1 +fi + +commit_len=$(git log --format="%s%n%b" -n 1 $1 | wc -L) + +if [ $commit_len -gt 73 ]; then + echo "FAIL: commit message exceeds 72 chars per line (commit_len)" + echo + git log -n 1 $1 | cat + exit 1 +fi diff --git a/src/pmdk/utils/check-commits.sh b/src/pmdk/utils/check-commits.sh new file mode 100755 index 000000000..762db22db --- /dev/null +++ b/src/pmdk/utils/check-commits.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# Used to check whether all the commit messages in a pull request +# follow the GIT/PMDK guidelines. +# +# usage: ./check-commits.sh [range] +# + +if [ -z "$1" ]; then + # on CI run this check only for pull requests + if [ -n "$CI_REPO_SLUG" ]; then + if [[ "$CI_REPO_SLUG" != "$GITHUB_REPO" \ + || $CI_EVENT_TYPE != "pull_request" ]]; + then + echo "SKIP: $0 can only be executed for pull requests to $GITHUB_REPO" + exit 0 + fi + fi + # CI_COMMIT_RANGE can be invalid for force pushes - use another + # method to determine the list of commits + if [[ $(git rev-list $CI_COMMIT_RANGE 2>/dev/null) || -n "$CI_COMMIT_RANGE" ]]; then + MERGE_BASE=$(echo $CI_COMMIT_RANGE | cut -d. -f1) + [ -z $MERGE_BASE ] && \ + MERGE_BASE=$(git log --pretty="%cN:%H" | grep GitHub | head -n1 | cut -d: -f2) + RANGE=$MERGE_BASE..$CI_COMMIT + else + MERGE_BASE=$(git log --pretty="%cN:%H" | grep GitHub | head -n1 | cut -d: -f2) + RANGE=$MERGE_BASE..HEAD + fi +else + RANGE="$1" +fi + +COMMITS=$(git log --pretty=%H $RANGE) + +set -e + +for commit in $COMMITS; do + `dirname $0`/check-commit.sh $commit +done diff --git a/src/pmdk/utils/check-manpage b/src/pmdk/utils/check-manpage new file mode 100755 index 000000000..d0c0d4d2e --- /dev/null +++ b/src/pmdk/utils/check-manpage @@ -0,0 +1,62 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2019, Intel Corporation + +# check-manpage -- a tool to check a single man page against errors +# +# While it can handle multiple files, it's recommended to use +# check-manpages instead. +set -e + +check_link() +{ + [ $(wc -l <"$file") = 1 ] || + { echo ".so link isn't the only line" && return; } + link=$(cat "$file") + link=${link#.so } + [ "${link##*/}" = "$link" ] || + grep -q '^\.so man\([0-9]\)/[a-z0-9+_-]\+\.\1$' "$file" || + { echo ".so link directory is not matching manX" && return; } + [ -e "${link##*/}" ] || + { echo ".so link target doesn't exist: ${link##*/}" && return; } +} + +for m in "$@"; do + dir="$(dirname $m)" + file="$(basename $m)" + [ -n "$dir" ] && pushd "$dir" >/dev/null + + if grep -q '^\.so' "$file"; then + err=$(check_link) + [ -z "$err" ] || { + echo >&2 "$file: $err" + FAILED=1 + } + popd >/dev/null 2>/dev/null + continue + fi + + # man can emit warnings and errors. Even non-fatal errors are normally + # suppressed if a pager is in use (ie, all interactive usage). Common + # messages include an unknown macro, an unbreakable line, etc. + err=$(MANWIDTH=80 man --warnings -E UTF-8 -l -Tutf8 -Z "$file" 2>&1 >/dev/null| + grep -v 'cannot adjust line' || true) + [ -z "$err" ] || { + echo >&2 "$file: $err" + FAILED=1 + } + + # If a "NAME" section exists, call lexgrog to see if it's properly + # formatted. + if grep -q '^\.SH NAME' "$file"; then + if ! lexgrog "$file" >/dev/null; then + # lexgrog doesn't give any interesting messages. + echo 2>&1 "lexgrog failed on $file" + FAILED=1 + fi + fi + + popd >/dev/null 2>/dev/null +done + +exit $FAILED diff --git a/src/pmdk/utils/check-manpages b/src/pmdk/utils/check-manpages new file mode 100755 index 000000000..415db124c --- /dev/null +++ b/src/pmdk/utils/check-manpages @@ -0,0 +1,16 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2019-2020, Intel Corporation + +# check-manpages -- a tool to test against some manpage errors + +MANS="$*" +[ -n "$MANS" ] || MANS="$(find doc -name '*.1' -o -name '*.3' -o -name '*.5' -o -name '*.7')" +[ -n "$MANS" ] || { echo >&2 "No man pages given, and none found in doc/"; exit 1;} + +for page in $MANS;do + if [ "${page/rpmem/}" != "$page" ] && [ "$BUILD_RPMEM" != "y" ]; then + continue + fi + echo $page +done | xargs -P `nproc` -n1 -- utils/check-manpage diff --git a/src/pmdk/utils/check-os.sh b/src/pmdk/utils/check-os.sh new file mode 100755 index 000000000..64d371c13 --- /dev/null +++ b/src/pmdk/utils/check-os.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2019, Intel Corporation + +# +# Used to check if there are no banned functions in .o file +# +# usage: ./check-os.sh [os.h path] [.o file] [.c file] + +EXCLUDE="os_posix|os_thread_posix" +if [[ $2 =~ $EXCLUDE ]]; then + echo "skip $2" + exit 0 +fi + +symbols=$(nm --demangle --undefined-only --format=posix $2 | sed 's/ U *//g') +functions=$(cat $1 | tr '\n' '|') +functions=${functions%?} # remove trailing | character +out=$( + for sym in $symbols + do + grep -wE $functions <<<"$sym" + done | sed 's/$/\(\)/g') + +[[ ! -z $out ]] && + echo -e "`pwd`/$3:1: non wrapped function(s):\n$out\nplease use os wrappers" && + rm -f $2 && # remove .o file as it don't match requirements + exit 1 + +exit 0 diff --git a/src/pmdk/utils/check-shebang.sh b/src/pmdk/utils/check-shebang.sh new file mode 100755 index 000000000..1051d178b --- /dev/null +++ b/src/pmdk/utils/check-shebang.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2019, Intel Corporation +# +# utils/check-shebang.sh -- interpreter directive check script +# +set -e + +err_count=0 + +for file in $@ ; do + [ ! -f $file ] && continue + SHEBANG=`head -n1 $file | cut -d" " -f1` + [ "${SHEBANG:0:2}" != "#!" ] && continue + if [ "$SHEBANG" != "#!/usr/bin/env" -a $SHEBANG != "#!/bin/sh" ]; then + INTERP=`echo $SHEBANG | rev | cut -d"/" -f1 | rev` + echo "$file:1: error: invalid interpreter directive:" >&2 + echo " (is: \"$SHEBANG\", should be: \"#!/usr/bin/env $INTERP\")" >&2 + ((err_count+=1)) + fi +done + +if [ "$err_count" == "0" ]; then + echo "Interpreter directives are OK." +else + echo "Found $err_count errors in interpreter directives!" >&2 + err_count=1 +fi + +exit $err_count diff --git a/src/pmdk/utils/check_license/.gitignore b/src/pmdk/utils/check_license/.gitignore new file mode 100644 index 000000000..8891a7f15 --- /dev/null +++ b/src/pmdk/utils/check_license/.gitignore @@ -0,0 +1 @@ +check-license diff --git a/src/pmdk/utils/check_license/check-headers.sh b/src/pmdk/utils/check_license/check-headers.sh new file mode 100755 index 000000000..f79453bd3 --- /dev/null +++ b/src/pmdk/utils/check_license/check-headers.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# check-headers.sh - check copyright and license in source files + +SELF=$0 + +function usage() { + echo "Usage: $SELF <source_root_path> <license_tag> [-h|-v|-a]" + echo " -h, --help this help message" + echo " -v, --verbose verbose mode" + echo " -a, --all check all files (only modified files are checked by default)" +} + +if [ "$#" -lt 2 ]; then + usage >&2 + exit 2 +fi + +SOURCE_ROOT=$1 +shift +LICENSE=$1 +shift + +PATTERN=`mktemp` +TMP=`mktemp` +TMP2=`mktemp` +TEMPFILE=`mktemp` +rm -f $PATTERN $TMP $TMP2 + +if [ "$1" == "-h" -o "$1" == "--help" ]; then + usage + exit 0 +fi + +export GIT="git -C ${SOURCE_ROOT}" +$GIT rev-parse || exit 1 + +if [ -f $SOURCE_ROOT/.git/shallow ]; then + SHALLOW_CLONE=1 + echo + echo "Warning: This is a shallow clone. Checking dates in copyright headers" + echo " will be skipped in case of files that have no history." + echo +else + SHALLOW_CLONE=0 +fi + +VERBOSE=0 +CHECK_ALL=0 +while [ "$1" != "" ]; do + case $1 in + -v|--verbose) + VERBOSE=1 + ;; + -a|--all) + CHECK_ALL=1 + ;; + esac + shift +done + +if [ $CHECK_ALL -eq 0 ]; then + CURRENT_COMMIT=$($GIT log --pretty=%H -1) + MERGE_BASE=$($GIT merge-base HEAD origin/master 2>/dev/null) + [ -z $MERGE_BASE ] && \ + MERGE_BASE=$($GIT log --pretty="%cN:%H" | grep GitHub | head -n1 | cut -d: -f2) + [ -z $MERGE_BASE -o "$CURRENT_COMMIT" = "$MERGE_BASE" ] && \ + CHECK_ALL=1 +fi + +if [ $CHECK_ALL -eq 1 ]; then + echo "Checking copyright headers of all files..." + GIT_COMMAND="ls-tree -r --name-only HEAD" +else + if [ $VERBOSE -eq 1 ]; then + echo + echo "Warning: will check copyright headers of modified files only," + echo " in order to check all files issue the following command:" + echo " $ $SELF <source_root_path> <license_tag> -a" + echo " (e.g.: $ $SELF $SOURCE_ROOT $LICENSE -a)" + echo + fi + echo "Checking copyright headers of modified files only..." + GIT_COMMAND="diff --name-only $MERGE_BASE $CURRENT_COMMIT" +fi + +FILES=$($GIT $GIT_COMMAND | ${SOURCE_ROOT}/utils/check_license/file-exceptions.sh | \ + grep -E -e '*\.[chs]$' -e '*\.[ch]pp$' -e '*\.sh$' \ + -e '*\.py$' -e '*\.link$' -e 'Makefile*' -e 'TEST*' \ + -e '/common.inc$' -e '/match$' -e '/check_whitespace$' \ + -e 'LICENSE$' -e 'CMakeLists.txt$' -e '*\.cmake$' | \ + xargs) + +RV=0 +for file in $FILES ; do + # The src_path is a path which should be used in every command except git. + # git is called with -C flag so filepaths should be relative to SOURCE_ROOT + src_path="${SOURCE_ROOT}/$file" + [ ! -f $src_path ] && continue + # ensure that file is UTF-8 encoded + ENCODING=`file -b --mime-encoding $src_path` + iconv -f $ENCODING -t "UTF-8" $src_path > $TEMPFILE + + if ! grep -q "SPDX-License-Identifier: $LICENSE" $src_path; then + echo "$src_path:1: no $LICENSE SPDX tag found " >&2 + RV=1 + elif [[ $file == *.c ]] || [[ $file == *.cpp ]]; then + if ! grep -q -e "\/\/ SPDX-License-Identifier: $LICENSE" $src_path; then + echo "$src_path:1: wrong format of $LICENSE SPDX tag" >&2 + RV=1 + fi + elif [[ $file == *.h ]] || [[ $file == *.hpp ]]; then + if ! grep -q -e "\/\* SPDX-License-Identifier: $LICENSE \*\/" $src_path; then + echo "$src_path:1: wrong format of $LICENSE SPDX tag" >&2 + RV=1 + fi + elif [[ $file != LICENSE ]]; then + if ! grep -q -e "# SPDX-License-Identifier: $LICENSE" $src_path; then + echo "$src_path:1: wrong format of $LICENSE SPDX tag" >&2 + RV=1 + fi + fi + + if [ $SHALLOW_CLONE -eq 0 ]; then + $GIT log --no-merges --format="%ai %aE" -- $file | sort > $TMP + else + # mark the grafted commits (commits with no parents) + $GIT log --no-merges --format="%ai %aE grafted-%p-commit" -- $file | sort > $TMP + fi + + # skip checking dates for non-Intel commits + [[ ! $(tail -n1 $TMP) =~ "@intel.com" ]] && continue + + # skip checking dates for new files + [ $(cat $TMP | wc -l) -le 1 ] && continue + + # grep out the grafted commits (commits with no parents) + # and skip checking dates for non-Intel commits + grep -v -e "grafted--commit" $TMP | grep -e "@intel.com" > $TMP2 + + [ $(cat $TMP2 | wc -l) -eq 0 ] && continue + + FIRST=`head -n1 $TMP2` + LAST=` tail -n1 $TMP2` + + YEARS=`sed ' +/Copyright [0-9-]\+.*, Intel Corporation/!d +s/.*Copyright \([0-9]\+\)-\([0-9]\+\),.*/\1-\2/ +s/.*Copyright \([0-9]\+\),.*/\1-\1/' $src_path` + if [ -z "$YEARS" ]; then + echo >&2 "$src_path:1: No copyright years found" + RV=1 + continue + fi + + HEADER_FIRST=`echo $YEARS | cut -d"-" -f1` + HEADER_LAST=` echo $YEARS | cut -d"-" -f2` + + COMMIT_FIRST=`echo $FIRST | cut -d"-" -f1` + COMMIT_LAST=` echo $LAST | cut -d"-" -f1` + if [ "$COMMIT_FIRST" != "" -a "$COMMIT_LAST" != "" ]; then + if [ $HEADER_LAST -lt $COMMIT_LAST ]; then + if [ $HEADER_FIRST -lt $COMMIT_FIRST ]; then + COMMIT_FIRST=$HEADER_FIRST + fi + COMMIT_LAST=`date +%G` + if [ $COMMIT_FIRST -eq $COMMIT_LAST ]; then + NEW=$COMMIT_LAST + else + NEW=$COMMIT_FIRST-$COMMIT_LAST + fi + echo "$file:1: error: wrong copyright date: (is: $YEARS, should be: $NEW)" >&2 + RV=1 + fi + else + echo "$file:1: unknown commit dates" >&2 + RV=1 + fi +done +rm -f $TMP $TMP2 $TEMPFILE + +$(dirname "$0")/check-ms-license.pl $FILES + +# check if error found +if [ $RV -eq 0 ]; then + echo "Copyright headers are OK." +else + echo "Error(s) in copyright headers found!" >&2 +fi +exit $RV diff --git a/src/pmdk/utils/check_license/check-ms-license.pl b/src/pmdk/utils/check_license/check-ms-license.pl new file mode 100755 index 000000000..fae3c9436 --- /dev/null +++ b/src/pmdk/utils/check_license/check-ms-license.pl @@ -0,0 +1,62 @@ +#!/usr/bin/perl -w +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020, Intel Corporation + +use Digest::MD5 "md5_hex"; + +my $BSD3 = <<EndText; +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +EndText + +# a runaway sed job may alter this file too +md5_hex($BSD3) eq "066df8826723b6db407a931e5d6084f4" + or die "Text of BSD3 license has been corrupted.\n"; + +my $err = 0; + +$BSD3 =~ s/^/ /mg; # indent the text +$BSD3 =~ s/\n \n/\n\n/sg; # except for empty lines + +undef $/; +for my $f (@ARGV) { + next unless -f $f; + open F, '<', $f or die "Can't read ｢$f｣\n"; + $_ = <F>; + close F; + next unless /Copyright.*(Microsoft Corporation|FUJITSU)/; + + s/^ \*//mg; + s/^#//mg; + if (index($_, $BSD3) == -1) { + $err = 1; + print STDERR "Outside copyright but no/wrong license text in $f\n"; + } +} + +exit $err diff --git a/src/pmdk/utils/check_license/file-exceptions.sh b/src/pmdk/utils/check_license/file-exceptions.sh new file mode 100755 index 000000000..a3bdb7425 --- /dev/null +++ b/src/pmdk/utils/check_license/file-exceptions.sh @@ -0,0 +1,7 @@ +#!/bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# file-exceptions.sh - filter out files not checked for copyright and license + +grep -v -E -e '/queue.h$' -e '/getopt.h$' -e '/getopt.c$' -e 'src/core/valgrind/' -e '/testconfig\...$' diff --git a/src/pmdk/utils/check_sdk_version.py b/src/pmdk/utils/check_sdk_version.py new file mode 100755 index 000000000..7dc134f01 --- /dev/null +++ b/src/pmdk/utils/check_sdk_version.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2019-2020, Intel Corporation + +import argparse +import os +from subprocess import check_output, CalledProcessError +import sys +import shlex +from xml.dom import minidom +from xml.parsers.expat import ExpatError + +VALID_SDK_VERSION = '10.0.17134.0' + + +def get_vcxproj_files(root_dir, ignored): + """Get a list ".vcxproj" files under PMDK directory.""" + to_format = [] + command = 'git ls-files *.vcxproj' + try: + output = check_output(shlex.split(command), + cwd=root_dir).decode("UTF-8") + except CalledProcessError as e: + sys.exit('Error: "' + command + '" failed with returncode: ' + + str(e.returncode)) + + for line in output.splitlines(): + if not line: + continue + file_path = os.path.join(root_dir, line) + if os.path.isfile(file_path): + to_format.append(file_path) + + return to_format + + +def get_sdk_version(file): + """ + Get Windows SDK version from modified/new files from the current + pull request. + """ + tag = 'WindowsTargetPlatformVersion' + try: + xml_file = minidom.parse(file) + except ExpatError as e: + sys.exit('Error: "' + file + '" is incorrect.\n' + str(e)) + version_list = xml_file.getElementsByTagName(tag) + if len(version_list) != 1: + sys.exit('Error: the amount of tags "' + tag + '" is other than 1.') + version = version_list[0].firstChild.data + + return version + + +def main(): + parser = argparse.ArgumentParser(prog='check_sdk_version.py', + description='The script checks Windows SDK version in .vcxproj files.') + parser.add_argument('-d', '--directory', + help='Directory of PMDK tree.', required=True) + args = parser.parse_args() + current_directory = args.directory + if not os.path.isdir(current_directory): + sys.exit('"' + current_directory + '" is not a directory.') + + files = get_vcxproj_files(current_directory, '') + if not files: + sys.exit(0) + for file in files: + sdk_version = get_sdk_version(file) + if sdk_version != VALID_SDK_VERSION: + sys.exit('Wrong Windows SDK version: ' + sdk_version + + ' in file: "' + file + '". Please use: ' + VALID_SDK_VERSION) + + +if __name__ == '__main__': + main() diff --git a/src/pmdk/utils/check_whitespace b/src/pmdk/utils/check_whitespace new file mode 100755 index 000000000..083b3e8ff --- /dev/null +++ b/src/pmdk/utils/check_whitespace @@ -0,0 +1,210 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2015-2020, Intel Corporation + +# +# check_whitespace -- scrub source tree for whitespace errors +# + +use strict; +use warnings; + +use File::Basename; +use File::Find; +use Encode; +use v5.16; + +my $Me = $0; +$Me =~ s,.*/,,; + +$SIG{HUP} = $SIG{INT} = $SIG{TERM} = $SIG{__DIE__} = sub { + die @_ if $^S; + + my $errstr = shift; + + die "$Me: ERROR: $errstr"; +}; + +my $Errcount = 0; + +# +# err -- emit error, keep total error count +# +sub err { + warn @_, "\n"; + $Errcount++; +} + +# +# decode_file_as_string -- slurp an entire file into memory and decode +# +sub decode_file_as_string { + my ($full, $file) = @_; + my $fh; + open($fh, '<', $full) or die "$full $!\n"; + + local $/; + $_ = <$fh>; + close $fh; + + # check known encodings or die + my $decoded; + my @encodings = ("UTF-8", "UTF-16", "UTF-16LE", "UTF-16BE"); + + foreach my $enc (@encodings) { + eval { $decoded = decode( $enc, $_, Encode::FB_CROAK ) }; + + if (!$@) { + $decoded =~ s/\R/\n/g; + return $decoded; + } + } + + die "$Me: ERROR: Unknown file encoding"; +} + +# +# check_whitespace -- run the checks on the given file +# +sub check_whitespace { + my ($full, $file) = @_; + + my $line = 0; + my $eol; + my $nf = 0; + my $fstr = decode_file_as_string($full, $file); + my $empty = 0; + my $is_python = $full =~ /\.py$/; + + for (split /^/, $fstr) { + $line++; + if (!$is_python && /^$/) { + $empty++; + if ($empty > 1) { + err("$full:$line: ERROR duplicated empty line"); + } + } else { + $empty = 0; + } + + $eol = /[\n]/s; + if (/^\.nf$/) { + err("$full:$line: ERROR: nested .nf") if $nf; + $nf = 1; + } elsif (/^\.fi$/) { + $nf = 0; + } elsif ($nf == 0) { + chomp; + err("$full:$line: ERROR: trailing whitespace") if /\s$/; + err("$full:$line: ERROR: spaces before tabs") if / \t/; + } + } + + err("$full:$line: .nf without .fi") if $nf; + err("$full:$line: noeol") unless $eol; +} + +sub check_whitespace_with_exc { + my ($full) = @_; + + $_ = $full; + + return 0 if /^[.\/]*src\/common\/queue\.h/; + return 0 if /^[.\/]*src\/core\/valgrind\/.*\.h/; + + $_ = basename($full); + + return 0 unless /^(README.*|LICENSE.*|Makefile.*|CMakeLists.txt|.gitignore|TEST.*|RUNTESTS|check_whitespace|.*\.([chp13s]|sh|map|cpp|hpp|inc|PS1|ps1|py|md|cmake))$/; + return 0 if -z; + + check_whitespace($full, $_); + return 1; +} + +my $verbose = 0; +my $force = 0; +my $recursive = 0; + +sub check { + my ($file) = @_; + my $r; + + if ($force) { + $r = check_whitespace($file, basename($file)); + } else { + $r = check_whitespace_with_exc($file); + } + + if ($verbose) { + if ($r == 0) { + printf("skipped $file\n"); + } else { + printf("checked $file\n"); + } + } +} + +my @files = (); + +foreach my $arg (@ARGV) { + if ($arg eq '-v') { + $verbose = 1; + next; + } + if ($arg eq '-f') { + $force = 1; + next; + } + if ($arg eq '-r') { + $recursive = 1; + next; + } + if ($arg eq '-g') { + @files = `git ls-tree -r --name-only HEAD`; + chomp(@files); + next; + } + if ($arg eq '-h') { + printf "Options: + -g - check all files tracked by git + -r dir - recursively check all files in specified directory + -v verbose - print whether file was checked or not + -f force - disable blacklist\n"; + exit 1; + } + + if ($recursive == 1) { + find(sub { + my $full = $File::Find::name; + + if (!$force && + ($full eq './.git' || + $full eq './src/debug' || + $full eq './src/nondebug' || + $full eq './rpmbuild' || + $full eq './dpkgbuild')) { + $File::Find::prune = 1; + return; + } + + return unless -f; + + push @files, $full; + }, $arg); + + $recursive = 0; + next; + } + + push @files, $arg; +} + +if (!@files) { + printf "Empty file list!\n"; +} + +foreach (@files) { + check($_); +} + +exit $Errcount; diff --git a/src/pmdk/utils/copy-source.sh b/src/pmdk/utils/copy-source.sh new file mode 100755 index 000000000..5bb2d589d --- /dev/null +++ b/src/pmdk/utils/copy-source.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018, Intel Corporation + +# +# utils/copy-source.sh -- copy source files (from HEAD) to 'path_to_dir/pmdk' +# directory whether in git repository or not. +# +# usage: ./copy-source.sh [path_to_dir] [srcversion] + +set -e + +DESTDIR="$1" +SRCVERSION=$2 + +if [ -d .git ]; then + if [ -n "$(git status --porcelain)" ]; then + echo "Error: Working directory is dirty: $(git status --porcelain)" + exit 1 + fi +else + echo "Warning: You are not in git repository, working directory might be dirty." +fi + +mkdir -p "$DESTDIR"/pmdk +echo -n $SRCVERSION > "$DESTDIR"/pmdk/.version + +if [ -d .git ]; then + git archive HEAD | tar -x -C "$DESTDIR"/pmdk +else + find . \ + -maxdepth 1 \ + -not -name $(basename "$DESTDIR") \ + -not -name . \ + -exec cp -r "{}" "$DESTDIR"/pmdk \; +fi diff --git a/src/pmdk/utils/cstyle b/src/pmdk/utils/cstyle new file mode 100755 index 000000000..5221a2d56 --- /dev/null +++ b/src/pmdk/utils/cstyle @@ -0,0 +1,1037 @@ +#!/usr/bin/env perl +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# +# Portions copyright 2017, Intel Corporation. +# +# @(#)cstyle 1.58 98/09/09 (from shannon) +#ident "%Z%%M% %I% %E% SMI" +# +# cstyle - check for some common stylistic errors. +# +# cstyle is a sort of "lint" for C coding style. +# It attempts to check for the style used in the +# kernel, sometimes known as "Bill Joy Normal Form". +# +# There's a lot this can't check for, like proper indentation +# of code blocks. There's also a lot more this could check for. +# +# A note to the non perl literate: +# +# perl regular expressions are pretty much like egrep +# regular expressions, with the following special symbols +# +# \s any space character +# \S any non-space character +# \w any "word" character [a-zA-Z0-9_] +# \W any non-word character +# \d a digit [0-9] +# \D a non-digit +# \b word boundary (between \w and \W) +# \B non-word boundary +# + +require 5.0; +use IO::File; +use Getopt::Std; +use strict; +use warnings; + +my $usage = +"usage: cstyle [-chpvCP] [-o constructs] file ... + -c check continuation indentation inside functions + -h perform heuristic checks that are sometimes wrong + -p perform some of the more picky checks + -v verbose + -C don't check anything in header block comments + -P check for use of non-POSIX types + -o constructs + allow a comma-separated list of optional constructs: + doxygen allow doxygen-style block comments (/** /*!) + splint allow splint-style lint comments (/*@ ... @*/) +"; + +my %opts; + +if (!getopts("cho:pvCP", \%opts)) { + print $usage; + exit 2; +} + +my $check_continuation = $opts{'c'}; +my $heuristic = $opts{'h'}; +my $picky = $opts{'p'}; +my $verbose = $opts{'v'}; +my $ignore_hdr_comment = $opts{'C'}; +my $check_posix_types = $opts{'P'}; + +my $doxygen_comments = 0; +my $splint_comments = 0; + +if (defined($opts{'o'})) { + for my $x (split /,/, $opts{'o'}) { + if ($x eq "doxygen") { + $doxygen_comments = 1; + } elsif ($x eq "splint") { + $splint_comments = 1; + } else { + print "cstyle: unrecognized construct \"$x\"\n"; + print $usage; + exit 2; + } + } +} + +my ($filename, $line, $prev); # shared globals + +my $fmt; +my $hdr_comment_start; + +if ($verbose) { + $fmt = "%s:%d: %s\n%s\n"; +} else { + $fmt = "%s:%d: %s\n"; +} + +if ($doxygen_comments) { + # doxygen comments look like "/*!" or "/**"; allow them. + $hdr_comment_start = qr/^\s*\/\*[\!\*]?$/; +} else { + $hdr_comment_start = qr/^\s*\/\*$/; +} + +# Note, following must be in single quotes so that \s and \w work right. +my $typename = '(int|char|short|long|unsigned|float|double' . + '|\w+_t|struct\s+\w+|union\s+\w+|FILE|BOOL)'; + +# mapping of old types to POSIX compatible types +my %old2posix = ( + 'unchar' => 'uchar_t', + 'ushort' => 'ushort_t', + 'uint' => 'uint_t', + 'ulong' => 'ulong_t', + 'u_int' => 'uint_t', + 'u_short' => 'ushort_t', + 'u_long' => 'ulong_t', + 'u_char' => 'uchar_t', + 'quad' => 'quad_t' +); + +my $lint_re = qr/\/\*(?: + ARGSUSED[0-9]*|NOTREACHED|LINTLIBRARY|VARARGS[0-9]*| + CONSTCOND|CONSTANTCOND|CONSTANTCONDITION|EMPTY| + FALLTHRU|FALLTHROUGH|LINTED.*?|PRINTFLIKE[0-9]*| + PROTOLIB[0-9]*|SCANFLIKE[0-9]*|CSTYLED.*? + )\*\//x; + +my $splint_re = qr/\/\*@.*?@\*\//x; + +my $warlock_re = qr/\/\*\s*(?: + VARIABLES\ PROTECTED\ BY| + MEMBERS\ PROTECTED\ BY| + ALL\ MEMBERS\ PROTECTED\ BY| + READ-ONLY\ VARIABLES:| + READ-ONLY\ MEMBERS:| + VARIABLES\ READABLE\ WITHOUT\ LOCK:| + MEMBERS\ READABLE\ WITHOUT\ LOCK:| + LOCKS\ COVERED\ BY| + LOCK\ UNNEEDED\ BECAUSE| + LOCK\ NEEDED:| + LOCK\ HELD\ ON\ ENTRY:| + READ\ LOCK\ HELD\ ON\ ENTRY:| + WRITE\ LOCK\ HELD\ ON\ ENTRY:| + LOCK\ ACQUIRED\ AS\ SIDE\ EFFECT:| + READ\ LOCK\ ACQUIRED\ AS\ SIDE\ EFFECT:| + WRITE\ LOCK\ ACQUIRED\ AS\ SIDE\ EFFECT:| + LOCK\ RELEASED\ AS\ SIDE\ EFFECT:| + LOCK\ UPGRADED\ AS\ SIDE\ EFFECT:| + LOCK\ DOWNGRADED\ AS\ SIDE\ EFFECT:| + FUNCTIONS\ CALLED\ THROUGH\ POINTER| + FUNCTIONS\ CALLED\ THROUGH\ MEMBER| + LOCK\ ORDER: + )/x; + +my $err_stat = 0; # exit status + +if ($#ARGV >= 0) { + foreach my $arg (@ARGV) { + my $fh = new IO::File $arg, "r"; + if (!defined($fh)) { + printf "%s: can not open\n", $arg; + } else { + &cstyle($arg, $fh); + close $fh; + } + } +} else { + &cstyle("<stdin>", *STDIN); +} +exit $err_stat; + +my $no_errs = 0; # set for CSTYLED-protected lines + +sub err($) { + my ($error) = @_; + unless ($no_errs) { + if ($verbose) { + printf $fmt, $filename, $., $error, $line; + } else { + printf $fmt, $filename, $., $error; + } + $err_stat = 1; + } +} + +sub err_prefix($$) { + my ($prevline, $error) = @_; + my $out = $prevline."\n".$line; + unless ($no_errs) { + printf $fmt, $filename, $., $error, $out; + $err_stat = 1; + } +} + +sub err_prev($) { + my ($error) = @_; + unless ($no_errs) { + printf $fmt, $filename, $. - 1, $error, $prev; + $err_stat = 1; + } +} + +sub cstyle($$) { + +my ($fn, $filehandle) = @_; +$filename = $fn; # share it globally + +my $in_cpp = 0; +my $next_in_cpp = 0; + +my $in_comment = 0; +my $in_header_comment = 0; +my $comment_done = 0; +my $in_warlock_comment = 0; +my $in_function = 0; +my $in_function_header = 0; +my $in_declaration = 0; +my $note_level = 0; +my $nextok = 0; +my $nocheck = 0; + +my $in_string = 0; + +my ($okmsg, $comment_prefix); + +$line = ''; +$prev = ''; +reset_indent(); + +line: while (<$filehandle>) { + s/\r?\n$//; # strip return and newline + + # save the original line, then remove all text from within + # double or single quotes, we do not want to check such text. + + $line = $_; + + # + # C allows strings to be continued with a backslash at the end of + # the line. We translate that into a quoted string on the previous + # line followed by an initial quote on the next line. + # + # (we assume that no-one will use backslash-continuation with character + # constants) + # + $_ = '"' . $_ if ($in_string && !$nocheck && !$in_comment); + + # + # normal strings and characters + # + s/'([^\\']|\\[^xX0]|\\0[0-9]*|\\[xX][0-9a-fA-F]*)'/''/g; + s/"([^\\"]|\\.)*"/\"\"/g; + + # + # detect string continuation + # + if ($nocheck || $in_comment) { + $in_string = 0; + } else { + # + # Now that all full strings are replaced with "", we check + # for unfinished strings continuing onto the next line. + # + $in_string = + (s/([^"](?:"")*)"([^\\"]|\\.)*\\$/$1""/ || + s/^("")*"([^\\"]|\\.)*\\$/""/); + } + + # + # figure out if we are in a cpp directive + # + $in_cpp = $next_in_cpp || /^\s*#/; # continued or started + $next_in_cpp = $in_cpp && /\\$/; # only if continued + + # strip off trailing backslashes, which appear in long macros + s/\s*\\$//; + + # an /* END CSTYLED */ comment ends a no-check block. + if ($nocheck) { + if (/\/\* *END *CSTYLED *\*\//) { + $nocheck = 0; + } else { + reset_indent(); + next line; + } + } + + # a /*CSTYLED*/ comment indicates that the next line is ok. + if ($nextok) { + if ($okmsg) { + err($okmsg); + } + $nextok = 0; + $okmsg = 0; + if (/\/\* *CSTYLED.*\*\//) { + /^.*\/\* *CSTYLED *(.*) *\*\/.*$/; + $okmsg = $1; + $nextok = 1; + } + $no_errs = 1; + } elsif ($no_errs) { + $no_errs = 0; + } + + # check length of line. + # first, a quick check to see if there is any chance of being too long. + if (($line =~ tr/\t/\t/) * 7 + length($line) > 80) { + # yes, there is a chance. + # replace tabs with spaces and check again. + my $eline = $line; + 1 while $eline =~ + s/\t+/' ' x (length($&) * 8 - length($`) % 8)/e; + if (length($eline) > 80) { + # allow long line if it is user visible string + # find if line start from " or L" and ends + # with " + 2 optional characters + # (these characters can be i.e. '");' '" \' or '",' etc...) + if($eline =~ /^ *L?".*"[^"]{0,2}$/) { + # check if entire line is one string literal + $eline =~ s/^ *L?"//; + $eline =~ s/"[^"]{0,2}$//; + + if($eline =~ /[^\\]"|[^\\](\\\\)+"/) { + err("line > 80 characters"); + } + } else { + err("line > 80 characters"); + } + } + } + + # ignore NOTE(...) annotations (assumes NOTE is on lines by itself). + if ($note_level || /\b_?NOTE\s*\(/) { # if in NOTE or this is NOTE + s/[^()]//g; # eliminate all non-parens + $note_level += s/\(//g - length; # update paren nest level + next; + } + + # a /* BEGIN CSTYLED */ comment starts a no-check block. + if (/\/\* *BEGIN *CSTYLED *\*\//) { + $nocheck = 1; + } + + # a /*CSTYLED*/ comment indicates that the next line is ok. + if (/\/\* *CSTYLED.*\*\//) { + /^.*\/\* *CSTYLED *(.*) *\*\/.*$/; + $okmsg = $1; + $nextok = 1; + } + if (/\/\/ *CSTYLED/) { + /^.*\/\/ *CSTYLED *(.*)$/; + $okmsg = $1; + $nextok = 1; + } + + # universal checks; apply to everything + if (/\t +\t/) { + err("spaces between tabs"); + } + if (/ \t+ /) { + err("tabs between spaces"); + } + if (/\s$/) { + err("space or tab at end of line"); + } + if (/[^ \t(]\/\*/ && !/\w\(\/\*.*\*\/\);/) { + err("comment preceded by non-blank"); + } + + # is this the beginning or ending of a function? + # (not if "struct foo\n{\n") + if (/^{$/ && $prev =~ /\)\s*(const\s*)?(\/\*.*\*\/\s*)?\\?$/) { + $in_function = 1; + $in_declaration = 1; + $in_function_header = 0; + $prev = $line; + next line; + } + if (/^}\s*(\/\*.*\*\/\s*)*$/) { + if ($prev =~ /^\s*return\s*;/) { + err_prev("unneeded return at end of function"); + } + $in_function = 0; + reset_indent(); # we don't check between functions + $prev = $line; + next line; + } + if (/^\w*\($/) { + $in_function_header = 1; + } + + if ($in_warlock_comment && /\*\//) { + $in_warlock_comment = 0; + $prev = $line; + next line; + } + + # a blank line terminates the declarations within a function. + # XXX - but still a problem in sub-blocks. + if ($in_declaration && /^$/) { + $in_declaration = 0; + } + + if ($comment_done) { + $in_comment = 0; + $in_header_comment = 0; + $comment_done = 0; + } + # does this looks like the start of a block comment? + if (/$hdr_comment_start/) { + if (!/^\t*\/\*/) { + err("block comment not indented by tabs"); + } + $in_comment = 1; + /^(\s*)\//; + $comment_prefix = $1; + if ($comment_prefix eq "") { + $in_header_comment = 1; + } + $prev = $line; + next line; + } + # are we still in the block comment? + if ($in_comment) { + if (/^$comment_prefix \*\/$/) { + $comment_done = 1; + } elsif (/\*\//) { + $comment_done = 1; + err("improper block comment close") + unless ($ignore_hdr_comment && $in_header_comment); + } elsif (!/^$comment_prefix \*[ \t]/ && + !/^$comment_prefix \*$/) { + err("improper block comment") + unless ($ignore_hdr_comment && $in_header_comment); + } + } + + if ($in_header_comment && $ignore_hdr_comment) { + $prev = $line; + next line; + } + + # check for errors that might occur in comments and in code. + + # allow spaces to be used to draw pictures in header and block comments. + if (/[^ ] / && !/".* .*"/ && !$in_header_comment && !$in_comment) { + err("spaces instead of tabs"); + } + if (/^ / && !/^ \*[ \t\/]/ && !/^ \*$/ && + (!/^ \w/ || $in_function != 0)) { + err("indent by spaces instead of tabs"); + } + if (/^\t+ [^ \t\*]/ || /^\t+ \S/ || /^\t+ \S/) { + err("continuation line not indented by 4 spaces"); + } + if (/$warlock_re/ && !/\*\//) { + $in_warlock_comment = 1; + $prev = $line; + next line; + } + if (/^\s*\/\*./ && !/^\s*\/\*.*\*\// && !/$hdr_comment_start/) { + err("improper first line of block comment"); + } + + if ($in_comment) { # still in comment, don't do further checks + $prev = $line; + next line; + } + + if ((/[^(]\/\*\S/ || /^\/\*\S/) && + !(/$lint_re/ || ($splint_comments && /$splint_re/))) { + err("missing blank after open comment"); + } + if (/\S\*\/[^)]|\S\*\/$/ && + !(/$lint_re/ || ($splint_comments && /$splint_re/))) { + err("missing blank before close comment"); + } + if (/\/\/\S/) { # C++ comments + err("missing blank after start comment"); + } + # check for unterminated single line comments, but allow them when + # they are used to comment out the argument list of a function + # declaration. + if (/\S.*\/\*/ && !/\S.*\/\*.*\*\// && !/\(\/\*/) { + err("unterminated single line comment"); + } + + if (/^(#else|#endif|#include)(.*)$/) { + $prev = $line; + if ($picky) { + my $directive = $1; + my $clause = $2; + # Enforce ANSI rules for #else and #endif: no noncomment + # identifiers are allowed after #endif or #else. Allow + # C++ comments since they seem to be a fact of life. + if ((($1 eq "#endif") || ($1 eq "#else")) && + ($clause ne "") && + (!($clause =~ /^\s+\/\*.*\*\/$/)) && + (!($clause =~ /^\s+\/\/.*$/))) { + err("non-comment text following " . + "$directive (or malformed $directive " . + "directive)"); + } + } + next line; + } + + # + # delete any comments and check everything else. Note that + # ".*?" is a non-greedy match, so that we don't get confused by + # multiple comments on the same line. + # + s/\/\*.*?\*\//\x01/g; + s/\/\/.*$/\x01/; # C++ comments + + # delete any trailing whitespace; we have already checked for that. + s/\s*$//; + + # following checks do not apply to text in comments. + + if (/[^ \t\+]\+[^\+=]/ || /[^\+]\+[^ \+=]/) { + err("missing space around + operator"); + } + if (/[^ \t]\+=/ || /\+=[^ ]/) { + err("missing space around += operator"); + } + if (/[^ \t\-]\-[^\->]/ && !/\(\w+\)\-\w/ && !/[\(\[]\-[\w \t]+[\)\],]/) { + err("missing space before - operator"); + } + if (/[^\-]\-[^ \-=>]/ && !/\(\-\w+\)/ && + !/(return|case|=|>|<|\?|:|,|^[ \t]+)[ \t]+\-[\w\(]/ && !/(\([^\)]+\)|\[|\()\-[\w\(\]]/) { + err("missing space after - operator"); + } + if (/(return|case|=|\?|:|,|\[)[ \t]+\-[ \t]/ || /[\(\[]\-[ \t]/) { + err("extra space after - operator"); + } + if (/[ \t]\+\+ /) { + err("extra space before or after ++ operator"); + } + if (/[ \t]\-\- /) { + err("extra space before or after -- operator"); + } + if (/[^ \t]\-=/ || /\-=[^ ]/) { + err("missing space around -= operator"); + } + if (/[^ \t][\%\/]/ || /[\%\/][^ =]/ || /[\%\/]=[^ ]/) { + err("missing space around one of operators: % %= / /="); + } + if (/[^ \t]\*=/ || /\*=[^ ]/) { + err("missing space around *= operator"); + } + if (/[^ \t\(\)\*\[]\*/) { + err("missing space before * operator"); + } + if (/\*[^ =\*\w\(,]/ && !/\(.+ \*+\)/ && !/\*\[\]/ && + !/\*\-\-\w/ && !/\*\+\+\w/ && !/\*\)/) { + err("missing space after * operator"); + } + if (/[^<>\s][!<>=]=/ || /[^<>][!<>=]=[^\s,]/ || + (/[^->]>[^,=>\s]/ && !/[^->]>$/) || + (/[^<]<[^,=<\s]/ && !/[^<]<$/) || + /[^<\s]<[^<]/ || /[^->\s]>[^>]/) { + err("missing space around relational operator"); + } + if (/\S>>=/ || /\S<<=/ || />>=\S/ || /<<=\S/ || /\S[-+*\/&|^%]=/ || + (/[^-+*\/&|^%!<>=\s]=[^=]/ && !/[^-+*\/&|^%!<>=\s]=$/) || + (/[^!<>=]=[^=\s]/ && !/[^!<>=]=$/)) { + # XXX - should only check this for C++ code + # XXX - there are probably other forms that should be allowed + if (!/\soperator=/) { + err("missing space around assignment operator"); + } + } + if (/[,;]\S/ && !/\bfor \(;;\)/) { + err("comma or semicolon followed by non-blank"); + } + # allow "for" statements to have empty "while" clauses + if (/\s[,;]/ && !/^[\t]+;$/ && !/^\s*for \([^;]*; ;[^;]*\)/) { + err("comma or semicolon preceded by blank"); + } + if (/^\s*(&&|\|\|)/) { + err("improper boolean continuation"); + } + if (/\S *(&&|\|\|)/ || /(&&|\|\|) *\S/) { + err("more than one space around boolean operator"); + } + if (/\b(for|if|while|switch|return|case)\(/) { + err("missing space between keyword and paren"); + } + if (/(\b(for|if|while|switch|return)\b.*){2,}/ && !/^#define/) { + # multiple "case" and "sizeof" allowed + err("more than one keyword on line"); + } + if (/\b(for|if|while|switch|return|case)\s\s+\(/ && + !/^#if\s+\(/) { + err("extra space between keyword and paren"); + } + # try to detect "func (x)" but not "if (x)" or + # "#define foo (x)" or "int (*func)();" + if (/\w\s\(/) { + my $s = $_; + # strip off all keywords on the line + s/\b(for|if|while|switch|return|case)\s\(/XXX(/g; + s/\b(sizeof|typeof|__typeof__)\s*\(/XXX(/g; + s/#elif\s\(/XXX(/g; + s/^#define\s+\w+\s+\(/XXX(/; + # do not match things like "void (*f)();" + # or "typedef void (func_t)();" + s/\w\s\(+\*/XXX(*/g; + s/\b($typename|void)\s+\(+/XXX(/og; + s/\btypedef\s($typename|void)\s+\(+/XXX(/og; + # do not match "__attribute__ ((format (...)))" + s/\b__attribute__\s*\(\(format\s*\(/__attribute__((XXX(/g; + if (/\w\s\(/) { + err("extra space between function name and left paren"); + } + $_ = $s; + } + # try to detect "int foo(x)", but not "extern int foo(x);" + # XXX - this still trips over too many legitimate things, + # like "int foo(x,\n\ty);" +# if (/^(\w+(\s|\*)+)+\w+\(/ && !/\)[;,](\s|\x01)*$/ && +# !/^(extern|static)\b/) { +# err("return type of function not on separate line"); +# } + # this is a close approximation + if (/^(\w+(\s|\*)+)+\w+\(.*\)(\s|\x01)*$/ && + !/^(extern|static)\b/) { + err("return type of function not on separate line"); + } + if (/^#define\t/ || /^#ifdef\t/ || /^#ifndef\t/) { + err("#define/ifdef/ifndef followed by tab instead of space"); + } + if (/^#define\s\s+/ || /^#ifdef\s\s+/ || /^#ifndef\s\s+/) { + err("#define/ifdef/ifndef followed by more than one space"); + } + # AON C-style doesn't require this. + #if (/^\s*return\W[^;]*;/ && !/^\s*return\s*\(.*\);/) { + # err("unparenthesized return expression"); + #} + if (/\bsizeof\b/ && !/\bsizeof\s*\(.*\)/) { + err("unparenthesized sizeof expression"); + } + if (/\b(sizeof|typeof)\b/ && /\b(sizeof|typeof)\s+\(.*\)/) { + err("spaces between sizeof/typeof expression and paren"); + } + if (/\(\s/) { + err("whitespace after left paren"); + } + # allow "for" statements to have empty "continue" clauses + if (/\s\)/ && !/^\s*for \([^;]*;[^;]*; \)/) { + err("whitespace before right paren"); + } + if (/^\s*\(void\)[^ ]/) { + err("missing space after (void) cast"); + } + if (/\S\{/ && !/\{\{/ && !/\(struct \w+\)\{/) { + err("missing space before left brace"); + } + if ($in_function && /^\s+{/ && + ($prev =~ /\)\s*$/ || $prev =~ /\bstruct\s+\w+$/)) { + err("left brace starting a line"); + } + if (/}(else|while)/) { + err("missing space after right brace"); + } + if (/}\s\s+(else|while)/) { + err("extra space after right brace"); + } + if (/\b_VOID\b|\bVOID\b|\bSTATIC\b/) { + err("obsolete use of VOID or STATIC"); + } + if (/\b($typename|void)\*/o) { + err("missing space between type name and *"); + } + if (/^\s+#/) { + err("preprocessor statement not in column 1"); + } + if (/^#\s/) { + err("blank after preprocessor #"); + } + if (/!\s*(strcmp|strncmp|bcmp)\s*\(/) { + err("don't use boolean ! with comparison functions"); + } + if (/^\S+\([\S\s]*\)\s*{/) { + err("brace of function definition not at beginning of line"); + } + if (/static\s+\S+\s*=\s*(0|NULL)\s*;/) { + err("static variable initialized with 0 or NULL"); + } + if (/typedef[\S\s]+\*\s*\w+\s*;/) { + err("typedefed pointer type"); + } + if (/unsigned\s+int\s/) { + err("'unsigned int' instead of just 'unsigned'"); + } + if (/long\s+long\s+int\s/) { + err("'long long int' instead of just 'long long'"); + } elsif (/long\s+int\s/) { + err("'long int' instead of just 'long'"); + } + + # + # We completely ignore, for purposes of indentation: + # * lines outside of functions + # * preprocessor lines + # + if ($check_continuation && $in_function && !$in_cpp) { + process_indent($_); + } + if ($picky) { + # try to detect spaces after casts, but allow (e.g.) + # "sizeof (int) + 1", "void (*funcptr)(int) = foo;", and + # "int foo(int) __NORETURN;" + if ((/^\($typename( \*+)?\)\s/o || + /\W\($typename( \*+)?\)\s/o) && + !/sizeof\($typename( \*)?\)\s/o && + !/\($typename( \*+)?\)\s+=[^=]/o) { + err("space after cast"); + } + if (/\b($typename|void)\s*\*\s/o && + !/\b($typename|void)\s*\*\s+const\b/o) { + err("unary * followed by space"); + } + } + if ($check_posix_types) { + # try to detect old non-POSIX types. + # POSIX requires all non-standard typedefs to end in _t, + # but historically these have been used. + if (/\b(unchar|ushort|uint|ulong|u_int|u_short|u_long|u_char|quad)\b/) { + err("non-POSIX typedef $1 used: use $old2posix{$1} instead"); + } + } + if ($heuristic) { + # cannot check this everywhere due to "struct {\n...\n} foo;" + if ($in_function && !$in_declaration && + /}./ && !/}\s+=/ && !/{.*}[;,]$/ && !/}(\s|\x01)*$/ && + !/} (else|while)/ && !/}}/) { + err("possible bad text following right brace"); + } + # cannot check this because sub-blocks in + # the middle of code are ok + if ($in_function && /^\s+{/) { + err("possible left brace starting a line"); + } + } + if (/^\s*else\W/) { + if ($prev =~ /^\s*}$/) { + err_prefix($prev, + "else and right brace should be on same line"); + } + } + $prev = $line; +} + +if ($prev eq "") { + err("last line in file is blank"); +} + +} + +# +# Continuation-line checking +# +# The rest of this file contains the code for the continuation checking +# engine. It's a pretty simple state machine which tracks the expression +# depth (unmatched '('s and '['s). +# +# Keep in mind that the argument to process_indent() has already been heavily +# processed; all comments have been replaced by control-A, and the contents of +# strings and character constants have been elided. +# + +my $cont_in; # currently inside of a continuation +my $cont_off; # skipping an initializer or definition +my $cont_noerr; # suppress cascading errors +my $cont_start; # the line being continued +my $cont_base; # the base indentation +my $cont_first; # this is the first line of a statement +my $cont_multiseg; # this continuation has multiple segments + +my $cont_special; # this is a C statement (if, for, etc.) +my $cont_macro; # this is a macro +my $cont_case; # this is a multi-line case + +my @cont_paren; # the stack of unmatched ( and [s we've seen + +sub +reset_indent() +{ + $cont_in = 0; + $cont_off = 0; +} + +sub +delabel($) +{ + # + # replace labels with tabs. Note that there may be multiple + # labels on a line. + # + local $_ = $_[0]; + + while (/^(\t*)( *(?:(?:\w+\s*)|(?:case\b[^:]*)): *)(.*)$/) { + my ($pre_tabs, $label, $rest) = ($1, $2, $3); + $_ = $pre_tabs; + while ($label =~ s/^([^\t]*)(\t+)//) { + $_ .= "\t" x (length($2) + length($1) / 8); + } + $_ .= ("\t" x (length($label) / 8)).$rest; + } + + return ($_); +} + +sub +process_indent($) +{ + require strict; + local $_ = $_[0]; # preserve the global $_ + + s/\x01//g; # No comments + s/\s+$//; # Strip trailing whitespace + + return if (/^$/); # skip empty lines + + # regexps used below; keywords taking (), macros, and continued cases + my $special = '(?:(?:\}\s*)?else\s+)?(?:if|for|while|switch)\b'; + my $macro = '[A-Z_][A-Z_0-9]*\('; + my $case = 'case\b[^:]*$'; + + # skip over enumerations, array definitions, initializers, etc. + if ($cont_off <= 0 && !/^\s*$special/ && + (/(?:(?:\b(?:enum|struct|union)\s*[^\{]*)|(?:\s+=\s*))\{/ || + (/^\s*{/ && $prev =~ /=\s*(?:\/\*.*\*\/\s*)*$/))) { + $cont_in = 0; + $cont_off = tr/{/{/ - tr/}/}/; + return; + } + if ($cont_off) { + $cont_off += tr/{/{/ - tr/}/}/; + return; + } + + if (!$cont_in) { + $cont_start = $line; + + if (/^\t* /) { + err("non-continuation indented 4 spaces"); + $cont_noerr = 1; # stop reporting + } + $_ = delabel($_); # replace labels with tabs + + # check if the statement is complete + return if (/^\s*\}?$/); + return if (/^\s*\}?\s*else\s*\{?$/); + return if (/^\s*do\s*\{?$/); + return if (/{$/); + return if (/}[,;]?$/); + + # Allow macros on their own lines + return if (/^\s*[A-Z_][A-Z_0-9]*$/); + + # cases we don't deal with, generally non-kosher + if (/{/) { + err("stuff after {"); + return; + } + + # Get the base line, and set up the state machine + /^(\t*)/; + $cont_base = $1; + $cont_in = 1; + @cont_paren = (); + $cont_first = 1; + $cont_multiseg = 0; + + # certain things need special processing + $cont_special = /^\s*$special/? 1 : 0; + $cont_macro = /^\s*$macro/? 1 : 0; + $cont_case = /^\s*$case/? 1 : 0; + } else { + $cont_first = 0; + + # Strings may be pulled back to an earlier (half-)tabstop + unless ($cont_noerr || /^$cont_base / || + (/^\t*(?: )?(?:gettext\()?\"/ && !/^$cont_base\t/)) { + err_prefix($cont_start, + "continuation should be indented 4 spaces"); + } + } + + my $rest = $_; # keeps the remainder of the line + + # + # The split matches 0 characters, so that each 'special' character + # is processed separately. Parens and brackets are pushed and + # popped off the @cont_paren stack. For normal processing, we wait + # until a ; or { terminates the statement. "special" processing + # (if/for/while/switch) is allowed to stop when the stack empties, + # as is macro processing. Case statements are terminated with a : + # and an empty paren stack. + # + foreach $_ (split /[^\(\)\[\]\{\}\;\:]*/) { + next if (length($_) == 0); + + # rest contains the remainder of the line + my $rxp = "[^\Q$_\E]*\Q$_\E"; + $rest =~ s/^$rxp//; + + if (/\(/ || /\[/) { + push @cont_paren, $_; + } elsif (/\)/ || /\]/) { + my $cur = $_; + tr/\)\]/\(\[/; + + my $old = (pop @cont_paren); + if (!defined($old)) { + err("unexpected '$cur'"); + $cont_in = 0; + last; + } elsif ($old ne $_) { + err("'$cur' mismatched with '$old'"); + $cont_in = 0; + last; + } + + # + # If the stack is now empty, do special processing + # for if/for/while/switch and macro statements. + # + next if (@cont_paren != 0); + if ($cont_special) { + if ($rest =~ /^\s*{?$/) { + $cont_in = 0; + last; + } + if ($rest =~ /^\s*;$/) { + err("empty if/for/while body ". + "not on its own line"); + $cont_in = 0; + last; + } + if (!$cont_first && $cont_multiseg == 1) { + err_prefix($cont_start, + "multiple statements continued ". + "over multiple lines"); + $cont_multiseg = 2; + } elsif ($cont_multiseg == 0) { + $cont_multiseg = 1; + } + # We've finished this section, start + # processing the next. + goto section_ended; + } + if ($cont_macro) { + if ($rest =~ /^$/) { + $cont_in = 0; + last; + } + } + } elsif (/\;/) { + if ($cont_case) { + err("unexpected ;"); + } elsif (!$cont_special) { + err("unexpected ;") if (@cont_paren != 0); + if (!$cont_first && $cont_multiseg == 1) { + err_prefix($cont_start, + "multiple statements continued ". + "over multiple lines"); + $cont_multiseg = 2; + } elsif ($cont_multiseg == 0) { + $cont_multiseg = 1; + } + if ($rest =~ /^$/) { + $cont_in = 0; + last; + } + if ($rest =~ /^\s*special/) { + err("if/for/while/switch not started ". + "on its own line"); + } + goto section_ended; + } + } elsif (/\{/) { + err("{ while in parens/brackets") if (@cont_paren != 0); + err("stuff after {") if ($rest =~ /[^\s}]/); + $cont_in = 0; + last; + } elsif (/\}/) { + err("} while in parens/brackets") if (@cont_paren != 0); + if (!$cont_special && $rest !~ /^\s*(while|else)\b/) { + if ($rest =~ /^$/) { + err("unexpected }"); + } else { + err("stuff after }"); + } + $cont_in = 0; + last; + } + } elsif (/\:/ && $cont_case && @cont_paren == 0) { + err("stuff after multi-line case") if ($rest !~ /$^/); + $cont_in = 0; + last; + } + next; +section_ended: + # End of a statement or if/while/for loop. Reset + # cont_special and cont_macro based on the rest of the + # line. + $cont_special = ($rest =~ /^\s*$special/)? 1 : 0; + $cont_macro = ($rest =~ /^\s*$macro/)? 1 : 0; + $cont_case = 0; + next; + } + $cont_noerr = 0 if (!$cont_in); +} diff --git a/src/pmdk/utils/docker/0001-travis-fix-travisci_build_coverity_scan.sh.patch b/src/pmdk/utils/docker/0001-travis-fix-travisci_build_coverity_scan.sh.patch new file mode 100644 index 000000000..9738942aa --- /dev/null +++ b/src/pmdk/utils/docker/0001-travis-fix-travisci_build_coverity_scan.sh.patch @@ -0,0 +1,27 @@ +From b5179dc4822eaab192361da05aa95d98f523960f Mon Sep 17 00:00:00 2001 +From: Lukasz Dorau <lukasz.dorau@intel.com> +Date: Mon, 7 May 2018 12:05:40 +0200 +Subject: [PATCH] travis: fix travisci_build_coverity_scan.sh + +--- + travisci_build_coverity_scan.sh | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/travisci_build_coverity_scan.sh b/travisci_build_coverity_scan.sh +index ad9d4afcf..562b08bcc 100644 +--- a/travisci_build_coverity_scan.sh ++++ b/travisci_build_coverity_scan.sh +@@ -92,8 +92,8 @@ response=$(curl \ + --form description="Travis CI build" \ + $UPLOAD_URL) + status_code=$(echo "$response" | sed -n '$p') +-if [ "$status_code" != "201" ]; then ++if [ "$status_code" != "200" ]; then + TEXT=$(echo "$response" | sed '$d') +- echo -e "\033[33;1mCoverity Scan upload failed: $TEXT.\033[0m" ++ echo -e "\033[33;1mCoverity Scan upload failed: $response.\033[0m" + exit 1 + fi +-- +2.13.6 + diff --git a/src/pmdk/utils/docker/README b/src/pmdk/utils/docker/README new file mode 100644 index 000000000..b4a271359 --- /dev/null +++ b/src/pmdk/utils/docker/README @@ -0,0 +1,19 @@ +Persistent Memory Development Kit + +This is utils/docker/README. + +Scripts in this directory let Travis CI run a Docker container with ubuntu- +or fedora-based environment and build PMDK project inside it. + +'build-local.sh' can be used to build PMDK locally. + +'build-CI.sh' is used for building PMDK on Travis and GitHub Actions CIs + +NOTE: +If you commit changes to any Dockerfile or shell script in the 'images' +subdirectory and then do git-rebase before pushing your commits to the +repository, make sure that you do not squash the commit which is the head in +your repository. This will let Travis and GitHub Actions CIs recreate +Docker images used during the build before the build. Otherwise the not-updated +Docker image will be pulled from the Docker Hub and used during the build on +Travis and GitHub Actions CIs. diff --git a/src/pmdk/utils/docker/build-CI.sh b/src/pmdk/utils/docker/build-CI.sh new file mode 100755 index 000000000..1b75d3b94 --- /dev/null +++ b/src/pmdk/utils/docker/build-CI.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# build-CI.sh - runs a Docker container from a Docker image with environment +# prepared for building PMDK project and starts building PMDK. +# +# This script is used for building PMDK on Travis and GitHub Actions CIs. +# + +set -e + +source $(dirname $0)/set-ci-vars.sh +source $(dirname $0)/set-vars.sh +source $(dirname $0)/valid-branches.sh + +if [[ "$CI_EVENT_TYPE" != "cron" && "$CI_BRANCH" != "coverity_scan" \ + && "$COVERITY" -eq 1 ]]; then + echo "INFO: Skip Coverity scan job if build is triggered neither by " \ + "'cron' nor by a push to 'coverity_scan' branch" + exit 0 +fi + +if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\ + && "$COVERITY" -ne 1 ]]; then + echo "INFO: Skip regular jobs if build is triggered either by 'cron'" \ + " or by a push to 'coverity_scan' branch" + exit 0 +fi + +if [[ -z "$OS" || -z "$OS_VER" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set properly " \ + "(eg. OS=ubuntu, OS_VER=16.04)." + exit 1 +fi + +if [[ -z "$HOST_WORKDIR" ]]; then + echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \ + "the root of the PMDK project on the host machine" + exit 1 +fi + +if [[ -z "$TEST_BUILD" ]]; then + TEST_BUILD=all +fi + +imageName=${DOCKERHUB_REPO}:1.10-${OS}-${OS_VER}-${CI_CPU_ARCH} +containerName=pmdk-${OS}-${OS_VER} + +if [[ $MAKE_PKG -eq 0 ]] ; then command="./run-build.sh"; fi +if [[ $MAKE_PKG -eq 1 ]] ; then command="./run-build-package.sh"; fi +if [[ $COVERAGE -eq 1 ]] ; then command="./run-coverage.sh"; ci_env=`bash <(curl -s https://codecov.io/env)`; fi + +if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\ + && "$COVERITY" -eq 1 ]]; then + command="./run-coverity.sh" +fi + +if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi +if [[ -f $CI_FILE_SKIP_BUILD_PKG_CHECK ]]; then BUILD_PACKAGE_CHECK=n; else BUILD_PACKAGE_CHECK=y; fi +if [ -z "$NDCTL_ENABLE" ]; then ndctl_enable=; else ndctl_enable="--env NDCTL_ENABLE=$NDCTL_ENABLE"; fi +if [[ $UBSAN -eq 1 ]]; then for x in C CPP LD; do declare EXTRA_${x}FLAGS=-fsanitize=undefined; done; fi + +# Only run doc update on $GITHUB_REPO master or stable branch +if [[ -z "${CI_BRANCH}" || -z "${TARGET_BRANCHES[${CI_BRANCH}]}" || "$CI_EVENT_TYPE" == "pull_request" || "$CI_REPO_SLUG" != "${GITHUB_REPO}" ]]; then + AUTO_DOC_UPDATE=0 +fi + +# Check if we are running on a CI (Travis or GitHub Actions) +[ -n "$GITHUB_ACTIONS" -o -n "$TRAVIS" ] && CI_RUN="YES" || CI_RUN="NO" + +# We have a blacklist only for ppc64le arch +if [[ "$CI_CPU_ARCH" == ppc64le ]] ; then BLACKLIST_FILE=../../utils/docker/ppc64le.blacklist; fi + +# docker on travis + ppc64le runs inside an LXD container and for security +# limits what can be done inside it, and as such, `docker run` fails with +# > the input device is not a TTY +# when using -t because of limited permissions to /dev imposed by LXD. +if [[ -n "$TRAVIS" && "$CI_CPU_ARCH" == ppc64le ]] || [[ -n "$GITHUB_ACTIONS" ]]; then + TTY='' +else + TTY='-t' +fi + +WORKDIR=/pmdk +SCRIPTSDIR=$WORKDIR/utils/docker + +# Run a container with +# - environment variables set (--env) +# - host directory containing PMDK source mounted (-v) +# - a tmpfs /tmp with the necessary size and permissions (--tmpfs)* +# - working directory set (-w) +# +# * We need a tmpfs /tmp inside docker but we cannot run it with --privileged +# and do it from inside, so we do using this docker-run option. +# By default --tmpfs add nosuid,nodev,noexec to the mount flags, we don't +# want that and just to make sure we add the usually default rw,relatime just +# in case docker change the defaults. +docker run --rm --name=$containerName -i $TTY \ + $DNS_SETTING \ + $ci_env \ + --env http_proxy=$http_proxy \ + --env https_proxy=$https_proxy \ + --env AUTO_DOC_UPDATE=$AUTO_DOC_UPDATE \ + --env CC=$PMDK_CC \ + --env CXX=$PMDK_CXX \ + --env VALGRIND=$VALGRIND \ + --env EXTRA_CFLAGS=$EXTRA_CFLAGS \ + --env EXTRA_CXXFLAGS=$EXTRA_CXXFLAGS \ + --env EXTRA_LDFLAGS=$EXTRA_LDFLAGS \ + --env REMOTE_TESTS=$REMOTE_TESTS \ + --env TEST_BUILD=$TEST_BUILD \ + --env WORKDIR=$WORKDIR \ + --env EXPERIMENTAL=$EXPERIMENTAL \ + --env BUILD_PACKAGE_CHECK=$BUILD_PACKAGE_CHECK \ + --env SCRIPTSDIR=$SCRIPTSDIR \ + --env TRAVIS=$TRAVIS \ + --env CI_COMMIT_RANGE=$CI_COMMIT_RANGE \ + --env CI_COMMIT=$CI_COMMIT \ + --env CI_REPO_SLUG=$CI_REPO_SLUG \ + --env CI_BRANCH=$CI_BRANCH \ + --env CI_EVENT_TYPE=$CI_EVENT_TYPE \ + --env DOC_UPDATE_GITHUB_TOKEN=$DOC_UPDATE_GITHUB_TOKEN \ + --env COVERITY_SCAN_TOKEN=$COVERITY_SCAN_TOKEN \ + --env COVERITY_SCAN_NOTIFICATION_EMAIL=$COVERITY_SCAN_NOTIFICATION_EMAIL \ + --env FAULT_INJECTION=$FAULT_INJECTION \ + --env GITHUB_ACTIONS=$GITHUB_ACTIONS \ + --env GITHUB_HEAD_REF=$GITHUB_HEAD_REF \ + --env GITHUB_REPO=$GITHUB_REPO \ + --env GITHUB_REPOSITORY=$GITHUB_REPOSITORY \ + --env GITHUB_REF=$GITHUB_REF \ + --env GITHUB_RUN_ID=$GITHUB_RUN_ID \ + --env GITHUB_SHA=$GITHUB_SHA \ + --env CI_RUN=$CI_RUN \ + --env SRC_CHECKERS=$SRC_CHECKERS \ + --env BLACKLIST_FILE=$BLACKLIST_FILE \ + $ndctl_enable \ + --tmpfs /tmp:rw,relatime,suid,dev,exec,size=6G \ + -v $HOST_WORKDIR:$WORKDIR \ + -v /etc/localtime:/etc/localtime \ + -w $SCRIPTSDIR \ + $imageName $command diff --git a/src/pmdk/utils/docker/build-local.sh b/src/pmdk/utils/docker/build-local.sh new file mode 100755 index 000000000..0fa5b2107 --- /dev/null +++ b/src/pmdk/utils/docker/build-local.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2020, Intel Corporation + +# +# build-local.sh - runs a Docker container from a Docker image with environment +# prepared for building PMDK project and starts building PMDK. +# +# This script is for building PMDK locally (not on CI). +# +# Notes: +# - run this script from its location or set the variable 'HOST_WORKDIR' to +# where the root of the PMDK project is on the host machine. +# - set variables 'OS' and 'OS_VER' properly to a system you want to build PMDK +# on (for proper values take a look on the list of Dockerfiles at the +# utils/docker/images directory), eg. OS=ubuntu, OS_VER=16.04. +# - set 'KEEP_TEST_CONFIG' variable to 1 if you do not want the tests to be +# reconfigured (your current test configuration will be preserved and used). +# - tests with Device Dax are not supported by pcheck yet, so do not provide +# these devices in your configuration. +# + +set -e + +# Environment variables that can be customized (default values are after dash): +export KEEP_CONTAINER=${KEEP_CONTAINER:-0} +export KEEP_TEST_CONFIG=${KEEP_TEST_CONFIG:-0} +export TEST_BUILD=${TEST_BUILD:-all} +export REMOTE_TESTS=${REMOTE_TESTS:-1} +export MAKE_PKG=${MAKE_PKG:-0} +export EXTRA_CFLAGS=${EXTRA_CFLAGS} +export EXTRA_CXXFLAGS=${EXTRA_CXXFLAGS:-} +export PMDK_CC=${PMDK_CC:-gcc} +export PMDK_CXX=${PMDK_CXX:-g++} +export EXPERIMENTAL=${EXPERIMENTAL:-n} +export VALGRIND=${VALGRIND:-1} +export DOCKERHUB_REPO=${DOCKERHUB_REPO:-pmem/pmdk} +export GITHUB_REPO=${GITHUB_REPO:-pmem/pmdk} + +if [[ -z "$OS" || -z "$OS_VER" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set " \ + "(eg. OS=ubuntu, OS_VER=16.04)." + exit 1 +fi + +if [[ -z "$HOST_WORKDIR" ]]; then + HOST_WORKDIR=$(readlink -f ../..) +fi + +if [[ "$KEEP_CONTAINER" != "1" ]]; then + RM_SETTING=" --rm" +fi + +imageName=${DOCKERHUB_REPO}:1.10-${OS}-${OS_VER}-${CI_CPU_ARCH} +containerName=pmdk-${OS}-${OS_VER} + +if [[ $MAKE_PKG -eq 1 ]] ; then + command="./run-build-package.sh" +else + command="./run-build.sh" +fi + +if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi +if [ -z "$NDCTL_ENABLE" ]; then ndctl_enable=; else ndctl_enable="--env NDCTL_ENABLE=$NDCTL_ENABLE"; fi + +WORKDIR=/pmdk +SCRIPTSDIR=$WORKDIR/utils/docker + +# Check if we are running on a CI (Travis or GitHub Actions) +[ -n "$GITHUB_ACTIONS" -o -n "$TRAVIS" ] && CI_RUN="YES" || CI_RUN="NO" + +echo Building ${OS}-${OS_VER} + +# Run a container with +# - environment variables set (--env) +# - host directory containing PMDK source mounted (-v) +# - a tmpfs /tmp with the necessary size and permissions (--tmpfs)* +# - working directory set (-w) +# +# * We need a tmpfs /tmp inside docker but we cannot run it with --privileged +# and do it from inside, so we do using this docker-run option. +# By default --tmpfs add nosuid,nodev,noexec to the mount flags, we don't +# want that and just to make sure we add the usually default rw,relatime just +# in case docker change the defaults. +docker run --name=$containerName -ti \ + $RM_SETTING \ + $DNS_SETTING \ + --env http_proxy=$http_proxy \ + --env https_proxy=$https_proxy \ + --env CC=$PMDK_CC \ + --env CXX=$PMDK_CXX \ + --env VALGRIND=$VALGRIND \ + --env EXTRA_CFLAGS=$EXTRA_CFLAGS \ + --env EXTRA_CXXFLAGS=$EXTRA_CXXFLAGS \ + --env EXTRA_LDFLAGS=$EXTRA_LDFLAGS \ + --env REMOTE_TESTS=$REMOTE_TESTS \ + --env CONFIGURE_TESTS=$CONFIGURE_TESTS \ + --env TEST_BUILD=$TEST_BUILD \ + --env WORKDIR=$WORKDIR \ + --env EXPERIMENTAL=$EXPERIMENTAL \ + --env SCRIPTSDIR=$SCRIPTSDIR \ + --env KEEP_TEST_CONFIG=$KEEP_TEST_CONFIG \ + --env CI_RUN=$CI_RUN \ + --env BLACKLIST_FILE=$BLACKLIST_FILE \ + $ndctl_enable \ + --tmpfs /tmp:rw,relatime,suid,dev,exec,size=6G \ + -v $HOST_WORKDIR:$WORKDIR \ + -v /etc/localtime:/etc/localtime \ + $DAX_SETTING \ + -w $SCRIPTSDIR \ + $imageName $command diff --git a/src/pmdk/utils/docker/configure-tests.sh b/src/pmdk/utils/docker/configure-tests.sh new file mode 100755 index 000000000..23148ed4f --- /dev/null +++ b/src/pmdk/utils/docker/configure-tests.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# configure-tests.sh - is called inside a Docker container; configures tests +# and ssh server for use during build of PMDK project. +# + +set -e + +# Configure tests +cat << EOF > $WORKDIR/src/test/testconfig.sh +LONGDIR=LoremipsumdolorsitametconsecteturadipiscingelitVivamuslacinianibhattortordictumsollicitudinNullamvariusvestibulumligulaetegestaselitsemperidMaurisultriciesligulaeuipsumtinciduntluctusMorbimaximusvariusdolorid +# this path is ~3000 characters long +DIRSUFFIX="$LONGDIR/$LONGDIR/$LONGDIR/$LONGDIR/$LONGDIR" +NON_PMEM_FS_DIR=/tmp +PMEM_FS_DIR=/tmp +PMEM_FS_DIR_FORCE_PMEM=1 +TEST_BUILD="debug nondebug" +ENABLE_SUDO_TESTS=y +TM=1 +EOF + +# Configure remote tests +if [[ $REMOTE_TESTS -eq 1 ]]; then + echo "Configuring remote tests" + cat << EOF >> $WORKDIR/src/test/testconfig.sh +NODE[0]=127.0.0.1 +NODE_WORKING_DIR[0]=/tmp/node0 +NODE_ADDR[0]=127.0.0.1 +NODE_ENV[0]="PMEM_IS_PMEM_FORCE=1" +NODE[1]=127.0.0.1 +NODE_WORKING_DIR[1]=/tmp/node1 +NODE_ADDR[1]=127.0.0.1 +NODE_ENV[1]="PMEM_IS_PMEM_FORCE=1" +NODE[2]=127.0.0.1 +NODE_WORKING_DIR[2]=/tmp/node2 +NODE_ADDR[2]=127.0.0.1 +NODE_ENV[2]="PMEM_IS_PMEM_FORCE=1" +NODE[3]=127.0.0.1 +NODE_WORKING_DIR[3]=/tmp/node3 +NODE_ADDR[3]=127.0.0.1 +NODE_ENV[3]="PMEM_IS_PMEM_FORCE=1" +TEST_BUILD="debug nondebug" +TEST_PROVIDERS=sockets +EOF + + mkdir -p ~/.ssh/cm + + cat << EOF >> ~/.ssh/config +Host 127.0.0.1 + StrictHostKeyChecking no + ControlPath ~/.ssh/cm/%r@%h:%p + ControlMaster auto + ControlPersist 10m +EOF + + if [ ! -f /etc/ssh/ssh_host_rsa_key ] + then + (echo $USERPASS | sudo -S ssh-keygen -t rsa -C $USER@$HOSTNAME -P '' -f /etc/ssh/ssh_host_rsa_key) + fi + echo $USERPASS | sudo -S sh -c 'cat /etc/ssh/ssh_host_rsa_key.pub >> /etc/ssh/authorized_keys' + ssh-keygen -t rsa -C $USER@$HOSTNAME -P '' -f ~/.ssh/id_rsa + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys + chmod -R 700 ~/.ssh + chmod 640 ~/.ssh/authorized_keys + chmod 600 ~/.ssh/config + + # Start ssh service + echo $USERPASS | sudo -S $START_SSH_COMMAND + + ssh 127.0.0.1 exit 0 +else + echo "Skipping remote tests" + echo + echo "Removing all libfabric.pc files in order to simulate that libfabric is not installed:" + find /usr -name "libfabric.pc" 2>/dev/null || true + echo $USERPASS | sudo -S sh -c 'find /usr -name "libfabric.pc" -exec rm -f {} + 2>/dev/null' +fi + +# Configure python tests + cat << EOF >> $WORKDIR/src/test/testconfig.py +config = { + 'unittest_log_level': 1, + 'cacheline_fs_dir': '/tmp', + 'force_cacheline': True, + 'page_fs_dir': '/tmp', + 'force_page': False, + 'byte_fs_dir': '/tmp', + 'force_byte': True, + 'tm': True, + 'test_type': 'check', + 'granularity': 'all', + 'fs_dir_force_pmem': 0, + 'keep_going': False, + 'timeout': '3m', + 'build': ['debug', 'release'], + 'force_enable': None, + 'device_dax_path': [], + 'fail_on_skip': False, + 'enable_admin_tests': True + } +EOF + diff --git a/src/pmdk/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch b/src/pmdk/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch new file mode 100644 index 000000000..7377d07f6 --- /dev/null +++ b/src/pmdk/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch @@ -0,0 +1,37 @@ +From d633d3b0a5f03be280efb80a69b9d5ed4e9c4d56 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= <lukasz.stolarczuk@intel.com> +Date: Tue, 14 Jul 2020 13:58:34 +0200 +Subject: [PATCH] fix generating gcov files and turn-off verbose log + +--- + codecov | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/codecov b/codecov +index e702ecd..0a2f4d8 100755 +--- a/codecov ++++ b/codecov +@@ -1108,9 +1108,9 @@ then + if [ "$ft_gcovout" = "0" ]; + then + # suppress gcov output +- bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -exec $gcov_exe -pb $gcov_arg {} +" >/dev/null 2>&1 || true ++ bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -execdir $gcov_exe -pb $gcov_arg {} \;" >/dev/null 2>&1 || true + else +- bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -exec $gcov_exe -pb $gcov_arg {} +" || true ++ bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -execdir $gcov_exe -pb $gcov_arg {} \;" || true + fi + else + say "${e}==>${x} gcov disabled" +@@ -1425,7 +1425,7 @@ do + report_len=$(wc -c < "$file") + if [ "$report_len" -ne 0 ]; + then +- say " ${g}+${x} $file ${e}bytes=$(echo "$report_len" | tr -d ' ')${x}" ++ #say " ${g}+${x} $file ${e}bytes=$(echo "$report_len" | tr -d ' ')${x}" + # append to to upload + _filename=$(basename "$file") + if [ "${_filename##*.}" = 'gcov' ]; +-- +2.25.1 + diff --git a/src/pmdk/utils/docker/images/Dockerfile.fedora-31 b/src/pmdk/utils/docker/images/Dockerfile.fedora-31 new file mode 100644 index 000000000..a699fee85 --- /dev/null +++ b/src/pmdk/utils/docker/images/Dockerfile.fedora-31 @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# Dockerfile - a 'recipe' for Docker to build an image of fedora-based +# environment for building the PMDK project. +# + +# Pull base image +FROM fedora:31 +MAINTAINER piotr.balcer@intel.com + +# libfabric (optional if libfabric-dev >= 1.4.2 is installed) +ENV FABRIC_DEPS "\ + autoconf \ + automake \ + libtool \ + wget" + +ENV VALGRIND_DEPS "\ + autoconf \ + automake \ + file \ + findutils \ + git" + +# pmdk base +ENV BASE_DEPS "\ + git \ + daxctl-devel \ + make \ + ndctl-devel \ + pkgconfig" + +# benchmarks (optional) +ENV BENCH_DEPS "\ + glib2-devel" + +# examples (optional) +ENV EXAMPLES_DEPS "\ + fuse \ + fuse-devel \ + ncurses-devel \ + libuv-devel" + +# documentation (optional) +ENV DOC_DEPS "\ + pandoc" + +# tests +ENV TESTS_DEPS "\ + bc \ + gdb \ + libunwind-devel \ + ndctl \ + openssh-server \ + strace" + +# packaging +ENV PACKAGING_DEPS "\ + rpm-build \ + rpm-build-libs \ + rpmdevtools" + +# Coverity +ENV COVERITY_DEPS "\ + gcc \ + wget" + +# misc +ENV MISC_DEPS "\ + clang \ + hub \ + lbzip2 \ + man \ + python3-flake8 \ + rsync \ + shadow-utils \ + sudo \ + tar \ + which \ + xmlto" + +# Copy install valgrind script +COPY install-valgrind.sh install-valgrind.sh + +# Copy install libfabric script +COPY install-libfabric.sh install-libfabric.sh + +RUN dnf update -y && dnf install -y \ + $FABRIC_DEPS \ + $VALGRIND_DEPS \ + $BASE_DEPS \ + $BENCH_DEPS \ + $EXAMPLES_DEPS \ + $DOC_DEPS \ + $TESTS_DEPS \ + $PACKAGING_DEPS \ + $COVERITY_DEPS \ + $MISC_DEPS \ + $TESTS_DEPS \ + && ./install-valgrind.sh fedora \ + && ./install-libfabric.sh fedora \ + && dnf clean all + +# Add user +ENV USER pmdkuser +ENV USERPASS pmdkpass +RUN useradd -m $USER +RUN echo "$USER:$USERPASS" | chpasswd +RUN gpasswd wheel -a $USER +RUN echo "%wheel ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +USER $USER + +# Set required environment variables +ENV OS fedora +ENV OS_VER 31 +ENV START_SSH_COMMAND /usr/sbin/sshd +ENV PACKAGE_MANAGER rpm +ENV NOTTY 1 diff --git a/src/pmdk/utils/docker/images/Dockerfile.ubuntu-19.10 b/src/pmdk/utils/docker/images/Dockerfile.ubuntu-19.10 new file mode 100644 index 000000000..9b61f8953 --- /dev/null +++ b/src/pmdk/utils/docker/images/Dockerfile.ubuntu-19.10 @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based +# environment for building the PMDK project. +# + +# Pull base image +FROM ubuntu:19.10 +MAINTAINER piotr.balcer@intel.com + +ENV DEBIAN_FRONTEND noninteractive + +# Additional parameters to build docker without building components +ARG SKIP_SCRIPTS_DOWNLOAD + +# libfabric (optional if libfabric-dev >= 1.4.2 is installed) +ENV FABRIC_DEPS "autoconf \ + automake \ + build-essential \ + libtool \ + unzip \ + wget" + +ENV VALGRIND_DEPS "autoconf \ + automake \ + build-essential \ + git" + +# pmdk base +ENV BASE_DEPS "build-essential \ + git \ + libdaxctl-dev \ + libndctl-dev \ + pkg-config" + +# benchmarks (optional) +ENV BENCH_DEPS libglib2.0-dev + +# examples (optional) +ENV EXAMPLES_DEPS "libfuse-dev \ + libncurses5-dev \ + libuv1-dev" + +# documentation (optional) +ENV DOC_DEPS pandoc + +# tests +ENV TESTS_DEPS "bc \ + gdb \ + libc6-dbg \ + libunwind-dev \ + ndctl \ + python3 \ + ssh \ + strace" + +# packaging +ENV PACKAGING_DEPS "debhelper \ + devscripts \ + fakeroot" + +# CodeCov +ENV CODECOV_DEPS curl + +# Coverity +ENV COVERITY_DEPS ruby gcc g++ wget + +# misc +ENV MISC_DEPS "clang \ + clang-format \ + flake8 \ + sudo \ + whois" + +# Copy install valgrind script +COPY install-valgrind.sh install-valgrind.sh + +# Copy install libfabric script +COPY install-libfabric.sh install-libfabric.sh + +# Copy codecov patch and script to download scripts required in run-*.sh +COPY download-scripts.sh download-scripts.sh +COPY 0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch \ + 0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch + +# Update the Apt cache and install basic tools +RUN apt-get update && apt-get dist-upgrade -y \ + && apt-get install -y --no-install-recommends \ + $FABRIC_DEPS \ + $VALGRIND_DEPS \ + $BASE_DEPS \ + $BENCH_DEPS \ + $EXAMPLES_DEPS \ + $DOC_DEPS \ + $TESTS_DEPS \ + $PACKAGING_DEPS \ + $CODECOV_DEPS \ + $COVERITY_DEPS \ + $MISC_DEPS \ + && ./install-valgrind.sh ubuntu \ + && ./install-libfabric.sh \ + && ./download-scripts.sh \ + && rm -rf /var/lib/apt/lists/* + +# Add user +ENV USER pmdkuser +ENV USERPASS pmdkpass +RUN useradd -m $USER -g sudo -p `mkpasswd $USERPASS` +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +# switch user +USER $USER + +# Set required environment variables +ENV OS ubuntu +ENV OS_VER 19.10 +ENV START_SSH_COMMAND service ssh start +ENV PACKAGE_MANAGER dpkg +ENV NOTTY 1 diff --git a/src/pmdk/utils/docker/images/README b/src/pmdk/utils/docker/images/README new file mode 100644 index 000000000..0b8c551e4 --- /dev/null +++ b/src/pmdk/utils/docker/images/README @@ -0,0 +1,6 @@ +Persistent Memory Development Kit + +This is utils/docker/images/README. + +Scripts in this directory let you prepare Docker images for building +PMDK project under specified OS (ubuntu, fedora). diff --git a/src/pmdk/utils/docker/images/build-image.sh b/src/pmdk/utils/docker/images/build-image.sh new file mode 100755 index 000000000..b9e7a2a4b --- /dev/null +++ b/src/pmdk/utils/docker/images/build-image.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# build-image.sh <OS-VER> <ARCH> - prepares a Docker image with <OS>-based +# environment intended for the <ARCH> CPU architecture +# designed for building PMDK project, according to +# the Dockerfile.<OS-VER> file located in the same directory. +# +# The script can be run locally. +# + +set -e + +OS_VER=$1 +CPU_ARCH=$2 + +function usage { + echo "Usage:" + echo " build-image.sh <OS-VER> <ARCH>" + echo "where:" + echo " <OS-VER> - can be for example 'ubuntu-19.10' provided "\ + "a Dockerfile named 'Dockerfile.ubuntu-19.10' "\ + "exists in the current directory and" + echo " <ARCH> - is a CPU architecture, for example 'x86_64'" +} + +# Check if two first arguments are not empty +if [[ -z "$2" ]]; then + usage + exit 1 +fi + +# Check if the file Dockerfile.OS-VER exists +if [[ ! -f "Dockerfile.$OS_VER" ]]; then + echo "Error: Dockerfile.$OS_VER does not exist." + echo + usage + exit 1 +fi + +if [[ -z "${DOCKERHUB_REPO}" ]]; then + echo "Error: DOCKERHUB_REPO environment variable is not set" + exit 1 +fi + +# Build a Docker image tagged with ${DOCKERHUB_REPO}:OS-VER-ARCH +tag=${DOCKERHUB_REPO}:1.10-${OS_VER}-${CPU_ARCH} +docker build -t $tag \ + --build-arg http_proxy=$http_proxy \ + --build-arg https_proxy=$https_proxy \ + -f Dockerfile.$OS_VER . diff --git a/src/pmdk/utils/docker/images/download-scripts.sh b/src/pmdk/utils/docker/images/download-scripts.sh new file mode 100755 index 000000000..fae37b464 --- /dev/null +++ b/src/pmdk/utils/docker/images/download-scripts.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020, Intel Corporation + +# +# download-scripts.sh - downloads specific version of codecov's bash +# script to generate and upload reports. It's useful, +# since unverified version may break coverage results. +# + +set -e + +# master: Merge pull request #342 from codecov/revert-proj-name-..., 18.08.2020 +CODECOV_VERSION="e877c1280cc6e902101fb5df2981ed1c962da7f0" + +if [ "${SKIP_SCRIPTS_DOWNLOAD}" ]; then + echo "Variable 'SKIP_SCRIPTS_DOWNLOAD' is set; skipping scripts' download" + exit +fi + +mkdir -p /opt/scripts + +# Download codecov's bash script +git clone https://github.com/codecov/codecov-bash +cd codecov-bash +git checkout $CODECOV_VERSION + +git apply ../0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch +mv -v codecov /opt/scripts/codecov + +cd .. +rm -rf codecov-bash diff --git a/src/pmdk/utils/docker/images/install-libfabric.sh b/src/pmdk/utils/docker/images/install-libfabric.sh new file mode 100755 index 000000000..355f24525 --- /dev/null +++ b/src/pmdk/utils/docker/images/install-libfabric.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# install-libfabric.sh - installs a customized version of libfabric +# + +set -e + +OS=$1 + +# Keep in sync with requirements in src/common.inc. +libfabric_ver=1.4.2 +libfabric_url=https://github.com/ofiwg/libfabric/archive +libfabric_dir=libfabric-$libfabric_ver +libfabric_tarball=v${libfabric_ver}.zip +wget "${libfabric_url}/${libfabric_tarball}" +unzip $libfabric_tarball + +cd $libfabric_dir + +# XXX HACK HACK HACK +# Disable use of spin locks in libfabric. +# +# spinlocks do not play well (IOW at all) with cpu-constrained environments, +# like GitHub Actions, and this leads to timeouts of some PMDK's tests. +# This change speeds up pmempool_sync_remote/TEST28-31 by a factor of 20-30. +# +perl -pi -e 's/have_spinlock=1/have_spinlock=0/' configure.ac +# XXX HACK HACK HACK + +./autogen.sh +./configure --prefix=/usr --enable-sockets +make -j$(nproc) +make -j$(nproc) install + +cd .. +rm -f ${libfabric_tarball} +rm -rf ${libfabric_dir} diff --git a/src/pmdk/utils/docker/images/install-libndctl.sh b/src/pmdk/utils/docker/images/install-libndctl.sh new file mode 100755 index 000000000..fc8cee3fb --- /dev/null +++ b/src/pmdk/utils/docker/images/install-libndctl.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2019, Intel Corporation + +# +# install-libndctl.sh - installs libndctl +# + +set -e + +OS=$2 + +echo "==== clone ndctl repo ====" +git clone https://github.com/pmem/ndctl.git +cd ndctl +git checkout $1 + +if [ "$OS" = "fedora" ]; then + +echo "==== setup rpmbuild tree ====" +rpmdev-setuptree + +RPMDIR=$HOME/rpmbuild/ +VERSION=$(./git-version) +SPEC=./rhel/ndctl.spec + +echo "==== create source tarball =====" +git archive --format=tar --prefix="ndctl-${VERSION}/" HEAD | gzip > "$RPMDIR/SOURCES/ndctl-${VERSION}.tar.gz" + +echo "==== build ndctl ====" +./autogen.sh +./configure --disable-docs +make -j$(nproc) + +echo "==== build ndctl packages ====" +rpmbuild -ba $SPEC + +echo "==== install ndctl packages ====" +RPM_ARCH=$(uname -m) +rpm -i $RPMDIR/RPMS/$RPM_ARCH/*.rpm + +echo "==== cleanup ====" +rm -rf $RPMDIR + +else + +echo "==== build ndctl ====" +./autogen.sh +./configure --disable-docs +make -j$(nproc) + +echo "==== install ndctl ====" +make -j$(nproc) install + +echo "==== cleanup ====" + +fi + +cd .. +rm -rf ndctl diff --git a/src/pmdk/utils/docker/images/install-valgrind.sh b/src/pmdk/utils/docker/images/install-valgrind.sh new file mode 100755 index 000000000..97babcb67 --- /dev/null +++ b/src/pmdk/utils/docker/images/install-valgrind.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# install-valgrind.sh - installs valgrind for persistent memory +# + +set -e + +OS=$1 + +install_upstream_from_distro() { + case "$OS" in + fedora) dnf install -y valgrind ;; + ubuntu) apt-get install -y --no-install-recommends valgrind ;; + *) return 1 ;; + esac +} + +install_upstream_3_16_1() { + git clone git://sourceware.org/git/valgrind.git + cd valgrind + # valgrind v3.16.1 upstream + git checkout VALGRIND_3_16_BRANCH + ./autogen.sh + ./configure + make -j$(nproc) + make -j$(nproc) install + cd .. + rm -rf valgrind +} + +install_custom-pmem_from_source() { + git clone https://github.com/pmem/valgrind.git + cd valgrind + # valgrind v3.15 with pmemcheck + # 2020.04.01 Merge pull request #78 from marcinslusarz/opt3 + git checkout 759686fd66cc0105df8311cfe676b0b2f9e89196 + ./autogen.sh + ./configure + make -j$(nproc) + make -j$(nproc) install + cd .. + rm -rf valgrind +} + +ARCH=$(uname -m) +case "$ARCH" in + ppc64le) install_upstream_3_16_1 ;; + *) install_custom-pmem_from_source ;; +esac diff --git a/src/pmdk/utils/docker/images/push-image.sh b/src/pmdk/utils/docker/images/push-image.sh new file mode 100755 index 000000000..03b935213 --- /dev/null +++ b/src/pmdk/utils/docker/images/push-image.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# push-image.sh - pushes the Docker image to the Docker Hub. +# +# The script utilizes $DOCKERHUB_USER and $DOCKERHUB_PASSWORD variables +# to log in to Docker Hub. The variables can be set in the Travis project's +# configuration for automated builds. +# + +set -e + +source $(dirname $0)/../set-ci-vars.sh + +if [[ -z "$OS" ]]; then + echo "OS environment variable is not set" + exit 1 +fi + +if [[ -z "$OS_VER" ]]; then + echo "OS_VER environment variable is not set" + exit 1 +fi + +if [[ -z "$CI_CPU_ARCH" ]]; then + echo "CI_CPU_ARCH environment variable is not set" + exit 1 +fi + +if [[ -z "${DOCKERHUB_REPO}" ]]; then + echo "DOCKERHUB_REPO environment variable is not set" + exit 1 +fi + +TAG="1.10-${OS}-${OS_VER}-${CI_CPU_ARCH}" + +# Check if the image tagged with pmdk/OS-VER exists locally +if [[ ! $(docker images -a | awk -v pattern="^${DOCKERHUB_REPO}:${TAG}\$" \ + '$1":"$2 ~ pattern') ]] +then + echo "ERROR: Docker image tagged ${DOCKERHUB_REPO}:${TAG} does not exists locally." + exit 1 +fi + +# Log in to the Docker Hub +docker login -u="$DOCKERHUB_USER" -p="$DOCKERHUB_PASSWORD" + +# Push the image to the repository +docker push ${DOCKERHUB_REPO}:${TAG} diff --git a/src/pmdk/utils/docker/ppc64le.blacklist b/src/pmdk/utils/docker/ppc64le.blacklist new file mode 100644 index 000000000..6002ad91f --- /dev/null +++ b/src/pmdk/utils/docker/ppc64le.blacklist @@ -0,0 +1,19 @@ +ex_librpmem_basic +ex_librpmem_hello +ex_librpmem_manpage +libpmempool_rm_remote +obj_basic_integration +obj_check_remote +obj_ctl_debug +obj_mem +obj_memcheck_register +obj_pmalloc_mt +obj_rpmem_basic_integration +obj_rpmem_heap_interrupt +obj_rpmem_heap_state +obj_ulog_size +pmempool_create +pmempool_sync_remote +pmempool_transform_remote +rpmem_basic +rpmem_fip diff --git a/src/pmdk/utils/docker/prepare-for-build.sh b/src/pmdk/utils/docker/prepare-for-build.sh new file mode 100755 index 000000000..f18c18c68 --- /dev/null +++ b/src/pmdk/utils/docker/prepare-for-build.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# prepare-for-build.sh - is called inside a Docker container; prepares +# the environment inside a Docker container for +# running build of PMDK project. +# + +set -e + +# This should be run only on CIs +if [ "$CI_RUN" == "YES" ]; then + # Make sure $WORKDIR has correct access rights + # - set them to the current UID and GID + echo $USERPASS | sudo -S chown -R $(id -u).$(id -g) $WORKDIR +fi + +# Configure tests (e.g. ssh for remote tests) unless the current configuration +# should be preserved +KEEP_TEST_CONFIG=${KEEP_TEST_CONFIG:-0} +if [[ "$KEEP_TEST_CONFIG" == 0 ]]; then + ./configure-tests.sh +fi diff --git a/src/pmdk/utils/docker/pull-or-rebuild-image.sh b/src/pmdk/utils/docker/pull-or-rebuild-image.sh new file mode 100755 index 000000000..a3a23cdd1 --- /dev/null +++ b/src/pmdk/utils/docker/pull-or-rebuild-image.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# pull-or-rebuild-image.sh - rebuilds the Docker image used in the +# current Travis build if necessary. +# +# The script rebuilds the Docker image if the Dockerfile for the current +# OS version (Dockerfile.${OS}-${OS_VER}) or any .sh script from the directory +# with Dockerfiles were modified and committed. +# +# If the Travis build is not of the "pull_request" type (i.e. in case of +# merge after pull_request) and it succeed, the Docker image should be pushed +# to the Docker Hub repository. An empty file is created to signal that to +# further scripts. +# +# If the Docker image does not have to be rebuilt, it will be pulled from +# Docker Hub. +# + +set -e + +source $(dirname $0)/set-ci-vars.sh +source $(dirname $0)/set-vars.sh + +if [[ "$CI_EVENT_TYPE" != "cron" && "$CI_BRANCH" != "coverity_scan" \ + && "$COVERITY" -eq 1 ]]; then + echo "INFO: Skip Coverity scan job if build is triggered neither by " \ + "'cron' nor by a push to 'coverity_scan' branch" + exit 0 +fi + +if [[ ( "$CI_EVENT_TYPE" == "cron" || "$CI_BRANCH" == "coverity_scan" )\ + && "$COVERITY" -ne 1 ]]; then + echo "INFO: Skip regular jobs if build is triggered either by 'cron'" \ + " or by a push to 'coverity_scan' branch" + exit 0 +fi + +if [[ -z "$OS" || -z "$OS_VER" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set properly " \ + "(eg. OS=ubuntu, OS_VER=16.04)." + exit 1 +fi + +if [[ -z "$HOST_WORKDIR" ]]; then + echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \ + "the root of the PMDK project on the host machine" + exit 1 +fi + +# Find all the commits for the current build +if [ -n "$CI_COMMIT_RANGE" ]; then + commits=$(git rev-list $CI_COMMIT_RANGE) +else + commits=$CI_COMMIT +fi + +echo "Commits in the commit range:" +for commit in $commits; do echo $commit; done + +# Get the list of files modified by the commits +files=$(for commit in $commits; do git diff-tree --no-commit-id --name-only \ + -r $commit; done | sort -u) +echo "Files modified within the commit range:" +for file in $files; do echo $file; done + +# Path to directory with Dockerfiles and image building scripts +images_dir_name=images +base_dir=utils/docker/$images_dir_name + +# Check if committed file modifications require the Docker image to be rebuilt +for file in $files; do + # Check if modified files are relevant to the current build + if [[ $file =~ ^($base_dir)\/Dockerfile\.($OS)-($OS_VER)$ ]] \ + || [[ $file =~ ^($base_dir)\/.*\.sh$ ]] + then + # Rebuild Docker image for the current OS version + echo "Rebuilding the Docker image for the Dockerfile.$OS-$OS_VER" + pushd $images_dir_name + ./build-image.sh ${OS}-${OS_VER} ${CI_CPU_ARCH} + popd + + # Check if the image has to be pushed to Docker Hub + # (i.e. the build is triggered by commits to the $GITHUB_REPO + # repository's stable-* or master branch, and the Travis build is not + # of the "pull_request" type). In that case, create the empty + # file. + if [[ "$CI_REPO_SLUG" == "$GITHUB_REPO" \ + && ($CI_BRANCH == stable-* || $CI_BRANCH == devel-* || $CI_BRANCH == master) \ + && $CI_EVENT_TYPE != "pull_request" \ + && $PUSH_IMAGE == "1" ]] + then + echo "The image will be pushed to Docker Hub" + touch $CI_FILE_PUSH_IMAGE_TO_REPO + else + echo "Skip pushing the image to Docker Hub" + fi + + if [[ $PUSH_IMAGE == "1" ]] + then + echo "Skip build package check if image has to be pushed" + touch $CI_FILE_SKIP_BUILD_PKG_CHECK + fi + exit 0 + fi +done + +# Getting here means rebuilding the Docker image is not required. +# Pull the image from Docker Hub. +docker pull ${DOCKERHUB_REPO}:1.10-${OS}-${OS_VER}-${CI_CPU_ARCH} diff --git a/src/pmdk/utils/docker/run-build-package.sh b/src/pmdk/utils/docker/run-build-package.sh new file mode 100755 index 000000000..5141904aa --- /dev/null +++ b/src/pmdk/utils/docker/run-build-package.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2019, Intel Corporation + +# +# run-build-package.sh - is called inside a Docker container; prepares +# the environment and starts a build of PMDK project. +# + +set -e + +# Prepare build enviromnent +./prepare-for-build.sh + +# Create fake tag, so that package has proper 'version' field +git config user.email "test@package.com" +git config user.name "test package" +git tag -a 1.4.99 -m "1.4" HEAD~1 || true + +# Build all and run tests +cd $WORKDIR +export PCHECK_OPTS="-j2 BLACKLIST_FILE=${BLACKLIST_FILE}" +make -j$(nproc) $PACKAGE_MANAGER + +# Install packages +if [[ "$PACKAGE_MANAGER" == "dpkg" ]]; then + cd $PACKAGE_MANAGER + echo $USERPASS | sudo -S dpkg --install *.deb +else + RPM_ARCH=$(uname -m) + cd $PACKAGE_MANAGER/$RPM_ARCH + echo $USERPASS | sudo -S rpm --install *.rpm +fi + +# Compile and run standalone test +cd $WORKDIR/utils/docker/test_package +make -j$(nproc) LIBPMEMOBJ_MIN_VERSION=1.4 +./test_package testfile1 + +# Use pmreorder installed in the system +pmreorder_version="$(pmreorder -v)" +pmreorder_pattern="pmreorder\.py .+$" +(echo "$pmreorder_version" | grep -Ev "$pmreorder_pattern") && echo "pmreorder version failed" && exit 1 + +touch testfile2 +touch logfile1 +pmreorder -p testfile2 -l logfile1 diff --git a/src/pmdk/utils/docker/run-build.sh b/src/pmdk/utils/docker/run-build.sh new file mode 100755 index 000000000..78edf09ec --- /dev/null +++ b/src/pmdk/utils/docker/run-build.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation + +# +# run-build.sh - is called inside a Docker container; prepares the environment +# and starts a build of PMDK project. +# + +set -e + +# Prepare build environment +./prepare-for-build.sh + +# Build all and run tests +cd $WORKDIR +if [ "$SRC_CHECKERS" != "0" ]; then + make -j$(nproc) check-license + make -j$(nproc) cstyle +fi + +make -j$(nproc) +make -j$(nproc) test +# do not change -j2 to -j$(nproc) in case of tests (make check/pycheck) +make -j2 pcheck TEST_BUILD=$TEST_BUILD +# do not change -j2 to -j$(nproc) in case of tests (make check/pycheck) +make -j2 pycheck +make -j$(nproc) DESTDIR=/tmp source + +# Create PR with generated docs +if [[ "$AUTO_DOC_UPDATE" == "1" ]]; then + echo "Running auto doc update" + ./utils/docker/run-doc-update.sh +fi diff --git a/src/pmdk/utils/docker/run-coverage.sh b/src/pmdk/utils/docker/run-coverage.sh new file mode 100755 index 000000000..b6f7ed0a8 --- /dev/null +++ b/src/pmdk/utils/docker/run-coverage.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2020, Intel Corporation + +# +# run-coverage.sh - is called inside a Docker container; runs tests +# to measure code coverage and sends report to codecov.io +# + +set -e + +# Get and prepare PMDK source +./prepare-for-build.sh + +# Hush error messages, mainly from Valgrind +export UT_DUMP_LINES=0 + +# Skip printing mismatched files for tests with Valgrind +export UT_VALGRIND_SKIP_PRINT_MISMATCHED=1 + +# Build all and run tests +cd $WORKDIR +make -j$(nproc) COVERAGE=1 +make -j$(nproc) test COVERAGE=1 + +# XXX: unfortunately valgrind raports issues in coverage instrumentation +# which we have to ignore (-k flag), also there is dependency between +# local and remote tests (which cannot be easily removed) we have to +# run local and remote tests separately +cd src/test +# do not change -j2 to -j$(nproc) in case of tests (make check/pycheck) +make -kj2 pcheck-local-quiet TEST_BUILD=debug || true +make check-remote-quiet TEST_BUILD=debug || true +# do not change -j2 to -j$(nproc) in case of tests (make check/pycheck) +make -j2 pycheck TEST_BUILD=debug || true +cd ../.. + +# prepare flag for codecov report to differentiate builds +flag=tests +[ -n "$GITHUB_ACTIONS" ] && flag=GHA +[ -n "$TRAVIS" ] && flag=Travis + +# run gcov exe, using codecov's bash (remove parsed coverage files, set flag and exit 1 if not successful) +/opt/scripts/codecov -c -F ${flag} -Z + +printf "check for any leftover gcov files\n" +leftover_files=$(find . -name "*.gcov" | wc -l) +if [[ $leftover_files > 0 ]]; then + # display found files and exit with error (they all should be parsed) + find . -name "*.gcov" + return 1 +fi diff --git a/src/pmdk/utils/docker/run-coverity.sh b/src/pmdk/utils/docker/run-coverity.sh new file mode 100755 index 000000000..140fa753d --- /dev/null +++ b/src/pmdk/utils/docker/run-coverity.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2020, Intel Corporation + +# +# run-coverity.sh - runs the Coverity scan build +# + +set -e + +if [[ "$CI_REPO_SLUG" != "$GITHUB_REPO" \ + && ( "$COVERITY_SCAN_NOTIFICATION_EMAIL" == "" \ + || "$COVERITY_SCAN_TOKEN" == "" ) ]]; then + echo + echo "Skipping Coverity build:"\ + "COVERITY_SCAN_TOKEN=\"$COVERITY_SCAN_TOKEN\" or"\ + "COVERITY_SCAN_NOTIFICATION_EMAIL="\ + "\"$COVERITY_SCAN_NOTIFICATION_EMAIL\" is not set" + exit 0 +fi + +# Prepare build environment +./prepare-for-build.sh + +CERT_FILE=/etc/ssl/certs/ca-certificates.crt +TEMP_CF=$(mktemp) +cp $CERT_FILE $TEMP_CF + +# Download Coverity certificate +echo -n | openssl s_client -connect scan.coverity.com:443 | \ + sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | \ + tee -a $TEMP_CF + +echo $USERPASS | sudo -S mv $TEMP_CF $CERT_FILE + +export COVERITY_SCAN_PROJECT_NAME="$CI_REPO_SLUG" +[[ "$CI_EVENT_TYPE" == "cron" ]] \ + && export COVERITY_SCAN_BRANCH_PATTERN="master" \ + || export COVERITY_SCAN_BRANCH_PATTERN="coverity_scan" +export COVERITY_SCAN_BUILD_COMMAND="make -j$(nproc) all" + +cd $WORKDIR + +# +# Run the Coverity scan +# + +# The 'travisci_build_coverity_scan.sh' script requires the following +# environment variables to be set: +# - TRAVIS_BRANCH - has to contain the name of the current branch +# - TRAVIS_PULL_REQUEST - has to be set to 'true' in case of pull requests +# +export TRAVIS_BRANCH=${CI_BRANCH} +[ "${CI_EVENT_TYPE}" == "pull_request" ] && export TRAVIS_PULL_REQUEST="true" + +# XXX: Patch the Coverity script. +# Recently, this script regularly exits with an error, even though +# the build is successfully submitted. Probably because the status code +# is missing in response, or it's not 201. +# Changes: +# 1) change the expected status code to 200 and +# 2) print the full response string. +# +# This change should be reverted when the Coverity script is fixed. +# +# The previous version was: +# curl -s https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh | bash + +wget https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh +patch < utils/docker/0001-travis-fix-travisci_build_coverity_scan.sh.patch +bash ./travisci_build_coverity_scan.sh diff --git a/src/pmdk/utils/docker/run-doc-update.sh b/src/pmdk/utils/docker/run-doc-update.sh new file mode 100755 index 000000000..132315eb8 --- /dev/null +++ b/src/pmdk/utils/docker/run-doc-update.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2019-2020, Intel Corporation + +set -e + +source `dirname $0`/valid-branches.sh + +BOT_NAME="pmem-bot" +USER_NAME="pmem" +REPO_NAME="pmdk" + +ORIGIN="https://${DOC_UPDATE_GITHUB_TOKEN}@github.com/${BOT_NAME}/${REPO_NAME}" +UPSTREAM="https://github.com/${USER_NAME}/${REPO_NAME}" +# master or stable-* branch +TARGET_BRANCH=${CI_BRANCH} +VERSION=${TARGET_BRANCHES[$TARGET_BRANCH]} + +if [ -z $VERSION ]; then + echo "Target location for branch $TARGET_BRANCH is not defined." + exit 1 +fi + +# Clone bot repo +git clone ${ORIGIN} +cd ${REPO_NAME} +git remote add upstream ${UPSTREAM} + +git config --local user.name ${BOT_NAME} +git config --local user.email "pmem-bot@intel.com" + +git remote update +git checkout -B ${TARGET_BRANCH} upstream/${TARGET_BRANCH} + +# Copy man & PR web md +cd ./doc +make -j$(nproc) web +cd .. + +mv ./doc/web_linux ../ +mv ./doc/web_windows ../ +mv ./doc/generated/libs_map.yml ../ + +# Checkout gh-pages and copy docs +GH_PAGES_NAME="gh-pages-for-${TARGET_BRANCH}" +git checkout -B $GH_PAGES_NAME upstream/gh-pages +git clean -dfx + +rsync -a ../web_linux/ ./manpages/linux/${VERSION}/ +rsync -a ../web_windows/ ./manpages/windows/${VERSION}/ \ + --exclude='librpmem' \ + --exclude='rpmemd' --exclude='pmreorder' \ + --exclude='daxio' + +rm -r ../web_linux +rm -r ../web_windows + +if [ $TARGET_BRANCH = "master" ]; then + [ ! -d _data ] && mkdir _data + cp ../libs_map.yml _data +fi + +# Add and push changes. +# git commit command may fail if there is nothing to commit. +# In that case we want to force push anyway (there might be open pull request +# with changes which were reverted). +git add -A +git commit -m "doc: automatic gh-pages docs update" && true +git push -f ${ORIGIN} $GH_PAGES_NAME + +GITHUB_TOKEN=${DOC_UPDATE_GITHUB_TOKEN} hub pull-request -f \ + -b ${USER_NAME}:gh-pages \ + -h ${BOT_NAME}:${GH_PAGES_NAME} \ + -m "doc: automatic gh-pages docs update" && true + +exit 0 diff --git a/src/pmdk/utils/docker/set-ci-vars.sh b/src/pmdk/utils/docker/set-ci-vars.sh new file mode 100755 index 000000000..af41a753e --- /dev/null +++ b/src/pmdk/utils/docker/set-ci-vars.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020, Intel Corporation + +# +# set-ci-vars.sh -- set CI variables common for both: +# Travis and GitHub Actions CIs +# + +set -e + +function get_commit_range_from_last_merge { + # get commit id of the last merge + LAST_MERGE=$(git log --merges --pretty=%H -1) + LAST_COMMIT=$(git log --pretty=%H -1) + if [ "$LAST_MERGE" == "$LAST_COMMIT" ]; then + # GitHub Actions commits its own merge in case of pull requests + # so the first merge commit has to be skipped. + LAST_MERGE=$(git log --merges --pretty=%H -2 | tail -n1) + fi + if [ "$LAST_MERGE" == "" ]; then + # possible in case of shallow clones + # or new repos with no merge commits yet + # - pick up the first commit + LAST_MERGE=$(git log --pretty=%H | tail -n1) + fi + COMMIT_RANGE="$LAST_MERGE..HEAD" + # make sure it works now + if ! git rev-list $COMMIT_RANGE >/dev/null; then + COMMIT_RANGE="" + fi + echo $COMMIT_RANGE +} + +COMMIT_RANGE_FROM_LAST_MERGE=$(get_commit_range_from_last_merge) + +if [ -n "$TRAVIS" ]; then + CI_COMMIT=$TRAVIS_COMMIT + CI_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE/.../..}" + CI_BRANCH=$TRAVIS_BRANCH + CI_EVENT_TYPE=$TRAVIS_EVENT_TYPE + CI_REPO_SLUG=$TRAVIS_REPO_SLUG + + # CI_COMMIT_RANGE is usually invalid for force pushes - fix it when used + # with non-upstream repository + if [ -n "$CI_COMMIT_RANGE" -a "$CI_REPO_SLUG" != "$GITHUB_REPO" ]; then + if ! git rev-list $CI_COMMIT_RANGE; then + CI_COMMIT_RANGE=$COMMIT_RANGE_FROM_LAST_MERGE + fi + fi + + case "$TRAVIS_CPU_ARCH" in + "amd64") + CI_CPU_ARCH="x86_64" + ;; + *) + CI_CPU_ARCH=$TRAVIS_CPU_ARCH + ;; + esac + +elif [ -n "$GITHUB_ACTIONS" ]; then + CI_COMMIT=$GITHUB_SHA + CI_COMMIT_RANGE=$COMMIT_RANGE_FROM_LAST_MERGE + CI_BRANCH=$(echo $GITHUB_REF | cut -d'/' -f3) + CI_REPO_SLUG=$GITHUB_REPOSITORY + CI_CPU_ARCH="x86_64" # GitHub Actions supports only x86_64 + + case "$GITHUB_EVENT_NAME" in + "schedule") + CI_EVENT_TYPE="cron" + ;; + *) + CI_EVENT_TYPE=$GITHUB_EVENT_NAME + ;; + esac + +else + CI_COMMIT=$(git log --pretty=%H -1) + CI_COMMIT_RANGE=$COMMIT_RANGE_FROM_LAST_MERGE + CI_CPU_ARCH="x86_64" +fi + +export CI_COMMIT=$CI_COMMIT +export CI_COMMIT_RANGE=$CI_COMMIT_RANGE +export CI_BRANCH=$CI_BRANCH +export CI_EVENT_TYPE=$CI_EVENT_TYPE +export CI_REPO_SLUG=$CI_REPO_SLUG +export CI_CPU_ARCH=$CI_CPU_ARCH + +echo CI_COMMIT=$CI_COMMIT +echo CI_COMMIT_RANGE=$CI_COMMIT_RANGE +echo CI_BRANCH=$CI_BRANCH +echo CI_EVENT_TYPE=$CI_EVENT_TYPE +echo CI_REPO_SLUG=$CI_REPO_SLUG +echo CI_CPU_ARCH=$CI_CPU_ARCH diff --git a/src/pmdk/utils/docker/set-vars.sh b/src/pmdk/utils/docker/set-vars.sh new file mode 100755 index 000000000..9a531a840 --- /dev/null +++ b/src/pmdk/utils/docker/set-vars.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2019, Intel Corporation + +# +# set-vars.sh - set required environment variables +# + +set -e + +export CI_FILE_PUSH_IMAGE_TO_REPO=/tmp/push_image_to_repo_flag +export CI_FILE_SKIP_BUILD_PKG_CHECK=/tmp/skip_build_package_check diff --git a/src/pmdk/utils/docker/test_package/.gitignore b/src/pmdk/utils/docker/test_package/.gitignore new file mode 100644 index 000000000..7e564dc4e --- /dev/null +++ b/src/pmdk/utils/docker/test_package/.gitignore @@ -0,0 +1 @@ +test_package diff --git a/src/pmdk/utils/docker/test_package/Makefile b/src/pmdk/utils/docker/test_package/Makefile new file mode 100644 index 000000000..f5970aac3 --- /dev/null +++ b/src/pmdk/utils/docker/test_package/Makefile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018, Intel Corporation + +check_package = $(shell pkg-config $(1) && echo y || echo n) + +HAS_LIBPMEMOBJ := $(call check_package, libpmemobj --atleast-version $(LIBPMEMOBJ_MIN_VERSION) --print-errors) + +LIBS += $(shell pkg-config --libs libpmemobj) +CFLAGS += $(shell pkg-config --cflags libpmemobj) + +ifeq ($(HAS_LIBPMEMOBJ),n) +$(error libpmemobj(version >= $(LIBPMEMOBJ_MIN_VERSION)) is missing) +endif + +test_package: test_package.c + $(CC) test_package.c $(LIBS) $(CFLAGS) -o $@ + +clean: + rm -f test_package + +.PHONY: clean diff --git a/src/pmdk/utils/docker/test_package/README b/src/pmdk/utils/docker/test_package/README new file mode 100644 index 000000000..de81a6a5b --- /dev/null +++ b/src/pmdk/utils/docker/test_package/README @@ -0,0 +1,6 @@ +Persistent Memory Development Kit + +This is utils/docker/test_package/README. + +This directory contains simple application which uses libpmemobj. +It can be used to test whether libpmemobj was installed properly. diff --git a/src/pmdk/utils/docker/test_package/test_package.c b/src/pmdk/utils/docker/test_package/test_package.c new file mode 100644 index 000000000..cf1e7eb8b --- /dev/null +++ b/src/pmdk/utils/docker/test_package/test_package.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright 2018, Intel Corporation */ + +#include <libpmemobj.h> +#include <stdio.h> +#include <sys/stat.h> + +#define LAYOUT_NAME "test" + +struct my_root { + int foo; +}; + +int +main(int argc, char *argv[]) +{ + if (argc < 2) { + printf("usage: %s file-name\n", argv[0]); + return 1; + } + + const char *path = argv[1]; + + PMEMobjpool *pop = pmemobj_create(path, LAYOUT_NAME, + PMEMOBJ_MIN_POOL, S_IWUSR | S_IRUSR); + + if (pop == NULL) { + printf("failed to create pool\n"); + return 1; + } + + PMEMoid root = pmemobj_root(pop, sizeof(struct my_root)); + struct my_root *rootp = pmemobj_direct(root); + + rootp->foo = 10; + pmemobj_persist(pop, &rootp->foo, sizeof(rootp->foo)); + + pmemobj_close(pop); + + return 0; +} diff --git a/src/pmdk/utils/docker/valid-branches.sh b/src/pmdk/utils/docker/valid-branches.sh new file mode 100755 index 000000000..b111c2a3d --- /dev/null +++ b/src/pmdk/utils/docker/valid-branches.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018-2020, Intel Corporation + +declare -A TARGET_BRANCHES=( \ + ["master"]="master" \ + ["stable-1.5"]="v1.5" \ + ["stable-1.6"]="v1.6" \ + ["stable-1.7"]="v1.7" \ + ["stable-1.8"]="v1.8" \ + ["stable-1.9"]="v1.9" \ + ) diff --git a/src/pmdk/utils/get_aliases.sh b/src/pmdk/utils/get_aliases.sh new file mode 100755 index 000000000..87978e2bc --- /dev/null +++ b/src/pmdk/utils/get_aliases.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2020, Intel Corporation +# + +# +# get_aliases.sh -- generate map of manuals functions and libraries +# +# usage: run from /pmdk/doc/generated location without parameters: +# ./../../utils/get_aliases.sh +# +# This script searches manpages from section 7 then +# takes all functions from each section using specified pattern +# and at the end to every function it assign real markdown file +# representation based on *.gz file content +# +# Generated libs_map.yml file is used on gh-pages +# to handle functions and their aliases +# + +list=("$@") +man_child=("$@") + +function search_aliases { +children=$1 +parent=$2 +for i in ${children[@]} +do + if [ -e ../$parent/$i ] + then + echo "Man: $i" + content=$(head -c 150 ../$parent/$i) + if [[ "$content" == ".so "* ]] ; + then + content=$(basename ${content#".so"}) + i="${i%.*}" + echo " $i: $content" >> $map_file + else + r="${i%.*}" + echo " $r: $i" >> $map_file + fi + fi +done +} + +function list_pages { + parent="${1%.*}" + list=("$@") + man_child=("$@") + + if [ "$parent" == "libpmem" ]; then + man_child=($(ls -1 ../libpmem | grep -e ".*\.3$")) + echo -n "- $parent: " >> $map_file + echo "${man_child[@]}" >> $map_file + fi + + if [ "$parent" == "libpmem2" ]; then + man_child=($(ls -1 ../libpmem2 | grep -e ".*\.3$")) + echo -n "- $parent: " >> $map_file + echo "${man_child[@]}" >> $map_file + fi + + if [ "$parent" == "libpmemblk" ]; then + man_child=($(ls -1 ../libpmemblk | grep -e ".*\.3$")) + echo -n "- $parent: " >> $map_file + echo "${man_child[@]}" >> $map_file + fi + + if [ "$parent" == "libpmemlog" ]; then + man_child=($(ls -1 ../libpmemlog | grep -e ".*\.3$")) + echo -n "- $parent: " >> $map_file + echo "${man_child[@]}" >> $map_file + fi + + if [ "$parent" == "libpmemobj" ]; then + man_child=($(ls -1 ../libpmemobj | grep -e ".*\.3$")) + echo -n "- $parent: " >> $map_file + echo "${man_child[@]}" >> $map_file + fi + + if [ "$parent" == "libpmempool" ]; then + man_child=($(ls -1 ../libpmempool | grep -e ".*\.3$")) + echo -n "- $parent: " >> $map_file + echo "${man_child[@]}" >> $map_file + fi + + if [ "$parent" == "librpmem" ]; then + man_child=($(ls -1 ../librpmem | grep -e ".*\.3$")) + echo -n "- $parent: " >> $map_file + echo "${man_child[@]}" >> $map_file + fi + + if [ ${#man_child[@]} -ne 0 ] + then + list=${man_child[@]} + search_aliases "${list[@]}" "$parent" + fi +} + +man7=($(ls -1 ../*/ | grep -e ".*\.7$")) + +map_file=libs_map.yml +[ -e $map_file ] && rm $map_file +touch $map_file + +for i in "${man7[@]}" +do +echo "Library: $i" + list_pages $i +done diff --git a/src/pmdk/utils/git-years b/src/pmdk/utils/git-years new file mode 100755 index 000000000..d54018ddf --- /dev/null +++ b/src/pmdk/utils/git-years @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2019, Intel Corporation + +# git-years -- calculate the range of years for a given file from git + +git log --pretty='%aI %aE' "$@"|grep '@intel\.com'|cut -d- -f1|sort| + sed '$p;2,$d'|uniq|tr '\n' -|sed 's/-$//' diff --git a/src/pmdk/utils/libpmem.pc.in b/src/pmdk/utils/libpmem.pc.in new file mode 100644 index 000000000..850b28198 --- /dev/null +++ b/src/pmdk/utils/libpmem.pc.in @@ -0,0 +1,9 @@ +includedir=${prefix}/include + +Name: libpmem +Description: libpmem library from PMDK project +Version: ${version} +URL: https://pmem.io/pmdk +Requires: +Libs: -L${libdir} -lpmem +Cflags: -I${includedir} diff --git a/src/pmdk/utils/libpmem2.pc.in b/src/pmdk/utils/libpmem2.pc.in new file mode 100644 index 000000000..d4ddc907c --- /dev/null +++ b/src/pmdk/utils/libpmem2.pc.in @@ -0,0 +1,9 @@ +includedir=${prefix}/include + +Name: libpmem2 +Description: libpmem2 library from PMDK project +Version: ${version} +URL: https://pmem.io/pmdk +Requires: +Libs: -L${libdir} -lpmem2 +Cflags: -I${includedir} diff --git a/src/pmdk/utils/libpmemblk.pc.in b/src/pmdk/utils/libpmemblk.pc.in new file mode 100644 index 000000000..5b0ffbf3e --- /dev/null +++ b/src/pmdk/utils/libpmemblk.pc.in @@ -0,0 +1,9 @@ +includedir=${prefix}/include + +Name: libpmemblk +Description: libpmemblk library from PMDK project +Version: ${version} +URL: https://pmem.io/pmdk +Requires.private: libpmem${rasdeps} +Libs: -L${libdir} -lpmemblk +Cflags: -I${includedir} diff --git a/src/pmdk/utils/libpmemlog.pc.in b/src/pmdk/utils/libpmemlog.pc.in new file mode 100644 index 000000000..f1658438b --- /dev/null +++ b/src/pmdk/utils/libpmemlog.pc.in @@ -0,0 +1,9 @@ +includedir=${prefix}/include + +Name: libpmemlog +Description: libpmemlog library from PMDK project +Version: ${version} +URL: https://pmem.io/pmdk +Requires.private: libpmem${rasdeps} +Libs: -L${libdir} -lpmemlog +Cflags: -I${includedir} diff --git a/src/pmdk/utils/libpmemobj.pc.in b/src/pmdk/utils/libpmemobj.pc.in new file mode 100644 index 000000000..12c396e9e --- /dev/null +++ b/src/pmdk/utils/libpmemobj.pc.in @@ -0,0 +1,10 @@ +includedir=${prefix}/include + +Name: libpmemobj +Description: libpmemobj library from PMDK project +Version: ${version} +URL: https://pmem.io/pmdk +Requires.private: libpmem${rasdeps} +Libs: -L${libdir} -lpmemobj +Libs.private: -ldl +Cflags: -I${includedir} diff --git a/src/pmdk/utils/libpmempool.pc.in b/src/pmdk/utils/libpmempool.pc.in new file mode 100644 index 000000000..660938a27 --- /dev/null +++ b/src/pmdk/utils/libpmempool.pc.in @@ -0,0 +1,10 @@ +includedir=${prefix}/include + +Name: libpmempool +Description: libpmempool library from PMDK project +Version: ${version} +URL: https://pmem.io/pmdk +Requires.private: libpmem${rasdeps} +Libs: -L${libdir} -lpmempool +Libs.private: -ldl +Cflags: -I${includedir} diff --git a/src/pmdk/utils/librpmem.pc.in b/src/pmdk/utils/librpmem.pc.in new file mode 100644 index 000000000..50bf1bb91 --- /dev/null +++ b/src/pmdk/utils/librpmem.pc.in @@ -0,0 +1,9 @@ +includedir=${prefix}/include + +Name: librpmem +Description: librpmem library from PMDK project +Version: ${version} +URL: https://pmem.io/pmdk +Requires: +Libs: -L${libdir} -lrpmem +Cflags: -I${includedir} diff --git a/src/pmdk/utils/magic-install.sh b/src/pmdk/utils/magic-install.sh new file mode 100644 index 000000000..3c65bab49 --- /dev/null +++ b/src/pmdk/utils/magic-install.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2014-2017, Intel Corporation +# +# magic-install.sh -- Script for installing magic script +# +set -e + +if ! grep -q "File: pmdk" /etc/magic +then + echo "Appending PMDK magic to /etc/magic" + cat /usr/share/pmdk/pmdk.magic >> /etc/magic +else + echo "PMDK magic already exists" +fi diff --git a/src/pmdk/utils/magic-uninstall.sh b/src/pmdk/utils/magic-uninstall.sh new file mode 100644 index 000000000..6733c7e72 --- /dev/null +++ b/src/pmdk/utils/magic-uninstall.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2014-2017, Intel Corporation +# +# magic-uninstall.sh -- Script for uninstalling magic script +# +set -e + +HDR_LOCAL=$(grep "File: pmdk" /etc/magic) +HDR_PKG=$(grep "File: pmdk" /usr/share/pmdk/pmdk.magic) + +if [[ $HDR_LOCAL == $HDR_PKG ]] +then + echo "Removing PMDK magic from /etc/magic" + HDR_LINE=$(grep -n "File: pmdk" /etc/magic | cut -f1 -d:) + HDR_PKG_LINE=$(grep -n "File: pmdk" /usr/share/pmdk/pmdk.magic | cut -f1 -d:) + HDR_LINES=$(cat /usr/share/pmdk/pmdk.magic | wc -l) + HDR_FIRST=$(($HDR_LINE - $HDR_PKG_LINE + 1)) + HDR_LAST=$(($HDR_FIRST + $HDR_LINES)) + sed -i "${HDR_FIRST},${HDR_LAST}d" /etc/magic +fi diff --git a/src/pmdk/utils/md2man.sh b/src/pmdk/utils/md2man.sh new file mode 100755 index 000000000..941bd3c96 --- /dev/null +++ b/src/pmdk/utils/md2man.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation +# + +# +# md2man.sh -- convert markdown to groff man pages +# +# usage: md2man.sh file template outfile +# +# This script converts markdown file into groff man page using pandoc. +# It performs some pre- and post-processing for better results: +# - uses m4 to preprocess OS-specific directives. See doc/macros.man. +# - parse input file for YAML metadata block and read man page title, +# section and version +# - cut-off metadata block and license +# - unindent code blocks +# - cut-off windows and web specific parts of documentation +# +# If the TESTOPTS variable is set, generates a preprocessed markdown file +# with the header stripped off for testing purposes. +# + +set -e +set -o pipefail + +filename=$1 +template=$2 +outfile=$3 +title=`sed -n 's/^title:\ _MP(*\([A-Za-z0-9_-]*\).*$/\1/p' $filename` +section=`sed -n 's/^title:.*\([0-9]\))$/\1/p' $filename` +version=`sed -n 's/^date:\ *\(.*\)$/\1/p' $filename` + +if [ "$TESTOPTS" != "" ]; then + m4 $TESTOPTS macros.man $filename | sed -n -e '/# NAME #/,$p' > $outfile +else + OPTS= + +if [ "$WIN32" == 1 ]; then + OPTS="$OPTS -DWIN32" +else + OPTS="$OPTS -UWIN32" +fi + +if [ "$(uname -s)" == "FreeBSD" ]; then + OPTS="$OPTS -DFREEBSD" +else + OPTS="$OPTS -UFREEBSD" +fi + +if [ "$WEB" == 1 ]; then + OPTS="$OPTS -DWEB" + mkdir -p "$(dirname $outfile)" + m4 $OPTS macros.man $filename | sed -n -e '/---/,$p' > $outfile +else + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(date +%s)}" + COPYRIGHT=$(grep -rwI "\[comment]: <> (Copyright" $filename |\ + sed "s/\[comment\]: <> (\([^)]*\))/\1/") + dt=$(date -u -d "@$SOURCE_DATE_EPOCH" +%F 2>/dev/null || + date -u -r "$SOURCE_DATE_EPOCH" +%F 2>/dev/null || date -u +%F) + m4 $OPTS macros.man $filename | sed -n -e '/# NAME #/,$p' |\ + pandoc -s -t man -o $outfile --template=$template \ + -V title=$title -V section=$section \ + -V date="$dt" -V version="$version" \ + -V copyright="$COPYRIGHT" +fi +fi diff --git a/src/pmdk/utils/os-banned b/src/pmdk/utils/os-banned new file mode 100644 index 000000000..57d162551 --- /dev/null +++ b/src/pmdk/utils/os-banned @@ -0,0 +1,63 @@ +pthread_once +pthread_key_create +pthread_key_delete +pthread_setspecific +pthread_getspecific +pthread_mutex_init +pthread_mutex_destroy +pthread_mutex_lock +pthread_mutex_trylock +pthread_mutex_unlock +pthread_mutex_timedlock +pthread_rwlock_init +pthread_rwlock_destroy +pthread_rwlock_rdlock +pthread_rwlock_wrlock +pthread_rwlock_tryrdlock +pthread_rwlock_trywrlock +pthread_rwlock_unlock +pthread_rwlock_timedrdlock +pthread_rwlock_timedwrlock +pthread_spin_init +pthread_spin_destroy +pthread_spin_lock +pthread_spin_unlock +pthread_spin_trylock +pthread_cond_init +pthread_cond_destroy +pthread_cond_broadcast +pthread_cond_signal +pthread_cond_timedwait +pthread_cond_wait +pthread_create +pthread_join +cpu_zero +cpu_set +pthread_setaffinity_np +pthread_atfork +sem_init +sem_destroy +sem_wait +sem_trywait +sem_post +fsync +fsync_dir +open +stat +unlink +access +fopen +fdopen +chmod +mkstemp +posix_fallocate +ftruncate +flock +writev +clock_gettime +rand_r +unsetenv +setenv +getenv +strsignal +execv diff --git a/src/pmdk/utils/pkg-common.sh b/src/pmdk/utils/pkg-common.sh new file mode 100644 index 000000000..f3711bc59 --- /dev/null +++ b/src/pmdk/utils/pkg-common.sh @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2014-2019, Intel Corporation + +# +# pkg-common.sh - common functions and variables for building packages +# + +export LC_ALL="C" + +function error() { + echo -e "error: $@" +} + +function check_dir() { + if [ ! -d $1 ] + then + error "Directory '$1' does not exist." + exit 1 + fi +} + +function check_file() { + if [ ! -f $1 ] + then + error "File '$1' does not exist." + exit 1 + fi +} + +function check_tool() { + local tool=$1 + if [ -z "$(which $tool 2>/dev/null)" ] + then + error "'${tool}' not installed or not in PATH" + exit 1 + fi +} + +function get_version() { + echo -n $1 | sed "s/-rc/~rc/" +} + +function get_os() { + if [ -f /etc/os-release ] + then + local OS=$(cat /etc/os-release | grep -m1 -o -P '(?<=NAME=).*($)') + [[ "$OS" =~ SLES|openSUSE ]] && echo -n "SLES_like" || + ([[ "$OS" =~ "Fedora"|"Red Hat"|"CentOS" ]] && echo -n "RHEL_like" || echo 1) + else + echo 1 + fi +} + +REGEX_DATE_AUTHOR="([a-zA-Z]{3} [a-zA-Z]{3} [0-9]{2} [0-9]{4})\s*(.*)" +REGEX_MESSAGE_START="\s*\*\s*(.*)" +REGEX_MESSAGE="\s*(\S.*)" diff --git a/src/pmdk/utils/pkg-config.sh b/src/pmdk/utils/pkg-config.sh new file mode 100644 index 000000000..7186fa2ee --- /dev/null +++ b/src/pmdk/utils/pkg-config.sh @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2014-2020, Intel Corporation + +# Name of package +PACKAGE_NAME="pmdk" + +# Name and email of package maintainer +PACKAGE_MAINTAINER="Piotr Balcer <piotr.balcer@intel.com>" + +# Brief description of the package +PACKAGE_SUMMARY="Persistent Memory Development Kit" + +# Full description of the package +PACKAGE_DESCRIPTION="The collection of libraries and utilities for Persistent Memory Programming" + +# Website +PACKAGE_URL="https://pmem.io/pmdk" diff --git a/src/pmdk/utils/pmdk.magic b/src/pmdk/utils/pmdk.magic new file mode 100644 index 000000000..7409ba1ef --- /dev/null +++ b/src/pmdk/utils/pmdk.magic @@ -0,0 +1,15 @@ + +#------------------------------------------------------------------------------ +# $File: pmdk,v 1.2 2017/12/11 20:00:00 +# pmdk: file(1) magic for Persistent Memory Development Kit pool files +# +# The PMDK specific format of pool files. +# +# PMEM signature +0 string PMEM +>4 string POOLSET Persistent Memory Poolset file +>>11 search REPLICA with replica +# Pool type signature +>4 regex LOG|BLK|OBJ Persistent Memory Pool file, type: %s, +# Major version number +>>8 lelong >0 version 0x%x diff --git a/src/pmdk/utils/pmdk.spec.in b/src/pmdk/utils/pmdk.spec.in new file mode 100644 index 000000000..094e9d0b4 --- /dev/null +++ b/src/pmdk/utils/pmdk.spec.in @@ -0,0 +1,710 @@ + +# rpmbuild options: +# --with | --without fabric +# --with | --without ndctl +# --define _testconfig <path to custom testconfig.sh> +# --define _skip_check 1 + +# do not terminate build if files in the $RPM_BUILD_ROOT +# directory are not found in the %files (without rpmem case) +%define _unpackaged_files_terminate_build 0 + +# disable 'make check' on suse +%if %{defined suse_version} + %define _skip_check 1 + %define dist .suse%{suse_version} +%endif + +# libfabric v1.4.2 is available on: +# openSUSE Tumbleweed, Leap 15.0, Leap 42.3; SLE 12 SP3, 15 +# Fedora >=27; RHEL >=7.5 +%if (0%{?suse_version} > 1315) || (0%{?fedora} >= 27) || (0%{?rhel} >= 7) +%bcond_without fabric +%else +%bcond_with fabric +%endif + +%bcond_without ndctl + +%define min_libfabric_ver __LIBFABRIC_MIN_VER__ +%define min_ndctl_ver __NDCTL_MIN_VER__ + +Name: pmdk +Version: __VERSION__ +Release: 1%{?dist} +Summary: __PACKAGE_SUMMARY__ +Packager: __PACKAGE_MAINTAINER__ +Group: __GROUP_SYS_LIBS__ +License: __LICENSE__ +URL: https://pmem.io/pmdk + +Source0: %{name}-%{version}.tar.gz + +BuildRequires: gcc +BuildRequires: make +BuildRequires: glibc-devel +BuildRequires: autoconf +BuildRequires: automake +BuildRequires: man +BuildRequires: pkgconfig +BuildRequires: gdb + +# fdupes package is available only on 'openSUSE Tumbleweed' and 'openSUSE Leap 15.1' +%if (0%{?suse_version} > 1500) || (0%{?sles_version} >= 150100 && 0%{?is_opensuse}) +BuildRequires: fdupes +%endif + +%if %{with ndctl} +%if %{defined suse_version} +BuildRequires: libndctl-devel >= %{min_ndctl_ver} +%else +BuildRequires: ndctl-devel >= %{min_ndctl_ver} +BuildRequires: daxctl-devel >= %{min_ndctl_ver} +%endif +%endif + +%if %{with fabric} +BuildRequires: libfabric-devel >= %{min_libfabric_ver} +%endif + + +# Debug variants of the libraries should be filtered out of the provides. +%global __provides_exclude_from ^%{_libdir}/pmdk_debug/.*\\.so.*$ + +# By design, PMDK does not support any 32-bit architecture. +# Due to dependency on xmmintrin.h and some inline assembly, it can be +# compiled only for x86_64 at the moment. +# Other 64-bit architectures could also be supported, if only there is +# a request for that, and if somebody provides the arch-specific +# implementation of the low-level routines for flushing to persistent +# memory. + +# https://bugzilla.redhat.com/show_bug.cgi?id=1340634 +# https://bugzilla.redhat.com/show_bug.cgi?id=1340635 +# https://bugzilla.redhat.com/show_bug.cgi?id=1340636 +# https://bugzilla.redhat.com/show_bug.cgi?id=1340637 + +ExclusiveArch: x86_64 ppc64le + +%description +The Persistent Memory Development Kit is a collection of libraries for +using memory-mapped persistence, optimized specifically for persistent memory. + + +%package -n libpmem2__PKG_NAME_SUFFIX__ +Summary: Low-level persistent memory support library +Group: __GROUP_SYS_LIBS__ +%description -n libpmem2__PKG_NAME_SUFFIX__ +The libpmem2 provides low level persistent memory support. In particular, +support for the persistent memory instructions for flushing changes +to pmem is provided. + +%files -n libpmem2__PKG_NAME_SUFFIX__ +%defattr(-,root,root,-) +%dir %{_datadir}/pmdk +%{_libdir}/libpmem2.so.* +%{_datadir}/pmdk/pmdk.magic +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmem2-devel +Summary: Development files for the low-level persistent memory library +Group: __GROUP_DEV_LIBS__ +Requires: libpmem2__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n libpmem2-devel +The libpmem2 provides low level persistent memory support. In particular, +support for the persistent memory instructions for flushing changes +to pmem is provided. + +This library is provided for software which tracks every store to +pmem and needs to flush those changes to durability. Most developers +will find higher level libraries like libpmemobj to be much more +convenient. + +%files -n libpmem2-devel +%defattr(-,root,root,-) +%{_libdir}/libpmem2.so +%{_libdir}/pkgconfig/libpmem2.pc +%{_includedir}/libpmem2.h +%{_mandir}/man7/libpmem2.7.gz +%{_mandir}/man3/pmem2_*.3.gz +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmem2-debug +Summary: Debug variant of the low-level persistent memory library +Group: __GROUP_DEV_LIBS__ +Requires: libpmem2__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n libpmem2-debug +The libpmem provides low level persistent memory support. In particular, +support for the persistent memory instructions for flushing changes +to pmem is provided. + +This sub-package contains debug variant of the library, providing +run-time assertions and trace points. The typical way to access the +debug version is to set the environment variable LD_LIBRARY_PATH to +/usr/lib64/pmdk_debug. + +%files -n libpmem2-debug +%defattr(-,root,root,-) +%dir %{_libdir}/pmdk_debug +%{_libdir}/pmdk_debug/libpmem2.so +%{_libdir}/pmdk_debug/libpmem2.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + + +%package -n libpmem__PKG_NAME_SUFFIX__ +Summary: Low-level persistent memory support library +Group: __GROUP_SYS_LIBS__ +%description -n libpmem__PKG_NAME_SUFFIX__ +The libpmem provides low level persistent memory support. In particular, +support for the persistent memory instructions for flushing changes +to pmem is provided. + +%files -n libpmem__PKG_NAME_SUFFIX__ +%defattr(-,root,root,-) +%dir %{_datadir}/pmdk +%{_libdir}/libpmem.so.* +%{_datadir}/pmdk/pmdk.magic +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmem-devel +Summary: Development files for the low-level persistent memory library +Group: __GROUP_DEV_LIBS__ +Requires: libpmem__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n libpmem-devel +The libpmem provides low level persistent memory support. In particular, +support for the persistent memory instructions for flushing changes +to pmem is provided. + +This library is provided for software which tracks every store to +pmem and needs to flush those changes to durability. Most developers +will find higher level libraries like libpmemobj to be much more +convenient. + +%files -n libpmem-devel +%defattr(-,root,root,-) +%{_libdir}/libpmem.so +%{_libdir}/pkgconfig/libpmem.pc +%{_includedir}/libpmem.h +%{_mandir}/man7/libpmem.7.gz +%{_mandir}/man3/pmem_*.3.gz +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmem-debug +Summary: Debug variant of the low-level persistent memory library +Group: __GROUP_DEV_LIBS__ +Requires: libpmem__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n libpmem-debug +The libpmem provides low level persistent memory support. In particular, +support for the persistent memory instructions for flushing changes +to pmem is provided. + +This sub-package contains debug variant of the library, providing +run-time assertions and trace points. The typical way to access the +debug version is to set the environment variable LD_LIBRARY_PATH to +/usr/lib64/pmdk_debug. + +%files -n libpmem-debug +%defattr(-,root,root,-) +%dir %{_libdir}/pmdk_debug +%{_libdir}/pmdk_debug/libpmem.so +%{_libdir}/pmdk_debug/libpmem.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmemblk__PKG_NAME_SUFFIX__ +Summary: Persistent Memory Resident Array of Blocks library +Group: __GROUP_SYS_LIBS__ +Requires: libpmem__PKG_NAME_SUFFIX__ >= %{version}-%{release} +%description -n libpmemblk__PKG_NAME_SUFFIX__ +The libpmemblk implements a pmem-resident array of blocks, all the same +size, where a block is updated atomically with respect to power +failure or program interruption (no torn blocks). + +%files -n libpmemblk__PKG_NAME_SUFFIX__ +%defattr(-,root,root,-) +%{_libdir}/libpmemblk.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmemblk-devel +Summary: Development files for the Persistent Memory Resident Array of Blocks library +Group: __GROUP_DEV_LIBS__ +Requires: libpmemblk__PKG_NAME_SUFFIX__ = %{version}-%{release} +Requires: libpmem-devel = %{version}-%{release} +%description -n libpmemblk-devel +The libpmemblk implements a pmem-resident array of blocks, all the same +size, where a block is updated atomically with respect to power +failure or program interruption (no torn blocks). + +For example, a program keeping a cache of fixed-size objects in pmem +might find this library useful. This library is provided for cases +requiring large arrays of objects at least 512 bytes each. Most +developers will find higher level libraries like libpmemobj to be +more generally useful. + +%files -n libpmemblk-devel +%defattr(-,root,root,-) +%{_libdir}/libpmemblk.so +%{_libdir}/pkgconfig/libpmemblk.pc +%{_includedir}/libpmemblk.h +%{_mandir}/man7/libpmemblk.7.gz +%{_mandir}/man5/poolset.5.gz +%{_mandir}/man3/pmemblk_*.3.gz +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmemblk-debug +Summary: Debug variant of the Persistent Memory Resident Array of Blocks library +Group: __GROUP_DEV_LIBS__ +Requires: libpmemblk__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n libpmemblk-debug +The libpmemblk implements a pmem-resident array of blocks, all the same +size, where a block is updated atomically with respect to power +failure or program interruption (no torn blocks). + +This sub-package contains debug variant of the library, providing +run-time assertions and trace points. The typical way to access the +debug version is to set the environment variable LD_LIBRARY_PATH to +/usr/lib64/pmdk_debug. + +%files -n libpmemblk-debug +%defattr(-,root,root,-) +%dir %{_libdir}/pmdk_debug +%{_libdir}/pmdk_debug/libpmemblk.so +%{_libdir}/pmdk_debug/libpmemblk.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmemlog__PKG_NAME_SUFFIX__ +Summary: Persistent Memory Resident Log File library +Group: __GROUP_SYS_LIBS__ +Requires: libpmem__PKG_NAME_SUFFIX__ >= %{version}-%{release} +%description -n libpmemlog__PKG_NAME_SUFFIX__ +The libpmemlog library provides a pmem-resident log file. This is +useful for programs like databases that append frequently to a log +file. + +%files -n libpmemlog__PKG_NAME_SUFFIX__ +%defattr(-,root,root,-) +%{_libdir}/libpmemlog.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmemlog-devel +Summary: Development files for the Persistent Memory Resident Log File library +Group: __GROUP_DEV_LIBS__ +Requires: libpmemlog__PKG_NAME_SUFFIX__ = %{version}-%{release} +Requires: libpmem-devel = %{version}-%{release} +%description -n libpmemlog-devel +The libpmemlog library provides a pmem-resident log file. This +library is provided for cases requiring an append-mostly file to +record variable length entries. Most developers will find higher +level libraries like libpmemobj to be more generally useful. + +%files -n libpmemlog-devel +%defattr(-,root,root,-) +%{_libdir}/libpmemlog.so +%{_libdir}/pkgconfig/libpmemlog.pc +%{_includedir}/libpmemlog.h +%{_mandir}/man7/libpmemlog.7.gz +%{_mandir}/man5/poolset.5.gz +%{_mandir}/man3/pmemlog_*.3.gz +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmemlog-debug +Summary: Debug variant of the Persistent Memory Resident Log File library +Group: __GROUP_DEV_LIBS__ +Requires: libpmemlog__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n libpmemlog-debug +The libpmemlog library provides a pmem-resident log file. This +library is provided for cases requiring an append-mostly file to +record variable length entries. Most developers will find higher +level libraries like libpmemobj to be more generally useful. + +This sub-package contains debug variant of the library, providing +run-time assertions and trace points. The typical way to access the +debug version is to set the environment variable LD_LIBRARY_PATH to +/usr/lib64/pmdk_debug. + +%files -n libpmemlog-debug +%defattr(-,root,root,-) +%dir %{_libdir}/pmdk_debug +%{_libdir}/pmdk_debug/libpmemlog.so +%{_libdir}/pmdk_debug/libpmemlog.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmemobj__PKG_NAME_SUFFIX__ +Summary: Persistent Memory Transactional Object Store library +Group: __GROUP_SYS_LIBS__ +Requires: libpmem__PKG_NAME_SUFFIX__ >= %{version}-%{release} +%description -n libpmemobj__PKG_NAME_SUFFIX__ +The libpmemobj library provides a transactional object store, +providing memory allocation, transactions, and general facilities for +persistent memory programming. + +%files -n libpmemobj__PKG_NAME_SUFFIX__ +%defattr(-,root,root,-) +%{_libdir}/libpmemobj.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmemobj-devel +Summary: Development files for the Persistent Memory Transactional Object Store library +Group: __GROUP_DEV_LIBS__ +Requires: libpmemobj__PKG_NAME_SUFFIX__ = %{version}-%{release} +Requires: libpmem-devel = %{version}-%{release} +%description -n libpmemobj-devel +The libpmemobj library provides a transactional object store, +providing memory allocation, transactions, and general facilities for +persistent memory programming. Developers new to persistent memory +probably want to start with this library. + +%files -n libpmemobj-devel +%defattr(-,root,root,-) +%{_libdir}/libpmemobj.so +%{_libdir}/pkgconfig/libpmemobj.pc +%{_includedir}/libpmemobj.h +%{_includedir}/libpmemobj/*.h +%{_mandir}/man7/libpmemobj.7.gz +%{_mandir}/man5/poolset.5.gz +%{_mandir}/man3/pmemobj_*.3.gz +%{_mandir}/man3/pobj_*.3.gz +%{_mandir}/man3/oid_*.3.gz +%{_mandir}/man3/toid*.3.gz +%{_mandir}/man3/direct_*.3.gz +%{_mandir}/man3/d_r*.3.gz +%{_mandir}/man3/tx_*.3.gz +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmemobj-debug +Summary: Debug variant of the Persistent Memory Transactional Object Store library +Group: __GROUP_DEV_LIBS__ +Requires: libpmemobj__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n libpmemobj-debug +The libpmemobj library provides a transactional object store, +providing memory allocation, transactions, and general facilities for +persistent memory programming. Developers new to persistent memory +probably want to start with this library. + +This sub-package contains debug variant of the library, providing +run-time assertions and trace points. The typical way to access the +debug version is to set the environment variable LD_LIBRARY_PATH to +/usr/lib64/pmdk_debug. + +%files -n libpmemobj-debug +%defattr(-,root,root,-) +%dir %{_libdir}/pmdk_debug +%{_libdir}/pmdk_debug/libpmemobj.so +%{_libdir}/pmdk_debug/libpmemobj.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmempool__PKG_NAME_SUFFIX__ +Summary: Persistent Memory pool management library +Group: __GROUP_SYS_LIBS__ +Requires: libpmem__PKG_NAME_SUFFIX__ >= %{version}-%{release} +%description -n libpmempool__PKG_NAME_SUFFIX__ +The libpmempool library provides a set of utilities for off-line +administration, analysis, diagnostics and repair of persistent memory +pools created by libpmemlog, libpemblk and libpmemobj libraries. + +%files -n libpmempool__PKG_NAME_SUFFIX__ +%defattr(-,root,root,-) +%{_libdir}/libpmempool.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmempool-devel +Summary: Development files for Persistent Memory pool management library +Group: __GROUP_DEV_LIBS__ +Requires: libpmempool__PKG_NAME_SUFFIX__ = %{version}-%{release} +Requires: libpmem-devel = %{version}-%{release} +%description -n libpmempool-devel +The libpmempool library provides a set of utilities for off-line +administration, analysis, diagnostics and repair of persistent memory +pools created by libpmemlog, libpemblk and libpmemobj libraries. + +%files -n libpmempool-devel +%defattr(-,root,root,-) +%{_libdir}/libpmempool.so +%{_libdir}/pkgconfig/libpmempool.pc +%{_includedir}/libpmempool.h +%{_mandir}/man7/libpmempool.7.gz +%{_mandir}/man5/poolset.5.gz +%{_mandir}/man3/pmempool_*.3.gz +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n libpmempool-debug +Summary: Debug variant of the Persistent Memory pool management library +Group: __GROUP_DEV_LIBS__ +Requires: libpmempool__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n libpmempool-debug +The libpmempool library provides a set of utilities for off-line +administration, analysis, diagnostics and repair of persistent memory +pools created by libpmemlog, libpemblk and libpmemobj libraries. + +This sub-package contains debug variant of the library, providing +run-time assertions and trace points. The typical way to access the +debug version is to set the environment variable LD_LIBRARY_PATH to +/usr/lib64/pmdk_debug. + +%files -n libpmempool-debug +%defattr(-,root,root,-) +%dir %{_libdir}/pmdk_debug +%{_libdir}/pmdk_debug/libpmempool.so +%{_libdir}/pmdk_debug/libpmempool.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%if %{with fabric} + +%package -n librpmem__PKG_NAME_SUFFIX__ +Summary: Remote Access to Persistent Memory library +Group: __GROUP_SYS_LIBS__ +Requires: libfabric >= %{min_libfabric_ver} +%if %{defined suse_version} +Requires: openssh +%else +Requires: openssh-clients +%endif +%description -n librpmem__PKG_NAME_SUFFIX__ +The librpmem library provides low-level support for remote access +to persistent memory utilizing RDMA-capable NICs. It can be used +to replicate persistent memory regions over RDMA protocol. + +%files -n librpmem__PKG_NAME_SUFFIX__ +%defattr(-,root,root,-) +%{_libdir}/librpmem.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n librpmem-devel +Summary: Development files for the Remote Access to Persistent Memory library +Group: __GROUP_DEV_LIBS__ +Requires: librpmem__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n librpmem-devel +The librpmem library provides low-level support for remote access +to persistent memory utilizing RDMA-capable NICs. It can be used +to replicate persistent memory regions over RDMA protocol. + +This sub-package contains libraries and header files for developing +applications that want to specifically make use of librpmem. + +%files -n librpmem-devel +%defattr(-,root,root,-) +%{_libdir}/librpmem.so +%{_libdir}/pkgconfig/librpmem.pc +%{_includedir}/librpmem.h +%{_mandir}/man7/librpmem.7.gz +%{_mandir}/man3/rpmem_*.3.gz +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n librpmem-debug +Summary: Debug variant of the Remote Access to Persistent Memory library +Group: __GROUP_DEV_LIBS__ +Requires: librpmem__PKG_NAME_SUFFIX__ = %{version}-%{release} +%description -n librpmem-debug +The librpmem library provides low-level support for remote access +to persistent memory utilizing RDMA-capable NICs. It can be used +to replicate persistent memory regions over RDMA protocol. + +This sub-package contains debug variant of the library, providing +run-time assertions and trace points. The typical way to access the +debug version is to set the environment variable LD_LIBRARY_PATH to +/usr/lib64/pmdk_debug. + +%files -n librpmem-debug +%defattr(-,root,root,-) +%dir %{_libdir}/pmdk_debug +%{_libdir}/pmdk_debug/librpmem.so +%{_libdir}/pmdk_debug/librpmem.so.* +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%package -n rpmemd +Group: __GROUP_SYS_BASE__ +Summary: Target node process executed by librpmem +Requires: libfabric >= %{min_libfabric_ver} +%description -n rpmemd +The rpmemd process is executed on a target node by librpmem library +and facilitates access to persistent memory over RDMA. + +%files -n rpmemd +%{_bindir}/rpmemd +%{_mandir}/man1/rpmemd.1.gz + +# end of "if _with_fabric" +%endif + +%package -n pmempool +Summary: Utilities for Persistent Memory +Group: __GROUP_SYS_BASE__ +Requires: libpmem__PKG_NAME_SUFFIX__ >= %{version}-%{release} +Requires: libpmemlog__PKG_NAME_SUFFIX__ >= %{version}-%{release} +Requires: libpmemblk__PKG_NAME_SUFFIX__ >= %{version}-%{release} +Requires: libpmemobj__PKG_NAME_SUFFIX__ >= %{version}-%{release} +Requires: libpmempool__PKG_NAME_SUFFIX__ >= %{version}-%{release} +Obsoletes: nvml-tools < %{version}-%{release} +%description -n pmempool +The pmempool is a standalone utility for management and off-line analysis +of Persistent Memory pools created by PMDK libraries. It provides a set +of utilities for administration and diagnostics of Persistent Memory pools. +The pmempool may be useful for troubleshooting by system administrators +and users of the applications based on PMDK libraries. + +%files -n pmempool +%{_bindir}/pmempool +%{_mandir}/man1/pmempool.1.gz +%{_mandir}/man1/pmempool-*.1.gz +%config(noreplace) %{_sysconfdir}/bash_completion.d/pmempool +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + +%package -n pmreorder +Summary: Consistency Checker for Persistent Memory +Group: __GROUP_SYS_BASE__ +%description -n pmreorder +The pmreorder tool is a collection of python scripts designed to parse +and replay operations logged by pmemcheck - a persistent memory checking tool. +Pmreorder performs the store reordering between persistent memory barriers - +a sequence of flush-fence operations. It uses a consistency checking routine +provided in the command line options to check whether files are in a consistent state. + +%files -n pmreorder +%{_bindir}/pmreorder +%{_datadir}/pmreorder/*.py +%{_mandir}/man1/pmreorder.1.gz +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + + +%if %{with ndctl} + +%package -n daxio +Summary: Perform I/O on Device DAX devices or zero a Device DAX device +Group: __GROUP_SYS_BASE__ +Requires: libpmem__PKG_NAME_SUFFIX__ >= %{version}-%{release} +%description -n daxio +The daxio utility performs I/O on Device DAX devices or zero +a Device DAX device. Since the standard I/O APIs (read/write) cannot be used +with Device DAX, data transfer is performed on a memory-mapped device. +The daxio may be used to dump Device DAX data to a file, restore data from +a backup copy, move/copy data to another device or to erase data from +a device. + +%files -n daxio +%{_bindir}/daxio +%{_mandir}/man1/daxio.1.gz +%license LICENSE +%doc ChangeLog CONTRIBUTING.md README.md + +# end of "if _with_ndctl" +%endif + +%prep +%setup -q -n %{name}-%{version} + + +%build +# For debug build default flags may be overridden to disable compiler +# optimizations. +CFLAGS="%{optflags}" \ +LDFLAGS="%{?__global_ldflags}" \ +make %{?_smp_mflags} \ +%if %{without ndctl} + NDCTL_ENABLE=n \ +%endif + __MAKE_FLAGS__ + + +# Override LIB_AR with empty string to skip installation of static libraries +%install +make install DESTDIR=%{buildroot} \ +%if %{without ndctl} + NDCTL_ENABLE=n \ +%endif + LIB_AR= \ + prefix=%{_prefix} \ + libdir=%{_libdir} \ + includedir=%{_includedir} \ + mandir=%{_mandir} \ + bindir=%{_bindir} \ + sysconfdir=%{_sysconfdir} \ + docdir=%{_docdir} +mkdir -p %{buildroot}%{_datadir}/pmdk +cp utils/pmdk.magic %{buildroot}%{_datadir}/pmdk/ +__MAKE_INSTALL_FDUPES__ + + +%check +%if 0%{?_skip_check} == 1 + echo "Check skipped" +%else + %if %{defined _testconfig} + cp %{_testconfig} src/test/testconfig.sh + %else + echo "PMEM_FS_DIR=/tmp" > src/test/testconfig.sh + echo "PMEM_FS_DIR_FORCE_PMEM=1" >> src/test/testconfig.sh + echo 'TEST_BUILD="debug nondebug"' >> src/test/testconfig.sh + echo 'TEST_FS="pmem any none"' >> src/test/testconfig.sh + %endif + make \ +%if %{without ndctl} + NDCTL_ENABLE=n \ +%endif + check +%endif + +%post -n libpmem__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%postun -n libpmem__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%post -n libpmemblk__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%postun -n libpmemblk__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%post -n libpmemlog__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%postun -n libpmemlog__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%post -n libpmemobj__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%postun -n libpmemobj__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%post -n libpmempool__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%postun -n libpmempool__PKG_NAME_SUFFIX__ -p /sbin/ldconfig + +%if %{with fabric} +%post -n librpmem__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%postun -n librpmem__PKG_NAME_SUFFIX__ -p /sbin/ldconfig +%endif + +%if 0%{?__debug_package} == 0 +%debug_package +%endif + + +%changelog diff --git a/src/pmdk/utils/ps_analyze.ps1 b/src/pmdk/utils/ps_analyze.ps1 new file mode 100644 index 000000000..b0e6bfe9e --- /dev/null +++ b/src/pmdk/utils/ps_analyze.ps1 @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017, Intel Corporation +# +# ps_analyze -- script to analyze ps1 files +# + +Write-Output "Starting PSScript analyzing ..." + +$scriptdir = Split-Path -Parent $PSCommandPath +$rootdir = $scriptdir + "\.." +$detected = 0 + +$include = @("*.ps1" ) +Get-ChildItem -Path $rootdir -Recurse -Include $include | ` + Where-Object { $_.FullName -notlike "*test*" } | ` + ForEach-Object { + $analyze_result = Invoke-ScriptAnalyzer -Path $_.FullName + if ($analyze_result) { + $detected = $detected + $analyze_result.Count + Write-Output $_.FullName + Write-Output $analyze_result + } + } + +if ($detected) { + Write-Output "PSScriptAnalyzer FAILED. Issues detected: $detected" + Exit 1 +} else { + Write-Output "PSScriptAnalyzer PASSED. No issue detected." + Exit 0 +} diff --git a/src/pmdk/utils/sort_solution b/src/pmdk/utils/sort_solution new file mode 100755 index 000000000..c14f57ece --- /dev/null +++ b/src/pmdk/utils/sort_solution @@ -0,0 +1,128 @@ +#!/usr/bin/perl +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016, Intel Corporation + +# +# sort_solution -- sort visual studio solution projects lists +# + +use strict; +use warnings; +# install libtext-diff-perl or perl-Text-Diff +use Text::Diff; + +use Cwd 'abs_path'; +use File::Basename; +use File::Compare; + + +sub help { + print "Usage: sort_solution [check|sort]\n"; + exit; +} + +sub sort_global_section { + my ($solution_fh, $temp_fh, $section_name) = @_; + my $line = ""; + my @array; + + while (defined($line = <$solution_fh>) && ($line !~ $section_name)) { + print $temp_fh $line; + } + print $temp_fh $line; + + while (defined($line = <$solution_fh>) && ($line !~ "EndGlobalSection")) { + push @array, $line; + } + + @array = sort @array; + + foreach (@array) { + print $temp_fh $_; + } + + print $temp_fh $line; # print EndGlobalSection line +} + +my $num_args = $#ARGV + 1; +if ($num_args != 1) { + help; +} +my $arg = $ARGV[0]; + +if($arg ne "check" && $arg ne "sort") { + help; +} +my $filename = dirname(abs_path($0)).'/../src/PMDK.sln'; +my $tempfile = dirname(abs_path($0)).'/../src/temp.sln'; + +open(my $temp_fh, '>', $tempfile) + or die "Could not open file '$tempfile' $!"; +open(my $solution_fh, '<:crlf', $filename) + or die "Could not open file '$filename' $!"; + +my $line; + +# Read a header of file +while (defined($line = <$solution_fh>) && ($line !~ "^Project")) { + print $temp_fh $line; +} + +my @part1; +my $buff; +my $guid; + +# Read the projects list with project dependencies +do { + if($line =~ "^Project") { + $buff = $line; + $guid = (split(/\,/, $line))[2]; + } elsif($line =~ "^EndProject") { + $buff .= $line; + my %table = ( + guid => $guid, + buff => $buff, + ); + push @part1, \%table; + } else { + $buff .= $line; + } + +} while (defined($line = <$solution_fh>) && $line ne "Global\n"); + +# sort the project list by a project GIUD and write to the tempfile +@part1 = sort { $a->{guid} cmp $b->{guid} } @part1; + +foreach (@part1) { + my %hash = %$_; + print $temp_fh $hash{"buff"}; +} +print $temp_fh $line; # EndProject line + +sort_global_section $solution_fh, $temp_fh, "ProjectConfigurationPlatforms"; + +sort_global_section $solution_fh, $temp_fh, "NestedProjects"; + +# read solution file to the end and copy it to the temp file +while (defined($line = <$solution_fh>)){ + print $temp_fh $line; +} + +close($temp_fh); +close($solution_fh); + +if($arg eq "check") { + my $diff = diff $filename => $tempfile; + if ($diff eq "") { + unlink $tempfile; + exit; + } + + print "PMDK solution file is not sorted, " . + "please use sort_solution script before pushing your changes\n"; + unlink $tempfile; + exit 1; +} else { + unlink $filename or die "Cannot replace solution file $!"; + rename $tempfile, $filename; +} diff --git a/src/pmdk/utils/style_check.sh b/src/pmdk/utils/style_check.sh new file mode 100755 index 000000000..c9d44e2a7 --- /dev/null +++ b/src/pmdk/utils/style_check.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2020, Intel Corporation +# +# utils/style_check.sh -- common style checking script +# +set -e + +ARGS=("$@") +CSTYLE_ARGS=() +CLANG_ARGS=() +FLAKE8_ARGS=() +CHECK_TYPE=$1 + +[ -z "$clang_format_bin" ] && which clang-format-9 >/dev/null && + clang_format_bin=clang-format-9 +[ -z "$clang_format_bin" ] && which clang-format >/dev/null && + clang_format_bin=clang-format +[ -z "$clang_format_bin" ] && clang_format_bin=clang-format + +# +# print script usage +# +function usage() { + echo "$0 <check|format> [C/C++ files]" +} + +# +# require clang-format version 9.0 +# +function check_clang_version() { + set +e + which ${clang_format_bin} &> /dev/null && ${clang_format_bin} --version |\ + grep "version 9\.0"\ + &> /dev/null + if [ $? -ne 0 ]; then + echo "SKIP: requires clang-format version 9.0" + exit 0 + fi + set -e +} + +# +# run old cstyle check +# +function run_cstyle() { + if [ $# -eq 0 ]; then + return + fi + + ${cstyle_bin} -pP $@ +} + +# +# generate diff with clang-format rules +# +function run_clang_check() { + if [ $# -eq 0 ]; then + return + fi + check_clang_version + + for file in $@ + do + LINES=$(${clang_format_bin} -style=file $file |\ + git diff --no-index $file - | wc -l) + if [ $LINES -ne 0 ]; then + ${clang_format_bin} -style=file $file | git diff --no-index $file - + fi + done +} + +# +# in-place format according to clang-format rules +# +function run_clang_format() { + if [ $# -eq 0 ]; then + return + fi + check_clang_version + + ${clang_format_bin} -style=file -i $@ +} + +function run_flake8() { + if [ $# -eq 0 ]; then + return + fi + ${flake8_bin} --exclude=testconfig.py,envconfig.py $@ +} + +for ((i=1; i<$#; i++)) { + + IGNORE="$(dirname ${ARGS[$i]})/.cstyleignore" + if [ -e $IGNORE ]; then + if grep -q ${ARGS[$i]} $IGNORE ; then + echo "SKIP ${ARGS[$i]}" + continue + fi + fi + case ${ARGS[$i]} in + *.[ch]pp) + CLANG_ARGS+="${ARGS[$i]} " + ;; + + *.[ch]) + CSTYLE_ARGS+="${ARGS[$i]} " + ;; + + *.py) + FLAKE8_ARGS+="${ARGS[$i]} " + ;; + + *) + echo "Unknown argument" + exit 1 + ;; + esac +} + +case $CHECK_TYPE in + check) + run_cstyle ${CSTYLE_ARGS} + run_clang_check ${CLANG_ARGS} + run_flake8 ${FLAKE8_ARGS} + ;; + + format) + run_clang_format ${CLANG_ARGS} + ;; + + *) + echo "Invalid parameters" + usage + exit 1 + ;; +esac diff --git a/src/pmdk/utils/version.sh b/src/pmdk/utils/version.sh new file mode 100755 index 000000000..fd751c9d9 --- /dev/null +++ b/src/pmdk/utils/version.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017-2020, Intel Corporation +# +# utils/version.sh -- determine project's version +# +set -e + +if [ -f "$1/VERSION" ]; then + cat "$1/VERSION" + exit 0 +fi + +if [ -f $1/GIT_VERSION ]; then + echo -n "\$Format:%h\$" | cmp -s $1/GIT_VERSION - && true + if [ $? -eq 0 ]; then + PARSE_GIT_VERSION=0 + else + PARSE_GIT_VERSION=1 + fi +else + PARSE_GIT_VERSION=0 +fi + +LATEST_RELEASE=$(cat $1/ChangeLog | grep "* Version" | cut -d " " -f 3 | sort -rd | head -n1) + +if [ $PARSE_GIT_VERSION -eq 1 ]; then + GIT_VERSION_HASH=$(cat $1/GIT_VERSION) + + if [ -n "$GIT_VERSION_HASH" ]; then + echo "$LATEST_RELEASE+git.$GIT_VERSION_HASH" + exit 0 + fi +fi + +cd "$1" + +GIT_DESCRIBE=$(git describe 2>/dev/null) && true +if [ -n "$GIT_DESCRIBE" ]; then + # 1.5-19-gb8f78a329 -> 1.5+git19.gb8f78a329 + # 1.5-rc1-19-gb8f78a329 -> 1.5-rc1+git19.gb8f78a329 + echo "$GIT_DESCRIBE" | sed "s/\([0-9.]*\)-rc\([0-9]*\)-\([0-9]*\)-\([0-9a-g]*\)/\1-rc\2+git\3.\4/" | sed "s/\([0-9.]*\)-\([0-9]*\)-\([0-9a-g]*\)/\1+git\2.\3/" + exit 0 +fi + +# try commit it, git describe can fail when there are no tags (e.g. with shallow clone, like on Travis) +GIT_COMMIT=$(git log -1 --format=%h) && true +if [ -n "$GIT_COMMIT" ]; then + echo "$LATEST_RELEASE+git.$GIT_COMMIT" + exit 0 +fi + +cd - >/dev/null + +# If nothing works, try to get version from directory name +VER=$(basename `realpath "$1"` | sed 's/pmdk[-]*\([0-9a-z.+-]*\).*/\1/') +if [ -n "$VER" ]; then + echo "$VER" + exit 0 +fi + +exit 1 |