diff options
Diffstat (limited to 'src/spdk/dpdk/devtools')
26 files changed, 2794 insertions, 0 deletions
diff --git a/src/spdk/dpdk/devtools/build-tags.sh b/src/spdk/dpdk/devtools/build-tags.sh new file mode 100755 index 000000000..276fff647 --- /dev/null +++ b/src/spdk/dpdk/devtools/build-tags.sh @@ -0,0 +1,200 @@ +#!/bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Cavium, Inc +# + +# +# Generate tags or gtags or cscope or etags files +# + +verbose=false +linux=true +bsd=true +x86_32=true +x86_64=true +ppc_64=true +arm_32=true +arm_64=true + +print_usage() +{ + echo "Usage: $(basename $0) [-h] [-v] tags|cscope|gtags|etags [config]" + echo "Valid configs are:" + make showconfigs | sed 's,^,\t,' +} + +# Move to the root of the git tree +cd $(dirname $0)/.. + +while getopts hv ARG ; do + case $ARG in + v ) verbose=true ;; + h ) print_usage; exit 0 ;; + ? ) print_usage; exit 1 ;; + esac +done +shift $(($OPTIND - 1)) + +#ignore version control files +ignore="( -name .svn -o -name CVS -o -name .hg -o -name .git ) -prune -o" + +source_dirs="app buildtools drivers examples lib" + +skip_bsd="( -name freebsd ) -prune -o" +skip_linux="( -name linux ) -prune -o" +skip_arch="( -name arch ) -prune -o" +skip_sse="( -name *_sse*.[chS] ) -prune -o" +skip_avx="( -name *_avx*.[chS] ) -prune -o" +skip_neon="( -name *_neon*.[chS] ) -prune -o" +skip_altivec="( -name *_altivec*.[chS] ) -prune -o" +skip_arm64="( -name *arm64*.[chS] ) -prune -o" +skip_x86="( -name *x86*.[chS] ) -prune -o" +skip_32b_files="( -name *_32.h ) -prune -o" +skip_64b_files="( -name *_64.h ) -prune -o" + +skiplist="$skip_bsd $skip_linux $skip_arch $skip_sse $skip_avx \ + $skip_neon $skip_altivec $skip_x86 $skip_arm64" + +find_sources() +{ + find $1 $ignore $3 -name $2 -not -type l -print +} + +common_sources() +{ + find_sources "$source_dirs" '*.[chS]' "$skiplist" +} + +linux_sources() +{ + find_sources "lib/librte_eal/linux" '*.[chS]' + find_sources "kernel/linux" '*.[chS]' +} + +bsd_sources() +{ + find_sources "lib/librte_eal/freebsd" '*.[chS]' + find_sources "kernel/freebsd" '*.[chS]' +} + +arm_common() +{ + find_sources "$source_dirs" '*neon*.[chS]' +} + +arm_32_sources() +{ + arm_common + find_sources "lib/librte_eal/arm" '*.[chS]' \ + "$skip_64b_files" +} + +arm_64_sources() +{ + arm_common + find_sources "lib/librte_eal/arm" '*.[chS]' \ + "$skip_32b_files" + find_sources "$source_dirs" '*arm64.[chS]' +} + +x86_common() +{ + find_sources "examples/performance-thread/common/arch/x86" '*.[chS]' + find_sources "$source_dirs" '*_sse*.[chS]' + find_sources "$source_dirs" '*_avx*.[chS]' + find_sources "$source_dirs" '*x86.[chS]' +} + +x86_32_sources() +{ + x86_common + find_sources "lib/librte_eal/x86" '*.[chS]' \ + "$skip_64b_files" +} + +x86_64_sources() +{ + x86_common + find_sources "lib/librte_eal/x86" '*.[chS]' \ + "$skip_32b_files" +} + +ppc_64_sources() +{ + find_sources "lib/librte_eal/ppc" '*.[chS]' + find_sources "$source_dirs" '*altivec*.[chS]' +} + +check_valid_target() +{ + if [ ! -f "config/defconfig_$1" ] ; then + echo "Invalid config: $1" + print_usage + exit 0 + fi +} + +if [ -n "$2" ]; then + check_valid_target $2 + + echo $2 | grep -q "linux" || linux=false + echo $2 | grep -q "bsd" || bsd=false + echo $2 | grep -q "x86_64-" || x86_64=false + echo $2 | grep -q "arm-" || arm_32=false + echo $2 | grep -q "arm64-" || arm_64=false + echo $2 | grep -q "ppc_64-" || ppc_64=false + echo $2 | grep -q -e "i686-" -e "x32-" || x86_32=false +fi + +all_sources() +{ + common_sources + if $linux ; then linux_sources ; fi + if $bsd ; then bsd_sources ; fi + if $x86_64 ; then x86_64_sources ; fi + if $x86_32 ; then x86_32_sources ; fi + if $ppc_64 ; then ppc_64_sources ; fi + if $arm_32 ; then arm_32_sources ; fi + if $arm_64 ; then arm_64_sources ; fi +} + +show_flags() +{ + if $verbose ; then + echo "mode: $1" + echo "config: $2" + echo "linux: $linux" + echo "bsd: $bsd" + echo "x86_32: $x86_32" + echo "x86_64: $x86_64" + echo "ppc_64: $ppc_64" + echo "arm_32: $arm_32" + echo "arm_64: $arm_64" + fi +} + +case "$1" in + "cscope") + show_flags $1 $2 + all_sources > cscope.files + cscope -q -b -f cscope.out + ;; + "gtags") + show_flags $1 $2 + all_sources | gtags -i -f - + ;; + "tags") + show_flags $1 $2 + rm -f tags + all_sources | xargs ctags -a + ;; + "etags") + show_flags $1 $2 + rm -f TAGS + all_sources | xargs etags -a + ;; + *) + echo "Invalid mode: $1" + print_usage + ;; +esac diff --git a/src/spdk/dpdk/devtools/check-abi-version.sh b/src/spdk/dpdk/devtools/check-abi-version.sh new file mode 100755 index 000000000..f0cca42a9 --- /dev/null +++ b/src/spdk/dpdk/devtools/check-abi-version.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2019 Intel Corporation + +# Check whether library symbols have correct +# version (provided ABI number or provided ABI +# number + 1 or EXPERIMENTAL or INTERNAL). +# Args: +# $1: path of the library .so file +# $2: ABI major version number to check +# (defaults to ABI_VERSION file value) + +if [ -z "$1" ]; then + echo "Script checks whether library symbols have" + echo "correct version (ABI_VER/ABI_VER+1/EXPERIMENTAL/INTERNAL)" + echo "Usage:" + echo " $0 SO_FILE_PATH [ABI_VER]" + exit 1 +fi + +LIB="$1" +DEFAULT_ABI=$(cat "$(dirname \ + $(readlink -f $0))/../ABI_VERSION" | \ + cut -d'.' -f 1) +ABIVER="DPDK_${2-$DEFAULT_ABI}" +NEXT_ABIVER="DPDK_$((${2-$DEFAULT_ABI}+1))" + +ret=0 + +# get output of objdump +OBJ_DUMP_OUTPUT=`objdump -TC --section=.text ${LIB} 2>&1 | grep ".text"` + +# there may not be any .text sections in the .so file, in which case exit early +echo "${OBJ_DUMP_OUTPUT}" | grep "not found in any input file" -q +if [ "$?" -eq 0 ]; then + exit 0 +fi + +# we have symbols, so let's see if the versions are correct +for SYM in $(echo "${OBJ_DUMP_OUTPUT}" | awk '{print $(NF-1) "-" $NF}') +do + version=$(echo $SYM | cut -d'-' -f 1) + symbol=$(echo $SYM | cut -d'-' -f 2) + case $version in (*"$ABIVER"*|*"$NEXT_ABIVER"*|"EXPERIMENTAL"|"INTERNAL") + ;; + (*) + echo "Warning: symbol $symbol ($version) should be annotated " \ + "as ABI version $ABIVER / $NEXT_ABIVER, EXPERIMENTAL, or INTERNAL." + ret=1 + ;; + esac +done + +exit $ret diff --git a/src/spdk/dpdk/devtools/check-abi.sh b/src/spdk/dpdk/devtools/check-abi.sh new file mode 100755 index 000000000..dd9120e69 --- /dev/null +++ b/src/spdk/dpdk/devtools/check-abi.sh @@ -0,0 +1,64 @@ +#!/bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2019 Red Hat, Inc. + +if [ $# != 2 ] && [ $# != 3 ]; then + echo "Usage: $0 refdir newdir [warnonly]" + exit 1 +fi + +refdir=$1 +newdir=$2 +warnonly=${3:-} +ABIDIFF_OPTIONS="--suppr $(dirname $0)/libabigail.abignore --no-added-syms" + +if [ ! -d $refdir ]; then + echo "Error: reference directory '$refdir' does not exist." + exit 1 +fi +incdir=$(find $refdir -type d -a -name include) +if [ -z "$incdir" ] || [ ! -e "$incdir" ]; then + echo "WARNING: could not identify a include directory for $refdir, expect false positives..." +else + ABIDIFF_OPTIONS="$ABIDIFF_OPTIONS --headers-dir1 $incdir" +fi + +if [ ! -d $newdir ]; then + echo "Error: directory to check '$newdir' does not exist." + exit 1 +fi +incdir2=$(find $newdir -type d -a -name include) +if [ -z "$incdir2" ] || [ ! -e "$incdir2" ]; then + echo "WARNING: could not identify a include directory for $newdir, expect false positives..." +else + ABIDIFF_OPTIONS="$ABIDIFF_OPTIONS --headers-dir2 $incdir2" +fi + +error= +for dump in $(find $refdir -name "*.dump"); do + name=$(basename $dump) + # skip glue drivers, example librte_pmd_mlx5_glue.dump + # We can't rely on a suppression rule for now: + # https://sourceware.org/bugzilla/show_bug.cgi?id=25480 + if grep -qE "\<soname='[^']*_glue\.so\.[^']*'" $dump; then + echo "Skipped glue library $name." + continue + fi + # skip experimental libraries, with a sover starting with 0. + if grep -qE "\<soname='[^']*\.so\.0\.[^']*'" $dump; then + echo "Skipped experimental library $name." + continue + fi + dump2=$(find $newdir -name $name) + if [ -z "$dump2" ] || [ ! -e "$dump2" ]; then + echo "Error: can't find $name in $newdir" + error=1 + continue + fi + if ! abidiff $ABIDIFF_OPTIONS $dump $dump2; then + echo "Error: ABI issue reported for 'abidiff $ABIDIFF_OPTIONS $dump $dump2'" + error=1 + fi +done + +[ -z "$error" ] || [ -n "$warnonly" ] diff --git a/src/spdk/dpdk/devtools/check-dup-includes.sh b/src/spdk/dpdk/devtools/check-dup-includes.sh new file mode 100755 index 000000000..591599949 --- /dev/null +++ b/src/spdk/dpdk/devtools/check-dup-includes.sh @@ -0,0 +1,18 @@ +#! /bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017 Mellanox Technologies, Ltd + +# Check C files in git repository for duplicated includes. +# Usage: devtools/check-dup-includes.sh [directory] + +dir=${1:-$(dirname $(readlink -f $0))/..} +cd $dir + +# speed up by ignoring Unicode details +export LC_ALL=C + +for file in $(git ls-files '*.[ch]') ; do + sed -rn 's,^[[:space:]]*#include[[:space:]]*[<"](.*)[>"].*,\1,p' $file | + sort | uniq -d | + sed "s,^,$file: duplicated include: ," +done diff --git a/src/spdk/dpdk/devtools/check-forbidden-tokens.awk b/src/spdk/dpdk/devtools/check-forbidden-tokens.awk new file mode 100755 index 000000000..8c89de3d4 --- /dev/null +++ b/src/spdk/dpdk/devtools/check-forbidden-tokens.awk @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018 Arnon Warshavsky <arnon@qwilt.com> + +# This awk script receives a list of expressions to monitor +# and a list of folders to search these expressions in +# - No search is done inside comments +# - Both additions and removals of the expressions are checked +# A positive balance of additions fails the check + +BEGIN { + split(FOLDERS,deny_folders," "); + split(EXPRESSIONS,deny_expr," "); + in_file=0; + in_comment=0; + count=0; + comment_start="/*" + comment_end="*/" +} +# search for add/remove instances in current file +# state machine assumes the comments structure is enforced by +# checkpatches.pl +(in_file) { + # comment start + if (index($0,comment_start) > 0) { + in_comment = 1 + } + # non comment code + if (in_comment == 0) { + for (i in deny_expr) { + forbidden_added = "^\\+.*" deny_expr[i]; + forbidden_removed="^-.*" deny_expr[i]; + current = expressions[deny_expr[i]] + if ($0 ~ forbidden_added) { + count = count + 1; + expressions[deny_expr[i]] = current + 1 + } + if ($0 ~ forbidden_removed) { + count = count - 1; + expressions[deny_expr[i]] = current - 1 + } + } + } + # comment end + if (index($0,comment_end) > 0) { + in_comment = 0 + } +} +# switch to next file , check if the balance of add/remove +# of previous filehad new additions +($0 ~ "^\\+\\+\\+ b/") { + in_file = 0; + if (count > 0) { + exit; + } + for (i in deny_folders) { + re = "^\\+\\+\\+ b/" deny_folders[i]; + if ($0 ~ deny_folders[i]) { + in_file = 1 + last_file = $0 + } + } +} +END { + if (count > 0) { + print "Warning in " substr(last_file,6) ":" + print MESSAGE + exit RET_ON_FAIL + } +} diff --git a/src/spdk/dpdk/devtools/check-git-log.sh b/src/spdk/dpdk/devtools/check-git-log.sh new file mode 100755 index 000000000..4e65be0e4 --- /dev/null +++ b/src/spdk/dpdk/devtools/check-git-log.sh @@ -0,0 +1,172 @@ +#! /bin/sh +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016 6WIND S.A. + +# Check commit logs (headlines and references) +# +# If any doubt about the formatting, please check in the most recent history: +# git log --format='%>|(15)%cr %s' --reverse | grep -i <pattern> + +if [ "$1" = '-h' -o "$1" = '--help' ] ; then + cat <<- END_OF_HELP + usage: $(basename $0) [-h] [range] + + Check commit log formatting. + The git range can be specified as a "git log" option, + e.g. -1 to check only the latest commit. + The default range starts from origin/master to HEAD. + END_OF_HELP + exit +fi + +selfdir=$(dirname $(readlink -f $0)) +range=${1:-origin/master..} +# convert -N to HEAD~N.. in order to comply with git-log-fixes.sh getopts +if printf -- $range | grep -q '^-[0-9]\+' ; then + range="HEAD$(printf -- $range | sed 's,^-,~,').." +fi + +commits=$(git log --format='%h' --reverse $range) +headlines=$(git log --format='%s' --reverse $range) +bodylines=$(git log --format='%b' --reverse $range) +fixes=$(git log --format='%h %s' --reverse $range | grep -i ': *fix' | cut -d' ' -f1) +stablefixes=$($selfdir/git-log-fixes.sh $range | sed '/(N\/A)$/d' | cut -d' ' -f2) +tags=$(git log --format='%b' --reverse $range | grep -i -e 'by *:' -e 'fix.*:') +bytag='\(Reported\|Suggested\|Signed-off\|Acked\|Reviewed\|Tested\)-by:' + +# check headline format (spacing, no punctuation, no code) +bad=$(echo "$headlines" | grep --color=always \ + -e ' ' \ + -e '^ ' \ + -e ' $' \ + -e '\.$' \ + -e '[,;!?&|]' \ + -e ':.*_' \ + -e '^[^:]\+$' \ + -e ':[^ ]' \ + -e ' :' \ + | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline format:\n$bad\n" + +# check headline prefix when touching only drivers, e.g. net/<driver name> +bad=$(for commit in $commits ; do + headline=$(git log --format='%s' -1 $commit) + files=$(git diff-tree --no-commit-id --name-only -r $commit) + [ -z "$(echo "$files" | grep -v '^\(drivers\|doc\|config\)/')" ] || + continue + drv=$(echo "$files" | grep '^drivers/' | cut -d "/" -f 2,3 | sort -u) + drvgrp=$(echo "$drv" | cut -d "/" -f 1 | uniq) + if [ $(echo "$drvgrp" | wc -l) -gt 1 ] ; then + echo "$headline" | grep -v '^drivers:' + elif [ $(echo "$drv" | wc -l) -gt 1 ] ; then + echo "$headline" | grep -v "^drivers/$drvgrp" + else + echo "$headline" | grep -v "^$drv" + fi +done | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline prefix:\n$bad\n" + +# check headline label for common typos +bad=$(echo "$headlines" | grep --color=always \ + -e '^example[:/]' \ + -e '^apps/' \ + -e '^testpmd' \ + -e 'test-pmd' \ + -e '^bond:' \ + | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline label:\n$bad\n" + +# check headline lowercase for first words +bad=$(echo "$headlines" | grep --color=always \ + -e '^.*[[:upper:]].*:' \ + -e ': *[[:upper:]]' \ + | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong headline uppercase:\n$bad\n" + +# check headline case (Rx/Tx, VF, L2, MAC, Linux ...) +IFS=' +' +words="$selfdir/words-case.txt" +for word in $(cat $words); do + bad=$(echo "$headlines" | grep -iw $word | grep -v $word) + if [ "$word" = "Tx" ]; then + bad=$(echo $bad | grep -v 'OCTEON\ TX') + fi + for bad_line in $bad; do + bad_word=$(echo $bad_line | cut -d":" -f2 | grep -io $word) + if [ -n "$bad_word" ]; then + printf "Wrong headline case:\n\"$bad_line\": $bad_word --> $word\n" + fi + done +done + +# check headline length (60 max) +bad=$(echo "$headlines" | + awk 'length>60 {print}' | + sed 's,^,\t,') +[ -z "$bad" ] || printf "Headline too long:\n$bad\n" + +# check body lines length (75 max) +bad=$(echo "$bodylines" | grep -v '^Fixes:' | + awk 'length>75 {print}' | + sed 's,^,\t,') +[ -z "$bad" ] || printf "Line too long:\n$bad\n" + +# check starting commit message with "It" +bad=$(for commit in $commits ; do + firstbodyline=$(git log --format='%b' -1 $commit | head -n1) + echo "$firstbodyline" | grep --color=always -ie '^It ' +done | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong beginning of commit message:\n$bad\n" + +# check tags spelling +bad=$(echo "$tags" | + grep -v "^$bytag [^,]* <.*@.*>$" | + grep -v '^Fixes: [0-9a-f]\{7\}[0-9a-f]* (".*")$' | + sed 's,^.,\t&,') +[ -z "$bad" ] || printf "Wrong tag:\n$bad\n" + +# check missing Coverity issue: tag +bad=$(for commit in $commits; do + body=$(git log --format='%b' -1 $commit) + echo "$body" | grep -qi coverity || continue + echo "$body" | grep -q '^Coverity issue:' && continue + git log --format='\t%s' -1 $commit +done) +[ -z "$bad" ] || printf "Missing 'Coverity issue:' tag:\n$bad\n" + +# check missing Bugzilla ID: tag +bad=$(for commit in $commits; do + body=$(git log --format='%b' -1 $commit) + echo "$body" | grep -qi bugzilla || continue + echo "$body" | grep -q '^Bugzilla ID:' && continue + git log --format='\t%s' -1 $commit +done) +[ -z "$bad" ] || printf "Missing 'Bugzilla ID:' tag:\n$bad\n" + +# check missing Fixes: tag +bad=$(for fix in $fixes ; do + git log --format='%b' -1 $fix | grep -q '^Fixes: ' || + git log --format='\t%s' -1 $fix +done) +[ -z "$bad" ] || printf "Missing 'Fixes' tag:\n$bad\n" + +# check Fixes: reference +fixtags=$(echo "$tags" | grep '^Fixes: ') +bad=$(for fixtag in $fixtags ; do + hash=$(echo "$fixtag" | sed 's,^Fixes: \([0-9a-f]*\).*,\1,') + if git branch --contains $hash 2>&- | grep -q '^\*' ; then + good="Fixes: $hash "$(git log --format='("%s")' -1 $hash 2>&-) + else + good="reference not in current branch" + fi + printf "$fixtag" | grep -v "^$good$" +done | sed 's,^,\t,') +[ -z "$bad" ] || printf "Wrong 'Fixes' reference:\n$bad\n" + +# check Cc: stable@dpdk.org for fixes +bad=$(for fix in $stablefixes ; do + git log --format='%b' -1 $fix | grep -qi '^Cc: *stable@dpdk.org' || + git log --format='\t%s' -1 $fix +done) +[ -z "$bad" ] || printf "Is it candidate for Cc: stable@dpdk.org backport?\n$bad\n" diff --git a/src/spdk/dpdk/devtools/check-includes.sh b/src/spdk/dpdk/devtools/check-includes.sh new file mode 100755 index 000000000..749b9b26d --- /dev/null +++ b/src/spdk/dpdk/devtools/check-includes.sh @@ -0,0 +1,260 @@ +#!/bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016 6WIND S.A. + +# This script checks that header files in a given directory do not miss +# dependencies when included on their own, do not conflict and accept being +# compiled with the strictest possible flags. +# +# Files are looked up in the directory provided as the first argument, +# otherwise build/include by default. +# +# Recognized environment variables: +# +# VERBOSE=1 is the same as -v. +# +# QUIET=1 is the same as -q. +# +# SUMMARY=1 is the same as -s. +# +# CC, CPPFLAGS, CFLAGS, EXTRA_CPPFLAGS, EXTRA_CFLAGS, CXX, CXXFLAGS and +# EXTRA_CXXFLAGS are taken into account. +# +# PEDANTIC_CFLAGS, PEDANTIC_CXXFLAGS and PEDANTIC_CPPFLAGS provide strict +# C/C++ compilation flags. +# +# IGNORE contains a list of globbing patterns matching files (relative to the +# include directory) to avoid. It is set by default to known DPDK headers +# which must not be included on their own. +# +# IGNORE_CXX provides additional files for C++. + +while getopts hqvs arg; do + case $arg in + h) + cat <<EOF +usage: $0 [-h] [-q] [-v] [-s] [DIR] + +This script checks that header files in a given directory do not miss +dependencies when included on their own, do not conflict and accept being +compiled with the strictest possible flags. + + -h display this help and exit + -q quiet mode, disable normal output + -v show command lines being executed + -s show summary + +With no DIR, default to build/include. + +Any failed header check yields a nonzero exit status. +EOF + exit + ;; + q) + QUIET=1 + ;; + v) + VERBOSE=1 + ;; + s) + SUMMARY=1 + ;; + *) + exit 1 + ;; + esac +done + +shift $(($OPTIND - 1)) + +include_dir=${1:-build/include} + +: ${PEDANTIC_CFLAGS=-std=c99 -pedantic -Wall -Wextra -Werror} +: ${PEDANTIC_CXXFLAGS=} +: ${PEDANTIC_CPPFLAGS=-D_XOPEN_SOURCE=600} +: ${CC:=cc} +: ${CXX:=c++} +: ${IGNORE= \ + 'rte_atomic_32.h' \ + 'rte_atomic_64.h' \ + 'rte_byteorder_32.h' \ + 'rte_byteorder_64.h' \ + 'generic/*' \ + 'rte_vhost.h' \ + 'rte_eth_vhost.h' \ + 'rte_eal_interrupts.h' \ +} +: ${IGNORE_CXX= \ + 'rte_vhost.h' \ + 'rte_eth_vhost.h' \ +} + +temp_cc=$(mktemp -t dpdk.${0##*/}.XXX.c) +pass_cc= +failures_cc=0 + +temp_cxx=$(mktemp -t dpdk.${0##*/}.XXX.cc) +pass_cxx= +failures_cxx=0 + +# Process output parameters. + +[ "$QUIET" = 1 ] && +exec 1> /dev/null + +[ "$VERBOSE" = 1 ] && +output () +{ + local CCV + local CXXV + + shift + CCV=$CC + CXXV=$CXX + CC="echo $CC" CXX="echo $CXX" "$@" + CC=$CCV + CXX=$CXXV + + "$@" +} || +output () +{ + + printf ' %s\n' "$1" + shift + "$@" +} + +trap 'rm -f "$temp_cc" "$temp_cxx"' EXIT + +compile_cc () +{ + ${CC} -I"$include_dir" \ + ${PEDANTIC_CPPFLAGS} ${CPPFLAGS} ${EXTRA_CPPFLAGS} \ + ${PEDANTIC_CFLAGS} ${CFLAGS} ${EXTRA_CFLAGS} \ + -c -o /dev/null "${temp_cc}" +} + +compile_cxx () +{ + ${CXX} -I"$include_dir" \ + ${PEDANTIC_CPPFLAGS} ${CPPFLAGS} ${EXTRA_CPPFLAGS} \ + ${PEDANTIC_CXXFLAGS} ${CXXFLAGS} ${EXTRA_CXXFLAGS} \ + -c -o /dev/null "${temp_cxx}" +} + +ignore () +{ + file="$1" + shift + while [ $# -ne 0 ]; do + case "$file" in + $1) + return 0 + ;; + esac + shift + done + return 1 +} + +# Check C/C++ compilation for each header file. + +while read -r path +do + file=${path#$include_dir} + file=${file##/} + if ignore "$file" $IGNORE; then + output "SKIP $file" : + continue + fi + if printf "\ +#include <%s> + +int main(void) +{ + return 0; +} +" "$file" > "$temp_cc" && + output "CC $file" compile_cc + then + pass_cc="$pass_cc $file" + else + failures_cc=$(($failures_cc + 1)) + fi + if ignore "$file" $IGNORE_CXX; then + output "SKIP CXX $file" : + continue + fi + if printf "\ +#include <%s> + +int main() +{ +} +" "$file" > "$temp_cxx" && + output "CXX $file" compile_cxx + then + pass_cxx="$pass_cxx $file" + else + failures_cxx=$(($failures_cxx + 1)) + fi +done <<EOF +$(find "$include_dir" -name '*.h') +EOF + +# Check C compilation with all includes. + +: > "$temp_cc" && +for file in $pass_cc; do + printf "\ +#include <%s> +" "$file" >> $temp_cc +done +if printf "\ +int main(void) +{ + return 0; +} +" >> "$temp_cc" && + output "CC (all includes that did not fail)" compile_cc +then + : +else + failures_cc=$(($failures_cc + 1)) +fi + +# Check C++ compilation with all includes. + +: > "$temp_cxx" && +for file in $pass_cxx; do + printf "\ +#include <%s> +" "$file" >> $temp_cxx +done +if printf "\ +int main() +{ +} +" >> "$temp_cxx" && + output "CXX (all includes that did not fail)" compile_cxx +then + : +else + failures_cxx=$(($failures_cxx + 1)) +fi + +# Report results. + +if [ "$SUMMARY" = 1 ]; then + printf "\ +Summary: + %u failure(s) for C using '%s'. + %u failure(s) for C++ using '%s'. +" $failures_cc "$CC" $failures_cxx "$CXX" 1>&2 +fi + +# Exit with nonzero status if there are failures. + +[ $failures_cc -eq 0 ] && +[ $failures_cxx -eq 0 ] diff --git a/src/spdk/dpdk/devtools/check-maintainers.sh b/src/spdk/dpdk/devtools/check-maintainers.sh new file mode 100755 index 000000000..85a300f0a --- /dev/null +++ b/src/spdk/dpdk/devtools/check-maintainers.sh @@ -0,0 +1,132 @@ +#! /bin/sh +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2015 6WIND S.A. + +# Do some basic checks in MAINTAINERS file + +cd $(dirname $0)/.. + +# speed up by ignoring Unicode details +export LC_ALL=C + +# Get files matching paths with wildcards and / meaning recursing +files () # <path> [<path> ...] +{ + if [ -z "$1" ] ; then + return + fi + if [ -d .git ] ; then + git ls-files "$1" + else + find "$1" -type f | + sed 's,^\./,,' + fi | + # if not ended by / + if ! echo "$1" | grep -q '/[[:space:]]*$' ; then + # filter out deeper directories + sed "/\(\/[^/]*\)\{$(($(echo "$1" | grep -o / | wc -l) + 1))\}/d" + else + cat + fi + # next path + shift + files "$@" +} + +# Get all files matching F: and X: fields +parse_fx () # <index file> +{ + IFS=' +' + # parse each line excepted underlining + for line in $( (sed '/^-\+$/d' $1 ; echo) | sed 's,^$,§,') ; do + if echo "$line" | grep -q '^§$' ; then + # empty line delimit end of section + whitelist=$(files $flines) + blacklist=$(files $xlines) + match=$(aminusb "$whitelist" "$blacklist") + if [ -n "$whitelist" ] ; then + printf "# $title " + maintainers=$(echo "$maintainers" | sed -r 's,.*<(.*)>.*,\1,') + maintainers=$(printf "$maintainers" | sed -e 's,^,<,' -e 's,$,>,') + echo $maintainers + fi + if [ -n "$match" ] ; then + echo "$match" + fi + # flush section + unset maintainers + unset flines + unset xlines + elif echo "$line" | grep -q '^[A-Z]: ' ; then + # maintainer + maintainers=$(add_line_to_if "$line" "$maintainers" 'M: ') + # file matching pattern + flines=$(add_line_to_if "$line" "$flines" 'F: ') + # file exclusion pattern + xlines=$(add_line_to_if "$line" "$xlines" 'X: ') + else # assume it is a title + title="$line" + fi + done +} + +# Check patterns in F: and X: +check_fx () # <index file> +{ + IFS=' +' + for line in $(sed -n 's,^[FX]: ,,p' $1 | tr '*' '#') ; do + line=$(printf "$line" | tr '#' '*') + match=$(files "$line") + if [ -z "$match" ] ; then + echo "$line" + fi + done +} + +# Add a line to a set of lines if it begins with right pattern +add_line_to_if () # <new line> <lines> <head pattern> +{ + ( + echo "$2" + echo "$1" | sed -rn "s,^$3(.*),\1,p" + ) | + sed '/^$/d' +} + +# Subtract two sets of lines +aminusb () # <lines a> <lines b> +{ + printf "$1\n$2\n$2" | sort | uniq -u | sed '/^$/d' +} + +printf 'sections: ' +parsed=$(parse_fx MAINTAINERS) +echo "$parsed" | grep -c '^#' +printf 'with maintainer: ' +echo "$parsed" | grep -c '^#.*@' +printf 'maintainers: ' +grep '^M:.*<' MAINTAINERS | sort -u | wc -l + +echo +echo '##########' +echo '# orphan areas' +echo '##########' +echo "$parsed" | sed -rn 's,^#([^@]*)$,\1,p' | uniq + +echo +echo '##########' +echo '# files not listed' +echo '##########' +all=$(files ./) +listed=$(echo "$parsed" | sed '/^#/d' | sort -u) +aminusb "$all" "$listed" + +echo +echo '##########' +echo '# wrong patterns' +echo '##########' +check_fx MAINTAINERS + +# TODO: check overlaps diff --git a/src/spdk/dpdk/devtools/check-symbol-change.sh b/src/spdk/dpdk/devtools/check-symbol-change.sh new file mode 100755 index 000000000..8fcd0ce1a --- /dev/null +++ b/src/spdk/dpdk/devtools/check-symbol-change.sh @@ -0,0 +1,186 @@ +#!/bin/sh +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Neil Horman <nhorman@tuxdriver.com> + +build_map_changes() +{ + local fname="$1" + local mapdb="$2" + + cat "$fname" | awk ' + # Initialize our variables + BEGIN {map="";sym="";ar="";sec=""; in_sec=0; in_map=0} + + # Anything that starts with + or -, followed by an a + # and ends in the string .map is the name of our map file + # This may appear multiple times in a patch if multiple + # map files are altered, and all section/symbol names + # appearing between a triggering of this rule and the + # next trigger of this rule are associated with this file + /[-+] [ab]\/.*\.map/ {map=$2; in_map=1; next} + + # The previous rule catches all .map files, anything else + # indicates we left the map chunk. + /[-+] [ab]\// {in_map=0} + + # Triggering this rule, which starts a line and ends it + # with a { identifies a versioned section. The section name is + # the rest of the line with the + and { symbols remvoed. + # Triggering this rule sets in_sec to 1, which actives the + # symbol rule below + /^.*{/ { + gsub("+", ""); + if (in_map == 1) { + sec=$(NF-1); in_sec=1; + } + } + + # This rule idenfies the end of a section, and disables the + # symbol rule + /.*}/ {in_sec=0} + + # This rule matches on a + followed by any characters except a : + # (which denotes a global vs local segment), and ends with a ;. + # The semicolon is removed and the symbol is printed with its + # association file name and version section, along with an + # indicator that the symbol is a new addition. Note this rule + # only works if we have found a version section in the rule + # above (hence the in_sec check) And found a map file (the + # in_map check). If we are not in a map chunk, do nothing. If + # we are in a map chunk but not a section chunk, record it as + # unknown. + /^+[^}].*[^:*];/ {gsub(";","");sym=$2; + if (in_map == 1) { + if (in_sec == 1) { + print map " " sym " " sec " add" + } else { + print map " " sym " unknown add" + } + } + } + + # This is the same rule as above, but the rule matches on a + # leading - rather than a +, denoting that the symbol is being + # removed. + /^-[^}].*[^:*];/ {gsub(";","");sym=$2; + if (in_map == 1) { + if (in_sec == 1) { + print map " " sym " " sec " del" + } else { + print map " " sym " unknown del" + } + } + }' > "$mapdb" + + sort -u "$mapdb" > "$mapdb.2" + mv -f "$mapdb.2" "$mapdb" + +} + +is_stable_section() { + [ "$1" != 'EXPERIMENTAL' ] && [ "$1" != 'INTERNAL' ] +} + +check_for_rule_violations() +{ + local mapdb="$1" + local mname + local symname + local secname + local ar + local ret=0 + + while read mname symname secname ar + do + if [ "$ar" = "add" ] + then + + if [ "$secname" = "unknown" ] + then + # Just inform the user of this occurrence, but + # don't flag it as an error + echo -n "INFO: symbol $symname is added but " + echo -n "patch has insuficient context " + echo -n "to determine the section name " + echo -n "please ensure the version is " + echo "EXPERIMENTAL" + continue + fi + + oldsecname=$(sed -n \ + "s#$mname $symname \(.*\) del#\1#p" "$mapdb") + + # A symbol can not enter a stable section directly + if [ -z "$oldsecname" ] + then + if ! is_stable_section $secname + then + echo -n "INFO: symbol $symname has " + echo -n "been added to the " + echo -n "$secname section of the " + echo "version map" + continue + else + echo -n "ERROR: symbol $symname " + echo -n "is added in the $secname " + echo -n "section, but is expected to " + echo -n "be added in the EXPERIMENTAL " + echo "section of the version map" + ret=1 + continue + fi + fi + + # This symbol is moving inside a section, nothing to do + if [ "$oldsecname" = "$secname" ] + then + continue + fi + + # This symbol is moving between two sections (the + # original section is a stable section). + # This can be legit, just warn. + if is_stable_section $oldsecname + then + echo -n "INFO: symbol $symname is being " + echo -n "moved from $oldsecname to $secname. " + echo -n "Ensure that it has gone through the " + echo "deprecation process" + continue + fi + else + + if ! grep -q "$mname $symname .* add" "$mapdb" && \ + is_stable_section $secname + then + # Just inform users that stable + # symbols need to go through a deprecation + # process + echo -n "INFO: symbol $symname is being " + echo -n "removed, ensure that it has " + echo "gone through the deprecation process" + fi + fi + done < "$mapdb" + + return $ret +} + +trap clean_and_exit_on_sig EXIT + +mapfile=`mktemp -t dpdk.mapdb.XXXXXX` +patch=$1 +exit_code=1 + +clean_and_exit_on_sig() +{ + rm -f "$mapfile" + exit $exit_code +} + +build_map_changes "$patch" "$mapfile" +check_for_rule_violations "$mapfile" +exit_code=$? +rm -f "$mapfile" + +exit $exit_code diff --git a/src/spdk/dpdk/devtools/check-symbol-maps.sh b/src/spdk/dpdk/devtools/check-symbol-maps.sh new file mode 100755 index 000000000..7fdfaa11c --- /dev/null +++ b/src/spdk/dpdk/devtools/check-symbol-maps.sh @@ -0,0 +1,34 @@ +#! /bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018 Mellanox Technologies, Ltd + +cd $(dirname $0)/.. + +# speed up by ignoring Unicode details +export LC_ALL=C + +find_orphan_symbols () +{ + for map in $(find lib drivers -name '*.map') ; do + for sym in $(sed -rn 's,^([^}]*_.*);,\1,p' $map) ; do + if echo $sym | grep -q '^per_lcore_' ; then + symsrc=${sym#per_lcore_} + elif echo $sym | grep -q '^__rte_.*_trace_' ; then + symsrc=${sym#__} + else + symsrc=$sym + fi + if ! grep -q -r --exclude=$(basename $map) \ + -w $symsrc $(dirname $map) ; then + echo "$map: $sym" + fi + done + done +} + +orphan_symbols=$(find_orphan_symbols) +if [ -n "$orphan_symbols" ] ; then + echo "Found only in symbol map file:" + echo "$orphan_symbols" | sed 's,^,\t,' + exit 1 +fi diff --git a/src/spdk/dpdk/devtools/checkpatches.sh b/src/spdk/dpdk/devtools/checkpatches.sh new file mode 100755 index 000000000..158087f1c --- /dev/null +++ b/src/spdk/dpdk/devtools/checkpatches.sh @@ -0,0 +1,284 @@ +#! /bin/sh +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2015 6WIND S.A. + +# Load config options: +# - DPDK_CHECKPATCH_PATH +# - DPDK_CHECKPATCH_CODESPELL +# - DPDK_CHECKPATCH_LINE_LENGTH +# - DPDK_CHECKPATCH_OPTIONS +. $(dirname $(readlink -f $0))/load-devel-config + +VALIDATE_NEW_API=$(dirname $(readlink -f $0))/check-symbol-change.sh + +# Enable codespell by default. This can be overwritten from a config file. +# Codespell can also be enabled by setting DPDK_CHECKPATCH_CODESPELL to a valid path +# to a dictionary.txt file if dictionary.txt is not in the default location. +codespell=${DPDK_CHECKPATCH_CODESPELL:-enable} +length=${DPDK_CHECKPATCH_LINE_LENGTH:-80} + +# override default Linux options +options="--no-tree" +if [ "$codespell" = "enable" ] ; then + options="$options --codespell" +elif [ -f "$codespell" ] ; then + options="$options --codespell" + options="$options --codespellfile $codespell" +fi +options="$options --max-line-length=$length" +options="$options --show-types" +options="$options --ignore=LINUX_VERSION_CODE,\ +FILE_PATH_CHANGES,MAINTAINERS_STYLE,SPDX_LICENSE_TAG,\ +VOLATILE,PREFER_PACKED,PREFER_ALIGNED,PREFER_PRINTF,\ +PREFER_KERNEL_TYPES,BIT_MACRO,CONST_STRUCT,\ +SPLIT_STRING,LONG_LINE_STRING,C99_COMMENT_TOLERANCE,\ +LINE_SPACING,PARENTHESIS_ALIGNMENT,NETWORKING_BLOCK_COMMENT_STYLE,\ +NEW_TYPEDEFS,COMPARISON_TO_NULL" +options="$options $DPDK_CHECKPATCH_OPTIONS" + +print_usage () { + cat <<- END_OF_HELP + usage: $(basename $0) [-q] [-v] [-nX|-r range|patch1 [patch2] ...]] + + Run Linux kernel checkpatch.pl with DPDK options. + The environment variable DPDK_CHECKPATCH_PATH must be set. + + The patches to check can be from stdin, files specified on the command line, + latest git commits limited with -n option, or commits in the git range + specified with -r option (default: "origin/master.."). + END_OF_HELP +} + +check_forbidden_additions() { # <patch> + res=0 + + # refrain from new additions of rte_panic() and rte_exit() + # multiple folders and expressions are separated by spaces + awk -v FOLDERS="lib drivers" \ + -v EXPRESSIONS="rte_panic\\\( rte_exit\\\(" \ + -v RET_ON_FAIL=1 \ + -v MESSAGE='Using rte_panic/rte_exit' \ + -f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \ + "$1" || res=1 + + # refrain from using compiler attribute without defining a common macro + awk -v FOLDERS="lib drivers app examples" \ + -v EXPRESSIONS="__attribute__" \ + -v RET_ON_FAIL=1 \ + -v MESSAGE='Using compiler attribute directly' \ + -f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \ + "$1" || res=1 + + # svg figures must be included with wildcard extension + # because of png conversion for pdf docs + awk -v FOLDERS='doc' \ + -v EXPRESSIONS='::[[:space:]]*[^[:space:]]*\\.svg' \ + -v RET_ON_FAIL=1 \ + -v MESSAGE='Using explicit .svg extension instead of .*' \ + -f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \ + "$1" || res=1 + + # links must prefer https over http + awk -v FOLDERS='doc' \ + -v EXPRESSIONS='http://.*dpdk.org' \ + -v RET_ON_FAIL=1 \ + -v MESSAGE='Using non https link to dpdk.org' \ + -f $(dirname $(readlink -f $0))/check-forbidden-tokens.awk \ + "$1" || res=1 + + return $res +} + +check_experimental_tags() { # <patch> + res=0 + + cat "$1" |awk ' + BEGIN { + current_file = ""; + ret = 0; + } + /^+++ b\// { + current_file = $2; + } + /^+.*__rte_experimental/ { + if (current_file ~ ".c$" ) { + print "Please only put __rte_experimental tags in " \ + "headers ("current_file")"; + ret = 1; + } + if ($1 != "+__rte_experimental" || $2 != "") { + print "__rte_experimental must appear alone on the line" \ + " immediately preceding the return type of a function." + ret = 1; + } + } + END { + exit ret; + }' || res=1 + + return $res +} + +check_internal_tags() { # <patch> + res=0 + + cat "$1" |awk ' + BEGIN { + current_file = ""; + ret = 0; + } + /^+++ b\// { + current_file = $2; + } + /^+.*__rte_internal/ { + if (current_file ~ ".c$" ) { + print "Please only put __rte_internal tags in " \ + "headers ("current_file")"; + ret = 1; + } + if ($1 != "+__rte_internal" || $2 != "") { + print "__rte_internal must appear alone on the line" \ + " immediately preceding the return type of" \ + " a function." + ret = 1; + } + } + END { + exit ret; + }' || res=1 + + return $res +} + +number=0 +range='origin/master..' +quiet=false +verbose=false +while getopts hn:qr:v ARG ; do + case $ARG in + n ) number=$OPTARG ;; + q ) quiet=true ;; + r ) range=$OPTARG ;; + v ) verbose=true ;; + h ) print_usage ; exit 0 ;; + ? ) print_usage ; exit 1 ;; + esac +done +shift $(($OPTIND - 1)) + +if [ ! -f "$DPDK_CHECKPATCH_PATH" ] || [ ! -x "$DPDK_CHECKPATCH_PATH" ] ; then + print_usage >&2 + echo + echo 'Cannot execute DPDK_CHECKPATCH_PATH' >&2 + exit 1 +fi + +print_headline() { # <title> + printf '\n### %s\n\n' "$1" + headline_printed=true +} + +total=0 +status=0 + +check () { # <patch> <commit> <title> + local ret=0 + headline_printed=false + + total=$(($total + 1)) + ! $verbose || print_headline "$3" + if [ -n "$1" ] ; then + tmpinput=$1 + else + tmpinput=$(mktemp -t dpdk.checkpatches.XXXXXX) + trap "rm -f '$tmpinput'" INT + + if [ -n "$2" ] ; then + git format-patch --find-renames \ + --no-stat --stdout -1 $commit > "$tmpinput" + else + cat > "$tmpinput" + fi + fi + + ! $verbose || printf 'Running checkpatch.pl:\n' + report=$($DPDK_CHECKPATCH_PATH $options "$tmpinput" 2>/dev/null) + if [ $? -ne 0 ] ; then + $headline_printed || print_headline "$3" + printf '%s\n' "$report" | sed -n '1,/^total:.*lines checked$/p' + ret=1 + fi + + ! $verbose || printf '\nChecking API additions/removals:\n' + report=$($VALIDATE_NEW_API "$tmpinput") + if [ $? -ne 0 ] ; then + $headline_printed || print_headline "$3" + printf '%s\n' "$report" + ret=1 + fi + + ! $verbose || printf '\nChecking forbidden tokens additions:\n' + report=$(check_forbidden_additions "$tmpinput") + if [ $? -ne 0 ] ; then + $headline_printed || print_headline "$3" + printf '%s\n' "$report" + ret=1 + fi + + ! $verbose || printf '\nChecking __rte_experimental tags:\n' + report=$(check_experimental_tags "$tmpinput") + if [ $? -ne 0 ] ; then + $headline_printed || print_headline "$3" + printf '%s\n' "$report" + ret=1 + fi + + ! $verbose || printf '\nChecking __rte_internal tags:\n' + report=$(check_internal_tags "$tmpinput") + if [ $? -ne 0 ] ; then + $headline_printed || print_headline "$3" + printf '%s\n' "$report" + ret=1 + fi + + if [ "$tmpinput" != "$1" ]; then + rm -f "$tmpinput" + trap - INT + fi + [ $ret -eq 0 ] && return 0 + + status=$(($status + 1)) +} + +if [ -n "$1" ] ; then + for patch in "$@" ; do + # Subject can be on 2 lines + subject=$(sed '/^Subject: */!d;s///;N;s,\n[[:space:]]\+, ,;s,\n.*,,;q' "$patch") + check "$patch" '' "$subject" + done +elif [ ! -t 0 ] ; then # stdin + subject=$(while read header value ; do + if [ "$header" = 'Subject:' ] ; then + IFS= read next + continuation=$(echo "$next" | sed -n 's,^[[:space:]]\+, ,p') + echo $value$continuation + break + fi + done) + check '' '' "$subject" +else + if [ $number -eq 0 ] ; then + commits=$(git rev-list --reverse $range) + else + commits=$(git rev-list --reverse --max-count=$number HEAD) + fi + for commit in $commits ; do + subject=$(git log --format='%s' -1 $commit) + check '' $commit "$subject" + done +fi +pass=$(($total - $status)) +$quiet || printf '\n%d/%d valid patch' $pass $total +$quiet || [ $pass -le 1 ] || printf 'es' +$quiet || printf '\n' +exit $status diff --git a/src/spdk/dpdk/devtools/cocci.sh b/src/spdk/dpdk/devtools/cocci.sh new file mode 100755 index 000000000..ab9a6efe9 --- /dev/null +++ b/src/spdk/dpdk/devtools/cocci.sh @@ -0,0 +1,36 @@ +#! /bin/sh +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2015-2020 Mellanox Technologies, Ltd + +# Apply coccinelle transforms. + +SRCTREE=$(readlink -f $(dirname $0)/..) +COCCI=$SRCTREE/devtools/cocci +[ -n "$SPATCH" ] || SPATCH=$(which spatch) + +PATCH_LIST="$@" +[ -n "$PATCH_LIST" ] || PATCH_LIST=$(echo $COCCI/*.cocci) + +[ -x "$SPATCH" ] || ( + echo "Coccinelle tools not installed." + exit 1 +) + +tmp=$(mktemp -t dpdk.cocci.XXX) + +for c in $PATCH_LIST; do + while true; do + echo -n "Applying $c..." + $SPATCH --sp-file $c -c --linux-spacing --very-quiet \ + --include-headers --preprocess \ + --in-place --dir $SRCTREE > $tmp + if [ -s $tmp ]; then + echo " changes applied, retrying." + else + echo " no change." + break; + fi + done +done + +rm -f $tmp diff --git a/src/spdk/dpdk/devtools/cocci/mtod-offset.cocci b/src/spdk/dpdk/devtools/cocci/mtod-offset.cocci new file mode 100644 index 000000000..3f83f3223 --- /dev/null +++ b/src/spdk/dpdk/devtools/cocci/mtod-offset.cocci @@ -0,0 +1,76 @@ +// +// Replace explicit packet offset computations with rte_pktmbuf_mtod_offset(). +// +@disable paren@ +typedef uint8_t; +expression M, O; +@@ +( +- rte_pktmbuf_mtod(M, char *) + O ++ rte_pktmbuf_mtod_offset(M, char *, O) +| +- rte_pktmbuf_mtod(M, char *) - O ++ rte_pktmbuf_mtod_offset(M, char *, -O) +| +- rte_pktmbuf_mtod(M, unsigned char *) + O ++ rte_pktmbuf_mtod_offset(M, unsigned char *, O) +| +- rte_pktmbuf_mtod(M, unsigned char *) - O ++ rte_pktmbuf_mtod_offset(M, unsigned char *, -O) +| +- rte_pktmbuf_mtod(M, uint8_t *) + O ++ rte_pktmbuf_mtod_offset(M, uint8_t *, O) +| +- rte_pktmbuf_mtod(M, uint8_t *) - O ++ rte_pktmbuf_mtod_offset(M, uint8_t *, -O) +) + + +// +// Fold subsequent offset terms into pre-existing offset used in +// rte_pktmbuf_mtod_offset(). +// +@disable paren@ +expression M, O1, O2; +@@ +( +- rte_pktmbuf_mtod_offset(M, char *, O1) + O2 ++ rte_pktmbuf_mtod_offset(M, char *, O1 + O2) +| +- rte_pktmbuf_mtod_offset(M, char *, O1) - O2 ++ rte_pktmbuf_mtod_offset(M, char *, O1 - O2) +| +- rte_pktmbuf_mtod_offset(M, unsigned char *, O1) + O2 ++ rte_pktmbuf_mtod_offset(M, unsigned char *, O1 + O2) +| +- rte_pktmbuf_mtod_offset(M, unsigned char *, O1) - O2 ++ rte_pktmbuf_mtod_offset(M, unsigned char *, O1 - O2) +| +- rte_pktmbuf_mtod_offset(M, uint8_t *, O1) + O2 ++ rte_pktmbuf_mtod_offset(M, uint8_t *, O1 + O2) +| +- rte_pktmbuf_mtod_offset(M, uint8_t *, O1) - O2 ++ rte_pktmbuf_mtod_offset(M, uint8_t *, O1 - O2) +) + + +// +// Cleanup rules. Fold in double casts, remove unnecessary parenthesis, etc. +// +@disable paren@ +expression M, O; +type C, T; +@@ +( +- (C)rte_pktmbuf_mtod_offset(M, T, O) ++ rte_pktmbuf_mtod_offset(M, C, O) +| +- (rte_pktmbuf_mtod_offset(M, T, O)) ++ rte_pktmbuf_mtod_offset(M, T, O) +| +- (C)rte_pktmbuf_mtod(M, T) ++ rte_pktmbuf_mtod(M, C) +| +- (rte_pktmbuf_mtod(M, T)) ++ rte_pktmbuf_mtod(M, T) +) diff --git a/src/spdk/dpdk/devtools/cocci/strlcpy-with-header.cocci b/src/spdk/dpdk/devtools/cocci/strlcpy-with-header.cocci new file mode 100644 index 000000000..046cdbdad --- /dev/null +++ b/src/spdk/dpdk/devtools/cocci/strlcpy-with-header.cocci @@ -0,0 +1,12 @@ +@include@ +@@ + +#include <rte_string_fns.h> + +@use_strlcpy depends on include@ +expression src, dst, size; +@@ +( +- snprintf(dst, size, "%s", src) ++ strlcpy(dst, src, size) +) diff --git a/src/spdk/dpdk/devtools/cocci/strlcpy.cocci b/src/spdk/dpdk/devtools/cocci/strlcpy.cocci new file mode 100644 index 000000000..6f6beb697 --- /dev/null +++ b/src/spdk/dpdk/devtools/cocci/strlcpy.cocci @@ -0,0 +1,7 @@ +@use_strlcpy@ +expression src, dst, size; +@@ +( +- snprintf(dst, size, "%s", src) ++ strlcpy(dst, src, size) +) diff --git a/src/spdk/dpdk/devtools/gen-abi.sh b/src/spdk/dpdk/devtools/gen-abi.sh new file mode 100755 index 000000000..c44b0e228 --- /dev/null +++ b/src/spdk/dpdk/devtools/gen-abi.sh @@ -0,0 +1,26 @@ +#!/bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2019 Red Hat, Inc. + +if [ $# != 1 ]; then + echo "Usage: $0 installdir" + exit 1 +fi + +installdir=$1 +if [ ! -d $installdir ]; then + echo "Error: install directory '$installdir' does not exist." + exit 1 +fi + +dumpdir=$installdir/dump +rm -rf $dumpdir +mkdir -p $dumpdir +for f in $(find $installdir -name "*.so.*"); do + if test -L $f; then + continue + fi + + libname=$(basename $f) + abidw --out-file $dumpdir/${libname%.so*}.dump $f +done diff --git a/src/spdk/dpdk/devtools/get-maintainer.sh b/src/spdk/dpdk/devtools/get-maintainer.sh new file mode 100755 index 000000000..85740f5af --- /dev/null +++ b/src/spdk/dpdk/devtools/get-maintainer.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + + +# Load config options: +# - DPDK_GETMAINTAINER_PATH +. $(dirname $(readlink -f $0))/load-devel-config + +options="--no-git-fallback" +options="$options --no-rolestats" + +print_usage () { + cat <<- END_OF_HELP + usage: $(basename $0) <patch> + + The DPDK_GETMAINTAINER_PATH variable should be set to the full path to + the get_maintainer.pl script located in Linux kernel sources. Example: + DPDK_GETMAINTAINER_PATH=~/linux/scripts/get_maintainer.pl + + Also refer to devtools/load-devel-config to store your configuration. + END_OF_HELP +} + +# Requires DPDK_GETMAINTAINER_PATH devel config option set +if [ ! -f "$DPDK_GETMAINTAINER_PATH" ] || + [ ! -x "$DPDK_GETMAINTAINER_PATH" ] ; then + print_usage >&2 + echo + echo 'Cannot execute DPDK_GETMAINTAINER_PATH' >&2 + exit 1 +fi + +FILES="COPYING CREDITS Kbuild" +FOLDERS="Documentation arch include fs init ipc scripts" + +# Kernel script checks for some files and folders to run +workaround () { + for f in $FILES; do + if [ ! -f $f ]; then touch $f; fi + done + + for d in $FOLDERS; do + if [ ! -d $d ]; then mkdir $d; fi + done +} + +fix_workaround () { + for f in $FILES; do if [ -f $f ]; then rm -f $f; fi; done + for d in $FOLDERS; do if [ -d $d ]; then rmdir $d; fi; done +} + +# clean workaround on exit +trap fix_workaround EXIT + +workaround +$DPDK_GETMAINTAINER_PATH $options $@ +# fix_workaround called on exit by trap diff --git a/src/spdk/dpdk/devtools/git-log-fixes.sh b/src/spdk/dpdk/devtools/git-log-fixes.sh new file mode 100755 index 000000000..6d468d673 --- /dev/null +++ b/src/spdk/dpdk/devtools/git-log-fixes.sh @@ -0,0 +1,124 @@ +#! /bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016 6WIND S.A. + +print_usage () +{ + echo "usage: $(basename $0) [-h] <git_range>" +} + +print_help () +{ + print_usage + cat <<- END_OF_HELP + + Find fixes to backport on previous versions. + It looks for the word "fix" in the headline or a tag "Fixes" or "Reverts". + The oldest bug origin is printed as well as partially fixed versions. + END_OF_HELP +} + +usage_error () # <message> +{ + echo "$*" >&2 + print_usage >&2 + exit 1 +} + +while getopts h ARG ; do + case $ARG in + h ) print_help ; exit 0 ;; + ? ) print_usage >&2 ; exit 1 ;; + esac +done +shift $(($OPTIND - 1)) +[ $# -ge 1 ] || usage_error 'range argument required' +range="$*" + +# get major release version of a commit +commit_version () # <hash> +{ + # use current branch as history reference + local refbranch=$(git rev-parse --abbrev-ref HEAD) + local tag=$( (git tag -l --contains $1 --merged $refbranch 2>&- || + # tag --merged option has been introduced in git 2.7.0 + # below is a fallback in case of old git version + for t in $(git tag -l --contains $1) ; do + git branch $refbranch --contains $t | + sed "s,.\+,$t," + done) | + head -n1) + if [ -z "$tag" ] ; then + # before -rc1 tag of release in progress + make showversion | cut -d'.' -f-2 + else + echo $tag | sed 's,^v,,' | sed 's,-rc.*,,' + fi +} + +# get bug origin hashes of a fix +origin_filter () # <hash> +{ + git log --format='%b' -1 $1 | + sed -n 's,^ *\([Ff]ixes\|[Rr]everts\): *\([0-9a-f]*\).*,\2,p' +} + +# get oldest major release version of bug origins +origin_version () # <origin_hash> ... +{ + for origin in $* ; do + # check hash is valid + git rev-parse -q --verify $1 >&- || continue + # get version of this bug origin + local origver=$(commit_version $origin) + local roothashes="$(origin_filter $origin)" + if [ -n "$roothashes" ] ; then + # look chained fix of fix recursively + local rootver="$(origin_version $roothashes)" + [ -n "$rootver" ] || continue + echo "$rootver (partially fixed in $origver)" + else + echo "$origver" + fi + # filter the oldest origin + done | sort -uV | head -n1 +} + +# print a marker for stable tag presence +stable_tag () # <hash> +{ + if git log --format='%b' -1 $1 | grep -qi '^Cc: *stable@dpdk.org' ; then + echo 'S' + else + echo '-' + fi +} + +# print a marker for fixes tag presence +fixes_tag () # <hash> +{ + if git log --format='%b' -1 $1 | grep -qi '^Fixes: *' ; then + echo 'F' + else + echo '-' + fi +} + +git log --oneline --reverse $range | +while read id headline ; do + origins=$(origin_filter $id) + stable=$(stable_tag $id) + fixes=$(fixes_tag $id) + [ "$stable" = "S" ] || [ "$fixes" = "F" ] || [ -n "$origins" ] || \ + echo "$headline" | grep -q fix || continue + version=$(commit_version $id) + if [ -n "$origins" ] ; then + origver="$(origin_version $origins)" + [ -n "$origver" ] || continue + # ignore fix of bug introduced in the same release + ! echo "$origver" | grep -q "^$version" || continue + else + origver='N/A' + fi + printf '%s %7s %s %s %s (%s)\n' $version $id $stable $fixes "$headline" "$origver" +done diff --git a/src/spdk/dpdk/devtools/libabigail.abignore b/src/spdk/dpdk/devtools/libabigail.abignore new file mode 100644 index 000000000..becbf842a --- /dev/null +++ b/src/spdk/dpdk/devtools/libabigail.abignore @@ -0,0 +1,79 @@ +[suppress_function] + symbol_version = EXPERIMENTAL +[suppress_variable] + symbol_version = EXPERIMENTAL + +[suppress_function] + symbol_version = INTERNAL +[suppress_variable] + symbol_version = INTERNAL + +; Ignore ABI 20.0.1 replaced with ABI 21 +[suppress_function] + symbol_version = DPDK_20.0.1 +[suppress_variable] + symbol_version = DPDK_20.0.1 + +; Explicit ignore for driver-only ABI +[suppress_type] + name = rte_cryptodev_ops +; Ignore this enum update as it is part of an experimental API +[suppress_type] + type_kind = enum + name = rte_crypto_asym_xform_type + changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END +; Ignore updates of ring prod/cons +[suppress_type] + type_kind = struct + name = rte_ring +[suppress_type] + type_kind = struct + name = rte_event_ring +; Ignore ethdev event enum update because new event cannot be +; received if not registered +[suppress_type] + type_kind = enum + name = rte_eth_event_type + changed_enumerators = RTE_ETH_EVENT_MAX +; Ignore this enum update as new flags remain unknown to applications +[suppress_type] + type_kind = enum + name = rte_cpu_flag_t + changed_enumerators = RTE_CPUFLAG_NUMFLAGS +; Ignore Cryptodev AEAD xform enum and AEAD xform strings change +; due to addition of Chacha20-Poly1305 +[suppress_type] + type_kind = enum + name = rte_crypto_aead_algorithm + changed_enumerators = RTE_CRYPTO_AEAD_LIST_END +[suppress_variable] + name = rte_crypto_aead_algorithm_strings + +;;;;;;;;;;;;;;;;;;;;;; +; Temporary exceptions for new __rte_internal marking till DPDK 20.11 +;;;;;;;;;;;;;;;;;;;;;; +; Ignore moving OCTEONTX2 stable functions to INTERNAL tag +[suppress_file] + file_name_regexp = ^librte_common_octeontx2\. +[suppress_file] + file_name_regexp = ^librte_mempool_octeontx2\. +; Ignore moving mlx5 stable functions to INTERNAL +[suppress_file] + file_name_regexp = ^librte_common_mlx5\. +; Ignore moving DPAAx stable functions to INTERNAL +[suppress_file] + file_name_regexp = ^librte_common_dpaax\. +[suppress_file] + file_name_regexp = ^librte_bus_fslmc\. +[suppress_file] + file_name_regexp = ^librte_bus_dpaa\. +[suppress_variable] + name = rte_dpaa_bpid_info +[suppress_variable] + name = rte_dpaa_memsegs +[suppress_variable] + name = rte_dpaa2_bpid_info +[suppress_function] + name = rte_dpaa2_mbuf_alloc_bulk +[suppress_function] + name_regexp = ^dpaa2?_.*tach$ diff --git a/src/spdk/dpdk/devtools/load-devel-config b/src/spdk/dpdk/devtools/load-devel-config new file mode 100644 index 000000000..69a0ac623 --- /dev/null +++ b/src/spdk/dpdk/devtools/load-devel-config @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: BSD-3-Clause + +# This file is intended to be sourced into shell + +# Load DPDK devel config and allow override +# from system file +test ! -r /etc/dpdk/devel.config || + . /etc/dpdk/devel.config +# from user file +test ! -r ~/.config/dpdk/devel.config || + . ~/.config/dpdk/devel.config +# from local file +test ! -r $(dirname $(readlink -f $0))/../.develconfig || + . $(dirname $(readlink -f $0))/../.develconfig + +# The config files must export variables in the shell style diff --git a/src/spdk/dpdk/devtools/test-build.sh b/src/spdk/dpdk/devtools/test-build.sh new file mode 100755 index 000000000..f01365602 --- /dev/null +++ b/src/spdk/dpdk/devtools/test-build.sh @@ -0,0 +1,315 @@ +#! /bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2015 6WIND S.A. + +default_path=$PATH + +# Load config options: +# - ARMV8_CRYPTO_LIB_PATH +# - DPDK_ABI_REF_DIR +# - DPDK_ABI_REF_VERSION +# - DPDK_BUILD_TEST_CONFIGS (defconfig1+option1+option2 defconfig2) +# - DPDK_BUILD_TEST_DIR +# - DPDK_DEP_ARCHIVE +# - DPDK_DEP_BPF (y/[n]) +# - DPDK_DEP_CFLAGS +# - DPDK_DEP_ELF (y/[n]) +# - DPDK_DEP_FDT (y/[n]) +# - DPDK_DEP_ISAL (y/[n]) +# - DPDK_DEP_JSON (y/[n]) +# - DPDK_DEP_LDFLAGS +# - DPDK_DEP_MLX (y/[n]) +# - DPDK_DEP_NFB (y/[n]) +# - DPDK_DEP_NUMA ([y]/n) +# - DPDK_DEP_PCAP (y/[n]) +# - DPDK_DEP_SSL (y/[n]) +# - DPDK_DEP_IPSEC_MB (y/[n]) +# - DPDK_DEP_SZE (y/[n]) +# - DPDK_DEP_ZLIB (y/[n]) +# - DPDK_MAKE_JOBS (int) +# - DPDK_NOTIFY (notify-send) +# - FLEXRAN_SDK +# - LIBMUSDK_PATH +devtools_dir=$(dirname $(readlink -f $0)) +. $devtools_dir/load-devel-config + +print_usage () { + echo "usage: $(basename $0) [-h] [-jX] [-s] [config1 [config2] ...]]" +} + +print_help () { + echo 'Test building several targets with different options' + echo + print_usage + cat <<- END_OF_HELP + + options: + -h this help + -jX use X parallel jobs in "make" + -s short test only first config without tests|examples|doc + -v verbose build + + config: defconfig[[~][+]option1[[~][+]option2...]] + Example: x86_64-native-linux-gcc+debug~RXTX_CALLBACKS + The lowercase options are defined inside $(basename $0). + The uppercase options can be the end of a defconfig option + to enable if prefixed with '+' or to disable if prefixed with '~'. + Default is to automatically enable most of the options. + The external dependencies are setup with DPDK_DEP_* variables. + If no config on command line, DPDK_BUILD_TEST_CONFIGS is used. + END_OF_HELP +} + +[ -z $MAKE ] && command -v gmake > /dev/null && MAKE=gmake +[ -z $MAKE ] && command -v make > /dev/null && MAKE=make +[ -z $MAKE ] && echo "Cannot find make or gmake" && exit 1 + +J=$DPDK_MAKE_JOBS +builds_dir=${DPDK_BUILD_TEST_DIR:-.} +short=false +unset verbose +while getopts hj:sv ARG ; do + case $ARG in + j ) J=$OPTARG ;; + s ) short=true ;; + v ) verbose='V=1' ;; + h ) print_help ; exit 0 ;; + ? ) print_usage ; exit 1 ;; + esac +done +shift $(($OPTIND - 1)) +configs=${*:-$DPDK_BUILD_TEST_CONFIGS} + +success=false +on_exit () +{ + if $success ; then + [ "$DPDK_NOTIFY" != notify-send ] || \ + notify-send -u low --icon=dialog-information 'DPDK build' 'finished' + elif [ -z "$signal" ] ; then + [ -z "$dir" ] || echo "failed to build $dir" >&2 + [ "$DPDK_NOTIFY" != notify-send ] || \ + notify-send -u low --icon=dialog-error 'DPDK build' 'failed' + fi +} +# catch manual interrupt to ignore notification +trap "signal=INT ; trap - INT ; kill -INT $$" INT +# notify result on exit +trap on_exit EXIT + +cd $devtools_dir/.. + +reset_env () +{ + export PATH=$default_path + unset CROSS + unset DPDK_DEP_ARCHIVE + unset DPDK_DEP_BPF + unset DPDK_DEP_CFLAGS + unset DPDK_DEP_ELF + unset DPDK_DEP_FDT + unset DPDK_DEP_ISAL + unset DPDK_DEP_JSON + unset DPDK_DEP_LDFLAGS + unset DPDK_DEP_MLX + unset DPDK_DEP_NFB + unset DPDK_DEP_NUMA + unset DPDK_DEP_PCAP + unset DPDK_DEP_SSL + unset DPDK_DEP_IPSEC_MB + unset DPDK_DEP_SZE + unset DPDK_DEP_ZLIB + unset ARMV8_CRYPTO_LIB_PATH + unset FLEXRAN_SDK + unset LIBMUSDK_PATH + unset PQOS_INSTALL_PATH +} + +config () # <directory> <target> <options> +{ + reconfig=false + if git rev-parse 2>&- && [ -n "$(git diff HEAD~ -- config)" ] ; then + echo 'Default config may have changed' + reconfig=true + fi + if [ ! -e $1/.config ] || $reconfig ; then + echo "================== Configure $1" + ${MAKE} T=$2 O=$1 config + + echo 'Customize configuration' + # Built-in options (lowercase) + ! echo $3 | grep -q '+default' || \ + sed -ri="" 's,(RTE_MACHINE=")native,\1default,' $1/.config + ! echo $3 | grep -q '+kmods' || \ + sed -ri="" 's,(IGB_UIO=|KNI_KMOD=)n,\1y,' $1/.config + echo $3 | grep -q '+next' || \ + sed -ri="" 's,(NEXT_ABI=)y,\1n,' $1/.config + ! echo $3 | grep -q '+shared' || \ + sed -ri="" 's,(SHARED_LIB=)n,\1y,' $1/.config + ! echo $3 | grep -q '+debug' || ( \ + sed -ri="" 's,(RTE_LOG_DP_LEVEL=).*,\1RTE_LOG_DEBUG,' $1/.config + sed -ri="" 's,(_DEBUG.*=)n,\1y,' $1/.config + sed -ri="" 's,(_STAT)([S_].*=|=)n,\1\2y,' $1/.config + sed -ri="" 's,(TEST_PMD_RECORD_.*=)n,\1y,' $1/.config ) + + # Automatic configuration + ! echo $2 | grep -q 'arm64' || \ + sed -ri="" 's,(ARM_USE_WFE=)n,\1y,' $1/.config + test "$DPDK_DEP_NUMA" != n || \ + sed -ri="" 's,(NUMA.*=)y,\1n,' $1/.config + sed -ri="" 's,(LIBRTE_IEEE1588=)n,\1y,' $1/.config + sed -ri="" 's,(BYPASS=)n,\1y,' $1/.config + test "$DPDK_DEP_ARCHIVE" != y || \ + sed -ri="" 's,(RESOURCE_TAR=)n,\1y,' $1/.config + test "$DPDK_DEP_BPF" != y || \ + sed -ri="" 's,(PMD_AF_XDP=)n,\1y,' $1/.config + test "$DPDK_DEP_FDT" != y || \ + sed -ri="" 's,(PMD_IFPGA_RAWDEV=)n,\1y,' $1/.config + test "$DPDK_DEP_FDT" != y || \ + sed -ri="" 's,(IPN3KE_PMD=)n,\1y,' $1/.config + test "$DPDK_DEP_ISAL" != y || \ + sed -ri="" 's,(PMD_ISAL=)n,\1y,' $1/.config + test "$DPDK_DEP_MLX" != y || \ + sed -ri="" 's,(MLX.*_PMD=)n,\1y,' $1/.config + test "$DPDK_DEP_NFB" != y || \ + sed -ri="" 's,(NFB_PMD=)n,\1y,' $1/.config + test "$DPDK_DEP_SZE" != y || \ + sed -ri="" 's,(PMD_SZEDATA2=)n,\1y,' $1/.config + test "$DPDK_DEP_ZLIB" != y || \ + sed -ri="" 's,(BNX2X_PMD=)n,\1y,' $1/.config + test "$DPDK_DEP_ZLIB" != y || \ + sed -ri="" 's,(PMD_ZLIB=)n,\1y,' $1/.config + test "$DPDK_DEP_ZLIB" != y || \ + sed -ri="" 's,(COMPRESSDEV_TEST=)n,\1y,' $1/.config + test "$DPDK_DEP_PCAP" != y || \ + sed -ri="" 's,(PCAP=)n,\1y,' $1/.config + test -z "$ARMV8_CRYPTO_LIB_PATH" || \ + sed -ri="" 's,(PMD_ARMV8_CRYPTO=)n,\1y,' $1/.config + test "$DPDK_DEP_IPSEC_MB" != y || \ + sed -ri="" 's,(PMD_AESNI_MB=)n,\1y,' $1/.config + test "$DPDK_DEP_IPSEC_MB" != y || \ + sed -ri="" 's,(PMD_AESNI_GCM=)n,\1y,' $1/.config + test "$DPDK_DEP_IPSEC_MB" != y || \ + sed -ri="" 's,(PMD_ZUC=)n,\1y,' $1/.config + test "$DPDK_DEP_IPSEC_MB" != y || \ + sed -ri="" 's,(PMD_KASUMI=)n,\1y,' $1/.config + test "$DPDK_DEP_IPSEC_MB" != y || \ + sed -ri="" 's,(PMD_SNOW3G=)n,\1y,' $1/.config + test "$DPDK_DEP_SSL" != y || \ + sed -ri="" 's,(PMD_CCP=)n,\1y,' $1/.config + test "$DPDK_DEP_SSL" != y || \ + sed -ri="" 's,(PMD_OPENSSL=)n,\1y,' $1/.config + test "$DPDK_DEP_SSL" != y || \ + sed -ri="" 's,(QAT_SYM=)n,\1y,' $1/.config + test -z "$FLEXRAN_SDK" || \ + sed -ri="" 's,(BBDEV_TURBO_SW=)n,\1y,' $1/.config + sed -ri="" 's,(SCHED_.*=)n,\1y,' $1/.config + test -z "$LIBMUSDK_PATH" || \ + sed -ri="" 's,(PMD_MVSAM_CRYPTO=)n,\1y,' $1/.config + test -z "$LIBMUSDK_PATH" || \ + sed -ri="" 's,(MVPP2_PMD=)n,\1y,' $1/.config + test -z "$LIBMUSDK_PATH" || \ + sed -ri="" 's,(MVNETA_PMD=)n,\1y,' $1/.config + test "$DPDK_DEP_ELF" != y || \ + sed -ri="" 's,(BPF_ELF=)n,\1y,' $1/.config + test "$DPDK_DEP_JSON" != y || \ + sed -ri="" 's,(TELEMETRY=)n,\1y,' $1/.config + build_config_hook $1 $2 $3 + + # Explicit enabler/disabler (uppercase) + for option in $(echo $3 | sed 's,[~+], &,g') ; do + pattern=$(echo $option | cut -c2-) + if echo $option | grep -q '^~' ; then + sed -ri="" "s,($pattern=)y,\1n," $1/.config + elif echo $option | grep -q '^+' ; then + sed -ri="" "s,($pattern=)n,\1y," $1/.config + fi + done + fi +} + +# default empty hook to override in devel config +build_config_hook () # <directory> <target> <options> +{ + : +} + +for conf in $configs ; do + target=$(echo $conf | sed 's,[~+].*,,') + # reload config with DPDK_TARGET set + DPDK_TARGET=$target + reset_env + . $devtools_dir/load-devel-config + + options=$(echo $conf | sed 's,[^~+]*,,') + dir=$builds_dir/$conf + config $dir $target $options + + echo "================== Build $conf" + ${MAKE} -j$J EXTRA_CFLAGS="-Wfatal-errors -g $DPDK_DEP_CFLAGS" \ + EXTRA_LDFLAGS="$DPDK_DEP_LDFLAGS" $verbose O=$dir + ! $short || break + export RTE_TARGET=$target + rm -rf $dir/install + ${MAKE} install O=$dir DESTDIR=$dir/install prefix= + echo "================== Build examples for $conf" + export RTE_SDK=$(readlink -f $dir)/install/share/dpdk + ln -sTf $(pwd)/lib $RTE_SDK/lib # workaround for vm_power_manager + grep -q 'SHARED_LIB=n' $dir/.config || # skip examples with static libs + ${MAKE} -j$J -sC examples \ + EXTRA_LDFLAGS="$DPDK_DEP_LDFLAGS" $verbose \ + O=$(readlink -f $dir)/examples + unset RTE_TARGET + grep -q 'SHARED_LIB=n' $dir/.config || # skip ABI check with static libs + if [ -n "$DPDK_ABI_REF_VERSION" ]; then + abirefdir=${DPDK_ABI_REF_DIR:-reference}/$DPDK_ABI_REF_VERSION + if [ ! -d $abirefdir/$conf ]; then + # clone current sources + if [ ! -d $abirefdir/src ]; then + git clone --local --no-hardlinks \ + --single-branch \ + -b $DPDK_ABI_REF_VERSION \ + $(pwd) $abirefdir/src + fi + + cd $abirefdir/src + + rm -rf $abirefdir/build + config $abirefdir/build $target $options + + echo -n "================== Build $conf " + echo "($DPDK_ABI_REF_VERSION)" + ${MAKE} -j$J \ + EXTRA_CFLAGS="-Wno-error -g $DPDK_DEP_CFLAGS" \ + EXTRA_LDFLAGS="$DPDK_DEP_LDFLAGS" $verbose \ + O=$abirefdir/build + export RTE_TARGET=$target + ${MAKE} install O=$abirefdir/build \ + DESTDIR=$abirefdir/$conf \ + prefix= + unset RTE_TARGET + $devtools_dir/gen-abi.sh $abirefdir/$conf + + # back to current workdir + cd $devtools_dir/.. + fi + + echo "================== Check ABI $conf" + $devtools_dir/gen-abi.sh $dir/install + $devtools_dir/check-abi.sh $abirefdir/$conf $dir/install + fi + echo "################## $conf done." + unset dir +done + +if ! $short ; then + mkdir -p .check + echo "================== Build doxygen HTML API" + ${MAKE} doc-api-html >/dev/null 2>.check/doc.txt + echo "================== Build sphinx HTML guides" + ${MAKE} doc-guides-html >/dev/null 2>>.check/doc.txt + echo "================== Check docs" + diff -u /dev/null .check/doc.txt +fi + +success=true diff --git a/src/spdk/dpdk/devtools/test-meson-builds.sh b/src/spdk/dpdk/devtools/test-meson-builds.sh new file mode 100755 index 000000000..18b874fac --- /dev/null +++ b/src/spdk/dpdk/devtools/test-meson-builds.sh @@ -0,0 +1,223 @@ +#! /bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Intel Corporation + +# Run meson to auto-configure the various builds. +# * all builds get put in a directory whose name starts with "build-" +# * if a build-directory already exists we assume it was properly configured +# Run ninja after configuration is done. + +# set pipefail option if possible +PIPEFAIL="" +set -o | grep -q pipefail && set -o pipefail && PIPEFAIL=1 + +srcdir=$(dirname $(readlink -f $0))/.. +. $srcdir/devtools/load-devel-config + +MESON=${MESON:-meson} +use_shared="--default-library=shared" +builds_dir=${DPDK_BUILD_TEST_DIR:-.} + +if command -v gmake >/dev/null 2>&1 ; then + MAKE=gmake +else + MAKE=make +fi +if command -v ninja >/dev/null 2>&1 ; then + ninja_cmd=ninja +elif command -v ninja-build >/dev/null 2>&1 ; then + ninja_cmd=ninja-build +else + echo "ERROR: ninja is not found" >&2 + exit 1 +fi +if command -v ccache >/dev/null 2>&1 ; then + CCACHE=ccache +else + CCACHE= +fi + +default_path=$PATH +default_pkgpath=$PKG_CONFIG_PATH +default_cppflags=$CPPFLAGS +default_cflags=$CFLAGS +default_ldflags=$LDFLAGS + +load_env () # <target compiler> +{ + targetcc=$1 + export PATH=$default_path + export PKG_CONFIG_PATH=$default_pkgpath + export CPPFLAGS=$default_cppflags + export CFLAGS=$default_cflags + export LDFLAGS=$default_ldflags + unset DPDK_MESON_OPTIONS + command -v $targetcc >/dev/null 2>&1 || return 1 + DPDK_TARGET=$($targetcc -v 2>&1 | sed -n 's,^Target: ,,p') + . $srcdir/devtools/load-devel-config +} + +config () # <dir> <builddir> <meson options> +{ + dir=$1 + shift + builddir=$1 + shift + if [ -f "$builddir/build.ninja" ] ; then + # for existing environments, switch to debugoptimized if unset + # so that ABI checks can run + if ! $MESON configure $builddir | + awk '$1=="buildtype" {print $2}' | + grep -qw debugoptimized; then + $MESON configure --buildtype=debugoptimized $builddir + fi + return + fi + options= + if echo $* | grep -qw -- '--default-library=shared' ; then + options="$options -Dexamples=all" + else + options="$options -Dexamples=l3fwd" # save disk space + fi + options="$options --buildtype=debugoptimized" + for option in $DPDK_MESON_OPTIONS ; do + options="$options -D$option" + done + options="$options $*" + echo "$MESON $options $dir $builddir" + $MESON $options $dir $builddir +} + +compile () # <builddir> +{ + builddir=$1 + if [ -n "$TEST_MESON_BUILD_VERY_VERBOSE" ] ; then + # for full output from ninja use "-v" + echo "$ninja_cmd -v -C $builddir" + $ninja_cmd -v -C $builddir + elif [ -n "$TEST_MESON_BUILD_VERBOSE" ] ; then + # for keeping the history of short cmds, pipe through cat + echo "$ninja_cmd -C $builddir | cat" + $ninja_cmd -C $builddir | cat + else + echo "$ninja_cmd -C $builddir" + $ninja_cmd -C $builddir + fi +} + +install_target () # <builddir> <installdir> +{ + rm -rf $2 + if [ -n "$TEST_MESON_BUILD_VERY_VERBOSE$TEST_MESON_BUILD_VERBOSE" ]; then + echo "DESTDIR=$2 $ninja_cmd -C $1 install" + DESTDIR=$2 $ninja_cmd -C $1 install + else + echo "DESTDIR=$2 $ninja_cmd -C $1 install >/dev/null" + DESTDIR=$2 $ninja_cmd -C $1 install >/dev/null + fi +} + +build () # <directory> <target compiler> <meson options> +{ + targetdir=$1 + shift + targetcc=$1 + shift + # skip build if compiler not available + command -v ${CC##* } >/dev/null 2>&1 || return 0 + load_env $targetcc || return 0 + config $srcdir $builds_dir/$targetdir --werror $* + compile $builds_dir/$targetdir + if [ -n "$DPDK_ABI_REF_VERSION" ]; then + abirefdir=${DPDK_ABI_REF_DIR:-reference}/$DPDK_ABI_REF_VERSION + if [ ! -d $abirefdir/$targetdir ]; then + # clone current sources + if [ ! -d $abirefdir/src ]; then + git clone --local --no-hardlinks \ + --single-branch \ + -b $DPDK_ABI_REF_VERSION \ + $srcdir $abirefdir/src + fi + + rm -rf $abirefdir/build + config $abirefdir/src $abirefdir/build $* + compile $abirefdir/build + install_target $abirefdir/build $abirefdir/$targetdir + $srcdir/devtools/gen-abi.sh $abirefdir/$targetdir + fi + + install_target $builds_dir/$targetdir \ + $(readlink -f $builds_dir/$targetdir/install) + $srcdir/devtools/gen-abi.sh \ + $(readlink -f $builds_dir/$targetdir/install) + $srcdir/devtools/check-abi.sh $abirefdir/$targetdir \ + $(readlink -f $builds_dir/$targetdir/install) + fi +} + +if [ "$1" = "-vv" ] ; then + TEST_MESON_BUILD_VERY_VERBOSE=1 +elif [ "$1" = "-v" ] ; then + TEST_MESON_BUILD_VERBOSE=1 +fi +# we can't use plain verbose when we don't have pipefail option so up-level +if [ -z "$PIPEFAIL" -a -n "$TEST_MESON_BUILD_VERBOSE" ] ; then + echo "# Missing pipefail shell option, changing VERBOSE to VERY_VERBOSE" + TEST_MESON_BUILD_VERY_VERBOSE=1 +fi + +# shared and static linked builds with gcc and clang +for c in gcc clang ; do + command -v $c >/dev/null 2>&1 || continue + for s in static shared ; do + export CC="$CCACHE $c" + build build-$c-$s $c --default-library=$s + unset CC + done +done + +# test compilation with minimal x86 instruction set +# Set the install path for libraries to "lib" explicitly to prevent problems +# with pkg-config prefixes if installed in "lib/x86_64-linux-gnu" later. +default_machine='nehalem' +ok=$(cc -march=$default_machine -E - < /dev/null > /dev/null 2>&1 || echo false) +if [ "$ok" = "false" ] ; then + default_machine='corei7' +fi +build build-x86-default cc -Dlibdir=lib -Dmachine=$default_machine $use_shared + +c=aarch64-linux-gnu-gcc +# generic armv8a with clang as host compiler +export CC="clang" +build build-arm64-host-clang $c $use_shared \ + --cross-file $srcdir/config/arm/arm64_armv8_linux_gcc +unset CC +# all gcc/arm configurations +for f in $srcdir/config/arm/arm64_[bdo]*gcc ; do + export CC="$CCACHE gcc" + build build-$(basename $f | tr '_' '-' | cut -d'-' -f-2) $c \ + $use_shared --cross-file $f + unset CC +done + +# Test installation of the x86-default target, to be used for checking +# the sample apps build using the pkg-config file for cflags and libs +build_path=$(readlink -f $builds_dir/build-x86-default) +export DESTDIR=$build_path/install +# No need to reinstall if ABI checks are enabled +if [ -z "$DPDK_ABI_REF_VERSION" ]; then + install_target $build_path $DESTDIR +fi + +load_env cc +pc_file=$(find $DESTDIR -name libdpdk.pc) +export PKG_CONFIG_PATH=$(dirname $pc_file):$PKG_CONFIG_PATH + +# if pkg-config defines the necessary flags, test building some examples +if pkg-config --define-prefix libdpdk >/dev/null 2>&1; then + export PKGCONF="pkg-config --define-prefix" + for example in cmdline helloworld l2fwd l3fwd skeleton timer; do + echo "## Building $example" + $MAKE -C $DESTDIR/usr/local/share/dpdk/examples/$example clean all + done +fi diff --git a/src/spdk/dpdk/devtools/test-null.sh b/src/spdk/dpdk/devtools/test-null.sh new file mode 100755 index 000000000..548de8113 --- /dev/null +++ b/src/spdk/dpdk/devtools/test-null.sh @@ -0,0 +1,32 @@ +#! /bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2015 6WIND S.A. +# Copyright 2019 Mellanox Technologies, Ltd + +# Run a quick testpmd forwarding with null PMD without hugepage + +build=${1:-build} # first argument can be the build directory +testpmd=$1 # or first argument can be the testpmd path +coremask=${2:-3} # default using cores 0 and 1 +eal_options=$3 +testpmd_options=$4 + +[ -f "$testpmd" ] && build=$(dirname $(dirname $testpmd)) +[ -f "$testpmd" ] || testpmd=$build/app/dpdk-testpmd +[ -f "$testpmd" ] || testpmd=$build/app/testpmd +if [ ! -f "$testpmd" ] ; then + echo 'ERROR: testpmd cannot be found' >&2 + exit 1 +fi + +if ldd $testpmd | grep -q librte_ ; then + export LD_LIBRARY_PATH=$build/drivers:$build/lib:$LD_LIBRARY_PATH + libs='-d librte_mempool_ring.so -d librte_pmd_null.so' +else + libs= +fi + +(sleep 1 && echo stop) | +$testpmd -c $coremask --no-huge -m 20 \ + $libs -w 0:0.0 --vdev net_null1 --vdev net_null2 $eal_options -- \ + --no-mlockall --total-num-mbufs=2048 $testpmd_options -ia diff --git a/src/spdk/dpdk/devtools/update-abi.sh b/src/spdk/dpdk/devtools/update-abi.sh new file mode 100755 index 000000000..b9b859a3e --- /dev/null +++ b/src/spdk/dpdk/devtools/update-abi.sh @@ -0,0 +1,46 @@ +#!/bin/sh -e +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2019 Intel Corporation + +abi_version=$1 +abi_version_file="./ABI_VERSION" +update_path="lib drivers" + +# check ABI version format string +check_abi_version() { + echo $1 | grep -q -e "^[[:digit:]]\{1,2\}\.[[:digit:]]\{1,2\}$" +} + +if [ -z "$1" ]; then + # output to stderr + >&2 echo "Please provide ABI version" + exit 1 +fi + +# check version string format +if ! check_abi_version $abi_version ; then + # output to stderr + >&2 echo "ABI version must be formatted as MAJOR.MINOR version" + exit 1 +fi + +if [ -n "$2" ]; then + abi_version_file=$2 +fi + +if [ -n "$3" ]; then + # drop $1 and $2 + shift 2 + # assign all other arguments as update paths + update_path=$@ +fi + +echo "New ABI version:" $abi_version +echo "ABI_VERSION path:" $abi_version_file +echo "Path to update:" $update_path + +echo $abi_version > $abi_version_file + +find $update_path -name \*version.map -exec \ + devtools/update_version_map_abi.py {} \ + $abi_version \; -print diff --git a/src/spdk/dpdk/devtools/update_version_map_abi.py b/src/spdk/dpdk/devtools/update_version_map_abi.py new file mode 100755 index 000000000..e2104e61e --- /dev/null +++ b/src/spdk/dpdk/devtools/update_version_map_abi.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2019 Intel Corporation + +""" +A Python program that updates and merges all available stable ABI versions into +one ABI version, while leaving experimental ABI exactly as it is. The intended +ABI version is supplied via command-line parameter. This script is to be called +from the devtools/update-abi.sh utility. +""" + +from __future__ import print_function +import argparse +import sys +import re + + +def __parse_map_file(f_in): + # match function name, followed by semicolon, followed by EOL, optionally + # with whitespace in between each item + func_line_regex = re.compile(r"\s*" + r"(?P<func>[a-zA-Z_0-9]+)" + r"\s*" + r";" + r"\s*" + r"$") + # match section name, followed by opening bracked, followed by EOL, + # optionally with whitespace in between each item + section_begin_regex = re.compile(r"\s*" + r"(?P<version>[a-zA-Z0-9_\.]+)" + r"\s*" + r"{" + r"\s*" + r"$") + # match closing bracket, optionally followed by section name (for when we + # inherit from another ABI version), followed by semicolon, followed by + # EOL, optionally with whitespace in between each item + section_end_regex = re.compile(r"\s*" + r"}" + r"\s*" + r"(?P<parent>[a-zA-Z0-9_\.]+)?" + r"\s*" + r";" + r"\s*" + r"$") + + # for stable ABI, we don't care about which version introduced which + # function, we just flatten the list. there are dupes in certain files, so + # use a set instead of a list + stable_lines = set() + # copy experimental section as is + experimental_lines = [] + # copy internal section as is + internal_lines = [] + in_experimental = False + in_internal = False + has_stable = False + + # gather all functions + for line in f_in: + # clean up the line + line = line.strip('\n').strip() + + # is this an end of section? + match = section_end_regex.match(line) + if match: + # whatever section this was, it's not active any more + in_experimental = False + in_internal = False + continue + + # if we're in the middle of experimental section, we need to copy + # the section verbatim, so just add the line + if in_experimental: + experimental_lines += [line] + continue + + # if we're in the middle of internal section, we need to copy + # the section verbatim, so just add the line + if in_internal: + internal_lines += [line] + continue + + # skip empty lines + if not line: + continue + + # is this a beginning of a new section? + match = section_begin_regex.match(line) + if match: + cur_section = match.group("version") + # is it experimental? + in_experimental = cur_section == "EXPERIMENTAL" + # is it internal? + in_internal = cur_section == "INTERNAL" + if not in_experimental and not in_internal: + has_stable = True + continue + + # is this a function? + match = func_line_regex.match(line) + if match: + stable_lines.add(match.group("func")) + + return has_stable, stable_lines, experimental_lines, internal_lines + + +def __generate_stable_abi(f_out, abi_version, lines): + # print ABI version header + print("DPDK_{} {{".format(abi_version), file=f_out) + + # print global section if it exists + if lines: + print("\tglobal:", file=f_out) + # blank line + print(file=f_out) + + # print all stable lines, alphabetically sorted + for line in sorted(lines): + print("\t{};".format(line), file=f_out) + + # another blank line + print(file=f_out) + + # print local section + print("\tlocal: *;", file=f_out) + + # end stable version + print("};", file=f_out) + + +def __generate_experimental_abi(f_out, lines): + # start experimental section + print("EXPERIMENTAL {", file=f_out) + + # print all experimental lines as they were + for line in lines: + # don't print empty whitespace + if not line: + print("", file=f_out) + else: + print("\t{}".format(line), file=f_out) + + # end section + print("};", file=f_out) + +def __generate_internal_abi(f_out, lines): + # start internal section + print("INTERNAL {", file=f_out) + + # print all internal lines as they were + for line in lines: + # don't print empty whitespace + if not line: + print("", file=f_out) + else: + print("\t{}".format(line), file=f_out) + + # end section + print("};", file=f_out) + +def __main(): + arg_parser = argparse.ArgumentParser( + description='Merge versions in linker version script.') + + arg_parser.add_argument("map_file", type=str, + help='path to linker version script file ' + '(pattern: *version.map)') + arg_parser.add_argument("abi_version", type=str, + help='target ABI version (pattern: MAJOR.MINOR)') + + parsed = arg_parser.parse_args() + + if not parsed.map_file.endswith('version.map'): + print("Invalid input file: {}".format(parsed.map_file), + file=sys.stderr) + arg_parser.print_help() + sys.exit(1) + + if not re.match(r"\d{1,2}\.\d{1,2}", parsed.abi_version): + print("Invalid ABI version: {}".format(parsed.abi_version), + file=sys.stderr) + arg_parser.print_help() + sys.exit(1) + + with open(parsed.map_file) as f_in: + has_stable, stable_lines, experimental_lines, internal_lines = __parse_map_file(f_in) + + with open(parsed.map_file, 'w') as f_out: + need_newline = has_stable and experimental_lines + if has_stable: + __generate_stable_abi(f_out, parsed.abi_version, stable_lines) + if need_newline: + # separate sections with a newline + print(file=f_out) + if experimental_lines: + __generate_experimental_abi(f_out, experimental_lines) + if internal_lines: + if has_stable or experimental_lines: + # separate sections with a newline + print(file=f_out) + __generate_internal_abi(f_out, internal_lines) + + +if __name__ == "__main__": + __main() diff --git a/src/spdk/dpdk/devtools/words-case.txt b/src/spdk/dpdk/devtools/words-case.txt new file mode 100644 index 000000000..bdfd2a2d9 --- /dev/null +++ b/src/spdk/dpdk/devtools/words-case.txt @@ -0,0 +1,65 @@ +aarch32 +aarch64 +AltiVec +API +Arm +armv7 +armv8 +BAR +CRC +DCB +DMA +EEPROM +FDIR +FreeBSD +FW +GENEVE +HW +IO +IOTLB +IOVA +IPsec +L2 +L3 +L4 +LACP +Linux +LRO +LSC +MAC +MPLS +MSS +MTU +NEON +NIC +NUMA +null +NVGRE +NVM +PCI +PF +PHY +PMD +PPPoE +PVID +RDMA +RETA +RSS +Rx +SCTP +SW +TC +TOS +TPID +TSO +TTL +Tx +UDP +vDPA +VF +VFIO +VLAN +VMDq +VSI +VXLAN +XDP |