diff options
Diffstat (limited to 'src/test/encoding')
| -rw-r--r-- | src/test/encoding/CMakeLists.txt | 3 | ||||
| -rwxr-xr-x | src/test/encoding/check-generated.sh | 101 | ||||
| -rwxr-xr-x | src/test/encoding/generate-corpus-objects.sh | 59 | ||||
| -rwxr-xr-x | src/test/encoding/identity.sh | 32 | ||||
| -rwxr-xr-x | src/test/encoding/import-generated.sh | 30 | ||||
| -rwxr-xr-x | src/test/encoding/import.sh | 23 | ||||
| -rwxr-xr-x | src/test/encoding/readable.sh | 241 |
7 files changed, 489 insertions, 0 deletions
diff --git a/src/test/encoding/CMakeLists.txt b/src/test/encoding/CMakeLists.txt new file mode 100644 index 000000000..e8c57a042 --- /dev/null +++ b/src/test/encoding/CMakeLists.txt @@ -0,0 +1,3 @@ +# scripts +add_ceph_test(check-generated.sh ${CMAKE_CURRENT_SOURCE_DIR}/check-generated.sh) +add_ceph_test(readable.sh ${CMAKE_CURRENT_SOURCE_DIR}/readable.sh) diff --git a/src/test/encoding/check-generated.sh b/src/test/encoding/check-generated.sh new file mode 100755 index 000000000..2569bc1a5 --- /dev/null +++ b/src/test/encoding/check-generated.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -e + +source $(dirname $0)/../detect-build-env-vars.sh +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +dir=$1 + +tmp1=`mktemp /tmp/typ-XXXXXXXXX` +tmp2=`mktemp /tmp/typ-XXXXXXXXX` +tmp3=`mktemp /tmp/typ-XXXXXXXXX` +tmp4=`mktemp /tmp/typ-XXXXXXXXX` + +failed=0 +numtests=0 +echo "checking ceph-dencoder generated test instances..." +echo "numgen type" +while read type; do + num=`ceph-dencoder type $type count_tests` + echo "$num $type" + for n in `seq 1 1 $num 2>/dev/null`; do + + pids="" + run_in_background pids save_stdout "$tmp1" ceph-dencoder type "$type" select_test "$n" dump_json + run_in_background pids save_stdout "$tmp2" ceph-dencoder type "$type" select_test "$n" encode decode dump_json + run_in_background pids save_stdout "$tmp3" ceph-dencoder type "$type" select_test "$n" copy dump_json + run_in_background pids save_stdout "$tmp4" ceph-dencoder type "$type" select_test "$n" copy_ctor dump_json + wait_background pids + + if [ $? -ne 0 ]; then + echo "**** $type test $n encode+decode check failed ****" + echo " ceph-dencoder type $type select_test $n encode decode" + failed=$(($failed + 3)) + continue + fi + + # nondeterministic classes may dump nondeterministically. compare + # the sorted json output. this is a weaker test, but is better + # than nothing. + deterministic=0 + if ceph-dencoder type "$type" is_deterministic; then + deterministic=1 + fi + + if [ $deterministic -eq 0 ]; then + echo " sorting json output for nondeterministic object" + for f in $tmp1 $tmp2 $tmp3 $tmp4; do + sort $f | sed 's/,$//' > $f.new + mv $f.new $f + done + fi + + if ! cmp $tmp1 $tmp2; then + echo "**** $type test $n dump_json check failed ****" + echo " ceph-dencoder type $type select_test $n dump_json > $tmp1" + echo " ceph-dencoder type $type select_test $n encode decode dump_json > $tmp2" + diff $tmp1 $tmp2 + failed=$(($failed + 1)) + fi + + if ! cmp $tmp1 $tmp3; then + echo "**** $type test $n copy dump_json check failed ****" + echo " ceph-dencoder type $type select_test $n dump_json > $tmp1" + echo " ceph-dencoder type $type select_test $n copy dump_json > $tmp2" + diff $tmp1 $tmp2 + failed=$(($failed + 1)) + fi + + if ! cmp $tmp1 $tmp4; then + echo "**** $type test $n copy_ctor dump_json check failed ****" + echo " ceph-dencoder type $type select_test $n dump_json > $tmp1" + echo " ceph-dencoder type $type select_test $n copy_ctor dump_json > $tmp2" + diff $tmp1 $tmp2 + failed=$(($failed + 1)) + fi + + if [ $deterministic -ne 0 ]; then + run_in_background pids ceph-dencoder type "$type" select_test $n encode export "$tmp1" + run_in_background pids ceph-dencoder type "$type" select_test $n encode decode encode export "$tmp2" + wait_background pids + + if ! cmp $tmp1 $tmp2; then + echo "**** $type test $n binary reencode check failed ****" + echo " ceph-dencoder type $type select_test $n encode export $tmp1" + echo " ceph-dencoder type $type select_test $n encode decode encode export $tmp2" + diff <(hexdump -C $tmp1) <(hexdump -C $tmp2) + failed=$(($failed + 1)) + fi + fi + + numtests=$(($numtests + 3)) + done +done < <(ceph-dencoder list_types) + +rm -f $tmp1 $tmp2 $tmp3 $tmp4 + +if [ $failed -gt 0 ]; then + echo "FAILED $failed / $numtests tests." + exit 1 +fi +echo "passed $numtests tests." diff --git a/src/test/encoding/generate-corpus-objects.sh b/src/test/encoding/generate-corpus-objects.sh new file mode 100755 index 000000000..559ac524d --- /dev/null +++ b/src/test/encoding/generate-corpus-objects.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -ex + +BDIR=`pwd` + +p=$1 +echo path $p +test ! -d $p +mkdir $p +strings bin/ceph-osd | grep "^$p/%s__%d.%x" + +v=`git describe | cut -c 2-` +echo version $v + +echo 'binaries look ok, vstarting' +echo + +MON=3 MDS=3 OSD=5 MDS=3 MGR=2 RGW=1 ../src/vstart.sh -x -n -l --bluestore -e + +export PATH=bin:$PATH + +# do some work to generate a hopefully braod set of object instances + +echo 'starting some background work' +../qa/workunits/rados/test.sh & +../qa/workunits/rbd/test_librbd.sh & +../qa/workunits/libcephfs/test.sh & +../qa/workunits/rgw/run-s3tests.sh & +ceph-syn --syn makedirs 3 3 3 & + +echo 'waiting a bit' + +sleep 10 +echo 'triggering some recovery' + +kill -9 `cat out/osd.0.pid` +sleep 10 +ceph osd out 0 +sleep 10 +init-ceph start osd.0 +ceph osd in 0 + +sleep 5 +echo 'triggering mds work' +bin/ceph mds fail 0 + +echo 'waiting for worker to join (and ignoring errors)' +wait || true + +echo 'importing' +../src/test/encoding/import.sh $p $v ../ceph-object-corpus/archive + +for d in ../ceph-object-corpus/archive/$v/objects/* +do + echo prune $d + ../ceph-object-corpus/bin/prune.sh $d 25 +done + +echo 'done' diff --git a/src/test/encoding/identity.sh b/src/test/encoding/identity.sh new file mode 100755 index 000000000..67c803c9d --- /dev/null +++ b/src/test/encoding/identity.sh @@ -0,0 +1,32 @@ +#!/bin/sh -e + +dir=$1 + +set -e + +tmp1=`mktemp /tmp/typ-XXXXXXXXX` +tmp2=`mktemp /tmp/typ-XXXXXXXXX` + +for type in `ls $dir` +do + if ./ceph-dencoder type $type 2>/dev/null; then + echo "type $type" + for o in `ls $dir/$type`; do + f="$dir/$type/$o" + echo "\t$f" + + ./ceph-dencoder type $type import $f decode dump_json > $tmp1 + ./ceph-dencoder type $type import $f decode encode decode dump_json > $tmp2 + cmp $tmp1 $tmp2 || exit 1 + + ./ceph-dencoder type $type import $f decode encode export $tmp1 + cmp $tmp1 $f || exit 1 + done + else + echo "skip $type" + fi +done + +rm -f $tmp1 $tmp2 + +echo OK diff --git a/src/test/encoding/import-generated.sh b/src/test/encoding/import-generated.sh new file mode 100755 index 000000000..c328e6fd8 --- /dev/null +++ b/src/test/encoding/import-generated.sh @@ -0,0 +1,30 @@ +#!/bin/sh -e + +archive=$1 + +[ -d "$archive" ] || echo "usage: $0 <archive>" + +ver=`bin/ceph-dencoder version` +echo "version $ver" + +[ -d "$archive/$ver" ] || mkdir "$archive/$ver" + +tmp1=`mktemp /tmp/typ-XXXXXXXXX` + +echo "numgen\ttype" +for type in `bin/ceph-dencoder list_types`; do + + [ -d "$archive/$ver/objects/$type" ] || mkdir -p "$archive/$ver/objects/$type" + + num=`bin/ceph-dencoder type $type count_tests` + echo "$num\t$type" + max=$(($num - 1)) + for n in `seq 0 $max`; do + bin/ceph-dencoder type $type select_test $n encode export $tmp1 + md=`md5sum $tmp1 | awk '{print $1}'` + echo "\t$md" + [ -e "$archive/$ver/objects/$type/$md" ] || cp $tmp1 $archive/$ver/objects/$type/$md + done +done + +rm $tmp1 diff --git a/src/test/encoding/import.sh b/src/test/encoding/import.sh new file mode 100755 index 000000000..eea96e353 --- /dev/null +++ b/src/test/encoding/import.sh @@ -0,0 +1,23 @@ +#!/bin/sh -e + +src=$1 +ver=$2 +archive=$3 + +[ -d "$archive" ] && [ -d "$src" ] || echo "usage: $0 <srcdir> <version> <archive>" + +[ -d "$archive/$ver" ] || mkdir "$archive/$ver" + +dest_dir="$archive/$ver/objects" + +[ -d "$dest_dir" ] || mkdir "$dest_dir" + +for f in `find $src -type f` +do + n=`basename $f` + type=`echo $n | sed 's/__.*//'` + md=`md5sum $f | awk '{print $1}'` + + [ -d "$dest_dir/$type" ] || mkdir $dest_dir/$type + [ -e "$dest_dir/$type/$md" ] || cp $f $dest_dir/$type/$md +done diff --git a/src/test/encoding/readable.sh b/src/test/encoding/readable.sh new file mode 100755 index 000000000..81545852c --- /dev/null +++ b/src/test/encoding/readable.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -e + +source $(dirname $0)/../detect-build-env-vars.sh + +[ -z "$CEPH_ROOT" ] && CEPH_ROOT=.. + +dir=$CEPH_ROOT/ceph-object-corpus + +failed=0 +numtests=0 +pids="" + +if [ -x ./ceph-dencoder ]; then + CEPH_DENCODER=./ceph-dencoder +else + CEPH_DENCODER=ceph-dencoder +fi + +myversion=`$CEPH_DENCODER version` +DEBUG=0 +WAITALL_DELAY=.1 +debug() { if [ "$DEBUG" -gt 0 ]; then echo "DEBUG: $*" >&2; fi } + +test_object() { + local type=$1 + local output_file=$2 + local failed=0 + local numtests=0 + + tmp1=`mktemp /tmp/test_object_1-XXXXXXXXX` + tmp2=`mktemp /tmp/test_object_2-XXXXXXXXX` + + rm -f $output_file + if $CEPH_DENCODER type $type 2>/dev/null; then + #echo "type $type"; + echo " $vdir/objects/$type" + + # is there a fwd incompat change between $arversion and $version? + incompat="" + incompat_paths="" + sawarversion=0 + for iv in `ls $dir/archive | sort -n`; do + if [ "$iv" = "$arversion" ]; then + sawarversion=1 + fi + + if [ $sawarversion -eq 1 ] && [ -e "$dir/archive/$iv/forward_incompat/$type" ]; then + incompat="$iv" + + # Check if we'll be ignoring only specified objects, not whole type. If so, remember + # all paths for this type into variable. Assuming that this path won't contain any + # whitechars (implication of above for loop). + if [ -d "$dir/archive/$iv/forward_incompat/$type" ]; then + if [ -n "`ls $dir/archive/$iv/forward_incompat/$type/ | sort -n`" ]; then + incompat_paths="$incompat_paths $dir/archive/$iv/forward_incompat/$type" + else + echo "type $type directory empty, ignoring whole type instead of single objects" + fi; + fi + fi + + if [ "$iv" = "$version" ]; then + rm -rf $tmp1 $tmp2 + break + fi + done + + if [ -n "$incompat" ]; then + if [ -z "$incompat_paths" ]; then + echo "skipping incompat $type version $arversion, changed at $incompat < code $myversion" + rm -rf $tmp1 $tmp2 + return + else + # If we are ignoring not whole type, but objects that are in $incompat_path, + # we don't skip here, just give info. + echo "postponed skip one of incompact $type version $arversion, changed at $incompat < code $myversion" + fi; + fi + + for f in `ls $vdir/objects/$type`; do + + skip=0; + # Check if processed object $f of $type should be skipped (postponed skip) + if [ -n "$incompat_paths" ]; then + for i_path in $incompat_paths; do + # Check if $f is a symbolic link and if it's pointing to existing target + if [ -L "$i_path/$f" ]; then + echo "skipping object $f of type $type" + skip=1 + break + fi; + done; + fi; + + if [ $skip -ne 0 ]; then + continue + fi; + + $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode dump_json > $tmp1 & + pid1="$!" + $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode encode decode dump_json > $tmp2 & + pid2="$!" + #echo "\t$vdir/$type/$f" + if ! wait $pid1; then + echo "**** failed to decode $vdir/objects/$type/$f ****" + failed=$(($failed + 1)) + rm -f $tmp1 $tmp2 + continue + fi + if ! wait $pid2; then + echo "**** failed to decode+encode+decode $vdir/objects/$type/$f ****" + failed=$(($failed + 1)) + rm -f $tmp1 $tmp2 + continue + fi + + # nondeterministic classes may dump + # nondeterministically. compare the sorted json + # output. this is a weaker test, but is better than + # nothing. + if ! $CEPH_DENCODER type $type is_deterministic; then + echo " sorting json output for nondeterministic object" + for f in $tmp1 $tmp2; do + sort $f | sed 's/,$//' > $f.new + mv $f.new $f + done + fi + + if ! cmp $tmp1 $tmp2; then + echo "**** reencode of $vdir/objects/$type/$f resulted in a different dump ****" + diff $tmp1 $tmp2 + failed=$(($failed + 1)) + fi + numtests=$(($numtests + 1)) + rm -f $tmp1 $tmp2 + done + else + echo "skipping unrecognized type $type" + rm -f $tmp1 $tmp2 + fi + + echo "failed=$failed" > $output_file + echo "numtests=$numtests" >> $output_file +} + +waitall() { # PID... + ## Wait for children to exit and indicate whether all exited with 0 status. + local errors=0 + while :; do + debug "Processes remaining: $*" + for pid in "$@"; do + shift + if kill -0 "$pid" 2>/dev/null; then + debug "$pid is still alive." + set -- "$@" "$pid" + elif wait "$pid"; then + debug "$pid exited with zero exit status." + else + debug "$pid exited with non-zero exit status." + errors=$(($errors + 1)) + fi + done + [ $# -eq 0 ] && break + sleep ${WAITALL_DELAY:-1} + done + [ $errors -eq 0 ] +} + +###### +# MAIN +###### + +do_join() { + waitall $pids + pids="" + # Reading the output of jobs to compute failed & numtests + # Tests are run in parallel but sum should be done sequentialy to avoid + # races between threads + while [ "$running_jobs" -ge 0 ]; do + if [ -f $output_file.$running_jobs ]; then + read_failed=$(grep "^failed=" $output_file.$running_jobs | cut -d "=" -f 2) + read_numtests=$(grep "^numtests=" $output_file.$running_jobs | cut -d "=" -f 2) + rm -f $output_file.$running_jobs + failed=$(($failed + $read_failed)) + numtests=$(($numtests + $read_numtests)) + fi + running_jobs=$(($running_jobs - 1)) + done + running_jobs=0 +} + +# Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical +# processors +if [ `uname` == FreeBSD -o `uname` == Darwin ]; then + NPROC=`sysctl -n hw.ncpu` + max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}} +else + max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)} +fi + +output_file=`mktemp /tmp/output_file-XXXXXXXXX` +running_jobs=0 + +for arversion in `ls $dir/archive | sort -n`; do + vdir="$dir/archive/$arversion" + #echo $vdir + + if [ ! -d "$vdir/objects" ]; then + continue; + fi + + for type in `ls $vdir/objects`; do + test_object $type $output_file.$running_jobs & + pids="$pids $!" + running_jobs=$(($running_jobs + 1)) + + # Once we spawned enough jobs, let's wait them to complete + # Every spawned job have almost the same execution time so + # it's not a big deal having them not ending at the same time + if [ "$running_jobs" -eq "$max_parallel_jobs" ]; then + do_join + fi + rm -f ${output_file}* + done +done + +do_join + +if [ $failed -gt 0 ]; then + echo "FAILED $failed / $numtests tests." + exit 1 +fi + +if [ $numtests -eq 0 ]; then + echo "FAILED: no tests found to run!" + exit 1 +fi + +echo "passed $numtests tests." + |