From e6918187568dbd01842d8d1d2c808ce16a894239 Mon Sep 17 00:00:00 2001
From: Daniel Baumann <daniel.baumann@progress-linux.org>
Date: Sun, 21 Apr 2024 13:54:28 +0200
Subject: Adding upstream version 18.2.2.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
---
 src/test/encoding/CMakeLists.txt             |   3 +
 src/test/encoding/check-generated.sh         | 101 +++++++++++
 src/test/encoding/generate-corpus-objects.sh |  59 +++++++
 src/test/encoding/identity.sh                |  32 ++++
 src/test/encoding/import-generated.sh        |  30 ++++
 src/test/encoding/import.sh                  |  23 +++
 src/test/encoding/readable.sh                | 241 +++++++++++++++++++++++++++
 7 files changed, 489 insertions(+)
 create mode 100644 src/test/encoding/CMakeLists.txt
 create mode 100755 src/test/encoding/check-generated.sh
 create mode 100755 src/test/encoding/generate-corpus-objects.sh
 create mode 100755 src/test/encoding/identity.sh
 create mode 100755 src/test/encoding/import-generated.sh
 create mode 100755 src/test/encoding/import.sh
 create mode 100755 src/test/encoding/readable.sh

(limited to 'src/test/encoding')

diff --git a/src/test/encoding/CMakeLists.txt b/src/test/encoding/CMakeLists.txt
new file mode 100644
index 000000000..e8c57a042
--- /dev/null
+++ b/src/test/encoding/CMakeLists.txt
@@ -0,0 +1,3 @@
+# scripts
+add_ceph_test(check-generated.sh ${CMAKE_CURRENT_SOURCE_DIR}/check-generated.sh)
+add_ceph_test(readable.sh ${CMAKE_CURRENT_SOURCE_DIR}/readable.sh)
diff --git a/src/test/encoding/check-generated.sh b/src/test/encoding/check-generated.sh
new file mode 100755
index 000000000..2569bc1a5
--- /dev/null
+++ b/src/test/encoding/check-generated.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+set -e
+
+source $(dirname $0)/../detect-build-env-vars.sh
+source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
+
+dir=$1
+
+tmp1=`mktemp /tmp/typ-XXXXXXXXX`
+tmp2=`mktemp /tmp/typ-XXXXXXXXX`
+tmp3=`mktemp /tmp/typ-XXXXXXXXX`
+tmp4=`mktemp /tmp/typ-XXXXXXXXX`
+
+failed=0
+numtests=0
+echo "checking ceph-dencoder generated test instances..."
+echo "numgen type"
+while read type; do
+    num=`ceph-dencoder type $type count_tests`
+    echo "$num $type"
+    for n in `seq 1 1 $num 2>/dev/null`; do
+
+	pids=""
+	run_in_background pids save_stdout "$tmp1" ceph-dencoder type "$type" select_test "$n" dump_json
+	run_in_background pids save_stdout "$tmp2" ceph-dencoder type "$type" select_test "$n" encode decode dump_json
+	run_in_background pids save_stdout "$tmp3" ceph-dencoder type "$type" select_test "$n" copy dump_json
+	run_in_background pids save_stdout "$tmp4" ceph-dencoder type "$type" select_test "$n" copy_ctor dump_json
+	wait_background pids
+
+	if [ $? -ne 0 ]; then
+	    echo "**** $type test $n encode+decode check failed ****"
+	    echo "   ceph-dencoder type $type select_test $n encode decode"
+	    failed=$(($failed + 3))
+	    continue
+	fi
+
+	# nondeterministic classes may dump nondeterministically.  compare
+	# the sorted json output.  this is a weaker test, but is better
+	# than nothing.
+	deterministic=0
+	if ceph-dencoder type "$type" is_deterministic; then
+	    deterministic=1
+	fi
+
+	if [ $deterministic -eq 0 ]; then
+	    echo "  sorting json output for nondeterministic object"
+	    for f in $tmp1 $tmp2 $tmp3 $tmp4; do
+		sort $f | sed 's/,$//' > $f.new
+		mv $f.new $f
+	    done
+	fi
+
+	if ! cmp $tmp1 $tmp2; then
+	    echo "**** $type test $n dump_json check failed ****"
+	    echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
+	    echo "   ceph-dencoder type $type select_test $n encode decode dump_json > $tmp2"
+	    diff $tmp1 $tmp2
+	    failed=$(($failed + 1))
+	fi
+
+	if ! cmp $tmp1 $tmp3; then
+	    echo "**** $type test $n copy dump_json check failed ****"
+	    echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
+	    echo "   ceph-dencoder type $type select_test $n copy dump_json > $tmp2"
+	    diff $tmp1 $tmp2
+	    failed=$(($failed + 1))
+	fi
+
+	if ! cmp $tmp1 $tmp4; then
+	    echo "**** $type test $n copy_ctor dump_json check failed ****"
+	    echo "   ceph-dencoder type $type select_test $n dump_json > $tmp1"
+	    echo "   ceph-dencoder type $type select_test $n copy_ctor dump_json > $tmp2"
+	    diff $tmp1 $tmp2
+	    failed=$(($failed + 1))
+	fi
+
+	if [ $deterministic -ne 0 ]; then
+	    run_in_background pids ceph-dencoder type "$type" select_test $n encode export "$tmp1"
+	    run_in_background pids ceph-dencoder type "$type" select_test $n encode decode encode export "$tmp2"
+	    wait_background pids
+
+	    if ! cmp $tmp1 $tmp2; then
+		echo "**** $type test $n binary reencode check failed ****"
+		echo "   ceph-dencoder type $type select_test $n encode export $tmp1"
+		echo "   ceph-dencoder type $type select_test $n encode decode encode export $tmp2"
+		diff <(hexdump -C $tmp1) <(hexdump -C $tmp2)
+		failed=$(($failed + 1))
+	    fi
+	fi
+
+	numtests=$(($numtests + 3))
+    done
+done < <(ceph-dencoder list_types)
+
+rm -f $tmp1 $tmp2 $tmp3 $tmp4
+
+if [ $failed -gt 0 ]; then
+    echo "FAILED $failed / $numtests tests."
+    exit 1
+fi
+echo "passed $numtests tests."
diff --git a/src/test/encoding/generate-corpus-objects.sh b/src/test/encoding/generate-corpus-objects.sh
new file mode 100755
index 000000000..559ac524d
--- /dev/null
+++ b/src/test/encoding/generate-corpus-objects.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -ex
+
+BDIR=`pwd`
+
+p=$1
+echo path $p
+test ! -d $p
+mkdir $p
+strings bin/ceph-osd | grep "^$p/%s__%d.%x"
+
+v=`git describe | cut -c 2-`
+echo version $v
+
+echo 'binaries look ok, vstarting'
+echo
+
+MON=3 MDS=3 OSD=5 MDS=3 MGR=2 RGW=1 ../src/vstart.sh -x -n -l --bluestore -e
+
+export PATH=bin:$PATH
+
+# do some work to generate a hopefully braod set of object instances
+
+echo 'starting some background work'
+../qa/workunits/rados/test.sh &
+../qa/workunits/rbd/test_librbd.sh &
+../qa/workunits/libcephfs/test.sh &
+../qa/workunits/rgw/run-s3tests.sh &
+ceph-syn --syn makedirs 3 3 3 &
+
+echo 'waiting a bit'
+
+sleep 10
+echo 'triggering some recovery'
+
+kill -9 `cat out/osd.0.pid`
+sleep 10
+ceph osd out 0
+sleep 10
+init-ceph start osd.0
+ceph osd in 0
+
+sleep 5
+echo 'triggering mds work'
+bin/ceph mds fail 0
+
+echo 'waiting for worker to join (and ignoring errors)'
+wait || true
+
+echo 'importing'
+../src/test/encoding/import.sh $p $v ../ceph-object-corpus/archive
+
+for d in ../ceph-object-corpus/archive/$v/objects/*
+do
+    echo prune $d
+    ../ceph-object-corpus/bin/prune.sh $d 25
+done
+
+echo 'done'
diff --git a/src/test/encoding/identity.sh b/src/test/encoding/identity.sh
new file mode 100755
index 000000000..67c803c9d
--- /dev/null
+++ b/src/test/encoding/identity.sh
@@ -0,0 +1,32 @@
+#!/bin/sh -e
+
+dir=$1
+
+set -e
+
+tmp1=`mktemp /tmp/typ-XXXXXXXXX`
+tmp2=`mktemp /tmp/typ-XXXXXXXXX`
+
+for type in `ls $dir`
+do
+    if ./ceph-dencoder type $type 2>/dev/null; then
+        echo "type $type"
+	for o in `ls $dir/$type`; do
+	    f="$dir/$type/$o"
+	    echo "\t$f"
+
+            ./ceph-dencoder type $type import $f decode dump_json > $tmp1
+            ./ceph-dencoder type $type import $f decode encode decode dump_json > $tmp2
+            cmp $tmp1 $tmp2 || exit 1
+
+            ./ceph-dencoder type $type import $f decode encode export $tmp1
+            cmp $tmp1 $f || exit 1
+	done
+    else
+        echo "skip $type"
+    fi
+done
+
+rm -f $tmp1 $tmp2
+
+echo OK
diff --git a/src/test/encoding/import-generated.sh b/src/test/encoding/import-generated.sh
new file mode 100755
index 000000000..c328e6fd8
--- /dev/null
+++ b/src/test/encoding/import-generated.sh
@@ -0,0 +1,30 @@
+#!/bin/sh -e
+
+archive=$1
+
+[ -d "$archive" ] || echo "usage: $0 <archive>"
+
+ver=`bin/ceph-dencoder version`
+echo "version $ver"
+
+[ -d "$archive/$ver" ] || mkdir "$archive/$ver"
+
+tmp1=`mktemp /tmp/typ-XXXXXXXXX`
+
+echo "numgen\ttype"
+for type in `bin/ceph-dencoder list_types`; do
+
+    [ -d "$archive/$ver/objects/$type" ] || mkdir -p "$archive/$ver/objects/$type"
+
+    num=`bin/ceph-dencoder type $type count_tests`
+    echo "$num\t$type"
+    max=$(($num - 1))
+    for n in `seq 0 $max`; do
+	bin/ceph-dencoder type $type select_test $n encode export $tmp1
+	md=`md5sum $tmp1 | awk '{print $1}'`
+	echo "\t$md"
+	[ -e "$archive/$ver/objects/$type/$md" ] || cp $tmp1 $archive/$ver/objects/$type/$md
+    done
+done
+
+rm $tmp1
diff --git a/src/test/encoding/import.sh b/src/test/encoding/import.sh
new file mode 100755
index 000000000..eea96e353
--- /dev/null
+++ b/src/test/encoding/import.sh
@@ -0,0 +1,23 @@
+#!/bin/sh -e
+
+src=$1
+ver=$2
+archive=$3
+
+[ -d "$archive" ] && [ -d "$src" ] || echo "usage: $0 <srcdir> <version> <archive>"
+
+[ -d "$archive/$ver" ] || mkdir "$archive/$ver"
+
+dest_dir="$archive/$ver/objects"
+
+[ -d "$dest_dir" ] || mkdir "$dest_dir"
+
+for f in `find $src -type f`
+do
+    n=`basename $f`
+    type=`echo $n | sed 's/__.*//'`
+    md=`md5sum $f | awk '{print $1}'`
+
+    [ -d "$dest_dir/$type" ] || mkdir $dest_dir/$type
+    [ -e "$dest_dir/$type/$md" ] || cp $f $dest_dir/$type/$md
+done
diff --git a/src/test/encoding/readable.sh b/src/test/encoding/readable.sh
new file mode 100755
index 000000000..81545852c
--- /dev/null
+++ b/src/test/encoding/readable.sh
@@ -0,0 +1,241 @@
+#!/usr/bin/env bash
+set -e
+
+source $(dirname $0)/../detect-build-env-vars.sh
+
+[ -z "$CEPH_ROOT" ] && CEPH_ROOT=..
+
+dir=$CEPH_ROOT/ceph-object-corpus
+
+failed=0
+numtests=0
+pids=""
+
+if [ -x ./ceph-dencoder ]; then
+  CEPH_DENCODER=./ceph-dencoder
+else
+  CEPH_DENCODER=ceph-dencoder
+fi
+
+myversion=`$CEPH_DENCODER version`
+DEBUG=0
+WAITALL_DELAY=.1
+debug() { if [ "$DEBUG" -gt 0 ]; then echo "DEBUG: $*" >&2; fi }
+
+test_object() {
+    local type=$1
+    local output_file=$2
+    local failed=0
+    local numtests=0
+
+    tmp1=`mktemp /tmp/test_object_1-XXXXXXXXX`
+    tmp2=`mktemp /tmp/test_object_2-XXXXXXXXX`
+
+    rm -f $output_file
+    if $CEPH_DENCODER type $type 2>/dev/null; then
+      #echo "type $type";
+      echo "        $vdir/objects/$type"
+
+      # is there a fwd incompat change between $arversion and $version?
+      incompat=""
+      incompat_paths=""
+      sawarversion=0
+      for iv in `ls $dir/archive | sort -n`; do
+        if [ "$iv" = "$arversion" ]; then
+          sawarversion=1
+        fi
+
+        if [ $sawarversion -eq 1 ] && [ -e "$dir/archive/$iv/forward_incompat/$type" ]; then
+          incompat="$iv"
+
+          # Check if we'll be ignoring only specified objects, not whole type. If so, remember
+          # all paths for this type into variable. Assuming that this path won't contain any
+          # whitechars (implication of above for loop).
+          if [ -d "$dir/archive/$iv/forward_incompat/$type" ]; then
+            if [ -n "`ls $dir/archive/$iv/forward_incompat/$type/ | sort -n`" ]; then
+              incompat_paths="$incompat_paths $dir/archive/$iv/forward_incompat/$type"
+            else
+              echo "type $type directory empty, ignoring whole type instead of single objects"
+            fi;
+          fi
+        fi
+
+        if [ "$iv" = "$version" ]; then
+          rm -rf $tmp1 $tmp2
+          break
+        fi
+      done
+
+      if [ -n "$incompat" ]; then
+        if [ -z "$incompat_paths" ]; then
+          echo "skipping incompat $type version $arversion, changed at $incompat < code $myversion"
+          rm -rf $tmp1 $tmp2
+	  return
+        else
+          # If we are ignoring not whole type, but objects that are in $incompat_path,
+          # we don't skip here, just give info.
+          echo "postponed skip one of incompact $type version $arversion, changed at $incompat < code $myversion"
+        fi;
+      fi
+
+      for f in `ls $vdir/objects/$type`; do
+
+        skip=0;
+        # Check if processed object $f of $type should be skipped (postponed skip)
+        if [ -n "$incompat_paths" ]; then
+            for i_path in $incompat_paths; do
+              # Check if $f is a symbolic link and if it's pointing to existing target
+              if [ -L "$i_path/$f" ]; then
+                echo "skipping object $f of type $type"
+                skip=1
+                break
+              fi;
+            done;
+        fi;
+
+        if [ $skip -ne 0 ]; then
+          continue
+        fi;
+
+        $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode dump_json > $tmp1 &
+        pid1="$!"
+        $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode encode decode dump_json > $tmp2 &
+        pid2="$!"
+        #echo "\t$vdir/$type/$f"
+        if ! wait $pid1; then
+          echo "**** failed to decode $vdir/objects/$type/$f ****"
+          failed=$(($failed + 1))
+          rm -f $tmp1 $tmp2
+          continue      
+        fi
+        if ! wait $pid2; then
+          echo "**** failed to decode+encode+decode $vdir/objects/$type/$f ****"
+          failed=$(($failed + 1))
+          rm -f $tmp1 $tmp2
+          continue
+        fi
+
+        # nondeterministic classes may dump
+        # nondeterministically.  compare the sorted json
+        # output.  this is a weaker test, but is better than
+        # nothing.
+        if ! $CEPH_DENCODER type $type is_deterministic; then
+          echo "  sorting json output for nondeterministic object"
+          for f in $tmp1 $tmp2; do
+            sort $f | sed 's/,$//' > $f.new
+            mv $f.new $f
+          done
+        fi
+
+        if ! cmp $tmp1 $tmp2; then
+          echo "**** reencode of $vdir/objects/$type/$f resulted in a different dump ****"
+          diff $tmp1 $tmp2
+          failed=$(($failed + 1))
+        fi
+        numtests=$(($numtests + 1))
+	rm -f $tmp1 $tmp2
+      done
+    else
+      echo "skipping unrecognized type $type"
+      rm -f $tmp1 $tmp2
+    fi
+
+    echo "failed=$failed" > $output_file
+    echo "numtests=$numtests" >> $output_file
+}
+
+waitall() { # PID...
+   ## Wait for children to exit and indicate whether all exited with 0 status.
+   local errors=0
+   while :; do
+     debug "Processes remaining: $*"
+     for pid in "$@"; do
+       shift
+       if kill -0 "$pid" 2>/dev/null; then
+         debug "$pid is still alive."
+         set -- "$@" "$pid"
+       elif wait "$pid"; then
+         debug "$pid exited with zero exit status."
+       else
+         debug "$pid exited with non-zero exit status."
+         errors=$(($errors + 1))
+       fi
+     done
+     [ $# -eq 0 ] && break
+     sleep ${WAITALL_DELAY:-1}
+    done
+   [ $errors -eq 0 ]
+}
+
+######
+# MAIN
+######
+
+do_join() {
+        waitall $pids
+        pids=""
+        # Reading the output of jobs to compute failed & numtests
+        # Tests are run in parallel but sum should be done sequentialy to avoid
+        # races between threads
+        while [ "$running_jobs" -ge 0 ]; do
+            if [ -f $output_file.$running_jobs ]; then
+                read_failed=$(grep "^failed=" $output_file.$running_jobs | cut -d "=" -f 2)
+                read_numtests=$(grep "^numtests=" $output_file.$running_jobs | cut -d "=" -f 2)
+                rm -f $output_file.$running_jobs
+                failed=$(($failed + $read_failed))
+                numtests=$(($numtests + $read_numtests))
+            fi
+            running_jobs=$(($running_jobs - 1))
+        done
+        running_jobs=0
+}
+
+# Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical
+# processors
+if [ `uname` == FreeBSD -o `uname` == Darwin ]; then
+  NPROC=`sysctl -n hw.ncpu`
+  max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}}
+else
+  max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)}
+fi
+
+output_file=`mktemp /tmp/output_file-XXXXXXXXX`
+running_jobs=0
+
+for arversion in `ls $dir/archive | sort -n`; do
+  vdir="$dir/archive/$arversion"
+  #echo $vdir
+
+  if [ ! -d "$vdir/objects" ]; then
+    continue;
+  fi
+
+  for type in `ls $vdir/objects`; do
+    test_object $type $output_file.$running_jobs &
+    pids="$pids $!"
+    running_jobs=$(($running_jobs + 1))
+
+    # Once we spawned enough jobs, let's wait them to complete
+    # Every spawned job have almost the same execution time so
+    # it's not a big deal having them not ending at the same time
+    if [ "$running_jobs" -eq "$max_parallel_jobs" ]; then
+	do_join
+    fi
+    rm -f ${output_file}*
+  done
+done
+
+do_join
+
+if [ $failed -gt 0 ]; then
+  echo "FAILED $failed / $numtests tests."
+  exit 1
+fi
+
+if [ $numtests -eq 0 ]; then
+  echo "FAILED: no tests found to run!"
+  exit 1
+fi
+
+echo "passed $numtests tests."
+
-- 
cgit v1.2.3