diff options
Diffstat (limited to '')
| -rw-r--r-- | src/test/encoding.cc | 541 | ||||
| -rw-r--r-- | src/test/encoding/CMakeLists.txt | 3 | ||||
| -rwxr-xr-x | src/test/encoding/check-generated.sh | 101 | ||||
| -rwxr-xr-x | src/test/encoding/generate-corpus-objects.sh | 59 | ||||
| -rwxr-xr-x | src/test/encoding/identity.sh | 32 | ||||
| -rwxr-xr-x | src/test/encoding/import-generated.sh | 30 | ||||
| -rwxr-xr-x | src/test/encoding/import.sh | 23 | ||||
| -rwxr-xr-x | src/test/encoding/readable.sh | 241 |
8 files changed, 1030 insertions, 0 deletions
diff --git a/src/test/encoding.cc b/src/test/encoding.cc new file mode 100644 index 000000000..f18901cbd --- /dev/null +++ b/src/test/encoding.cc @@ -0,0 +1,541 @@ +#include "include/buffer.h" +#include "include/encoding.h" + +#include "gtest/gtest.h" + +using namespace std; + +template < typename T > +static void test_encode_and_decode(const T& src) +{ + bufferlist bl(1000000); + encode(src, bl); + T dst; + auto i = bl.cbegin(); + decode(dst, i); + ASSERT_EQ(src, dst) << "Encoding roundtrip changed the string: orig=" << src << ", but new=" << dst; +} + +TEST(EncodingRoundTrip, StringSimple) { + string my_str("I am the very model of a modern major general"); + test_encode_and_decode < std::string >(my_str); +} + +TEST(EncodingRoundTrip, StringEmpty) { + string my_str(""); + test_encode_and_decode < std::string >(my_str); +} + +TEST(EncodingRoundTrip, StringNewline) { + string my_str("foo bar baz\n"); + test_encode_and_decode < std::string >(my_str); +} + +template <typename Size, typename T> +static void test_encode_and_nohead_nohead(Size len, const T& src) +{ + bufferlist bl(1000000); + encode(len, bl); + encode_nohead(src, bl); + T dst; + auto i = bl.cbegin(); + decode(len, i); + decode_nohead(len, dst, i); + ASSERT_EQ(src, dst) << "Encoding roundtrip changed the string: orig=" << src << ", but new=" << dst; +} + +TEST(EncodingRoundTrip, StringNoHead) { + const string str("The quick brown fox jumps over the lazy dog"); + auto size = str.size(); + test_encode_and_nohead_nohead(static_cast<int>(size), str); + test_encode_and_nohead_nohead(static_cast<unsigned>(size), str); + test_encode_and_nohead_nohead(static_cast<uint32_t>(size), str); + test_encode_and_nohead_nohead(static_cast<__u32>(size), str); + test_encode_and_nohead_nohead(static_cast<size_t>(size), str); +} + +TEST(EncodingRoundTrip, BufferListNoHead) { + bufferlist bl; + bl.append("is this a dagger which i see before me?"); + auto size = bl.length(); + test_encode_and_nohead_nohead(static_cast<int>(size), bl); + test_encode_and_nohead_nohead(static_cast<unsigned>(size), bl); + test_encode_and_nohead_nohead(static_cast<uint32_t>(size), bl); + test_encode_and_nohead_nohead(static_cast<__u32>(size), bl); + test_encode_and_nohead_nohead(static_cast<size_t>(size), bl); +} + +typedef std::multimap < int, std::string > multimap_t; +typedef multimap_t::value_type my_val_ty; + +namespace std { +static std::ostream& operator<<(std::ostream& oss, const multimap_t &multimap) +{ + for (multimap_t::const_iterator m = multimap.begin(); + m != multimap.end(); + ++m) + { + oss << m->first << "->" << m->second << " "; + } + return oss; +} +} + +TEST(EncodingRoundTrip, Multimap) { + multimap_t multimap; + multimap.insert( my_val_ty(1, "foo") ); + multimap.insert( my_val_ty(2, "bar") ); + multimap.insert( my_val_ty(2, "baz") ); + multimap.insert( my_val_ty(3, "lucky number 3") ); + multimap.insert( my_val_ty(10000, "large number") ); + + test_encode_and_decode < multimap_t >(multimap); +} + + + +/////////////////////////////////////////////////////// +// ConstructorCounter +/////////////////////////////////////////////////////// +template <typename T> +class ConstructorCounter +{ +public: + ConstructorCounter() : data(0) + { + default_ctor++; + } + + explicit ConstructorCounter(const T& data_) + : data(data_) + { + one_arg_ctor++; + } + + ConstructorCounter(const ConstructorCounter &rhs) + : data(rhs.data) + { + copy_ctor++; + } + + ConstructorCounter &operator=(const ConstructorCounter &rhs) + { + data = rhs.data; + assigns++; + return *this; + } + + static void init(void) + { + default_ctor = 0; + one_arg_ctor = 0; + copy_ctor = 0; + assigns = 0; + } + + static int get_default_ctor(void) + { + return default_ctor; + } + + static int get_one_arg_ctor(void) + { + return one_arg_ctor; + } + + static int get_copy_ctor(void) + { + return copy_ctor; + } + + static int get_assigns(void) + { + return assigns; + } + + bool operator<(const ConstructorCounter &rhs) const + { + return data < rhs.data; + } + + bool operator==(const ConstructorCounter &rhs) const + { + return data == rhs.data; + } + + friend void decode(ConstructorCounter &s, bufferlist::const_iterator& p) + { + decode(s.data, p); + } + + friend void encode(const ConstructorCounter &s, bufferlist& p) + { + encode(s.data, p); + } + + friend ostream& operator<<(ostream &oss, const ConstructorCounter &cc) + { + oss << cc.data; + return oss; + } + + T data; +private: + static int default_ctor; + static int one_arg_ctor; + static int copy_ctor; + static int assigns; +}; + +template class ConstructorCounter <int32_t>; +template class ConstructorCounter <int16_t>; + +typedef ConstructorCounter <int32_t> my_key_t; +typedef ConstructorCounter <int16_t> my_val_t; +typedef std::multimap < my_key_t, my_val_t > multimap2_t; +typedef multimap2_t::value_type val2_ty; + +template <class T> int ConstructorCounter<T>::default_ctor = 0; +template <class T> int ConstructorCounter<T>::one_arg_ctor = 0; +template <class T> int ConstructorCounter<T>::copy_ctor = 0; +template <class T> int ConstructorCounter<T>::assigns = 0; + +static std::ostream& operator<<(std::ostream& oss, const multimap2_t &multimap) +{ + for (multimap2_t::const_iterator m = multimap.begin(); + m != multimap.end(); + ++m) + { + oss << m->first << "->" << m->second << " "; + } + return oss; +} + +TEST(EncodingRoundTrip, MultimapConstructorCounter) { + multimap2_t multimap2; + multimap2.insert( val2_ty(my_key_t(1), my_val_t(10)) ); + multimap2.insert( val2_ty(my_key_t(2), my_val_t(20)) ); + multimap2.insert( val2_ty(my_key_t(2), my_val_t(30)) ); + multimap2.insert( val2_ty(my_key_t(3), my_val_t(40)) ); + multimap2.insert( val2_ty(my_key_t(10000), my_val_t(1)) ); + + my_key_t::init(); + my_val_t::init(); + test_encode_and_decode < multimap2_t >(multimap2); + + EXPECT_EQ(my_key_t::get_default_ctor(), 5); + EXPECT_EQ(my_key_t::get_one_arg_ctor(), 0); + EXPECT_EQ(my_key_t::get_copy_ctor(), 5); + EXPECT_EQ(my_key_t::get_assigns(), 0); + + EXPECT_EQ(my_val_t::get_default_ctor(), 5); + EXPECT_EQ(my_val_t::get_one_arg_ctor(), 0); + EXPECT_EQ(my_val_t::get_copy_ctor(), 5); + EXPECT_EQ(my_val_t::get_assigns(), 0); +} + +namespace ceph { +// make sure that the legacy encode/decode methods are selected +// over the ones defined using templates. the later is likely to +// be slower, see also the definition of "WRITE_INT_DENC" in +// include/denc.h +template<> +void encode<uint64_t, denc_traits<uint64_t>>(const uint64_t&, + bufferlist&, + uint64_t f) { + static_assert(denc_traits<uint64_t>::supported, + "should support new encoder"); + static_assert(!denc_traits<uint64_t>::featured, + "should not be featured"); + ASSERT_EQ(0UL, f); + // make sure the test fails if i get called + ASSERT_TRUE(false); +} + +template<> +void encode<ceph_le64, denc_traits<ceph_le64>>(const ceph_le64&, + bufferlist&, + uint64_t f) { + static_assert(denc_traits<ceph_le64>::supported, + "should support new encoder"); + static_assert(!denc_traits<ceph_le64>::featured, + "should not be featured"); + ASSERT_EQ(0UL, f); + // make sure the test fails if i get called + ASSERT_TRUE(false); +} +} + +namespace { + // search `underlying_type` in denc.h for supported underlying types + enum class Colour : int8_t { R,G,B }; + ostream& operator<<(ostream& os, Colour c) { + switch (c) { + case Colour::R: + return os << "Colour::R"; + case Colour::G: + return os << "Colour::G"; + case Colour::B: + return os << "Colour::B"; + default: + return os << "Colour::???"; + } + } +} + +TEST(EncodingRoundTrip, Integers) { + // int types + { + uint64_t i = 42; + test_encode_and_decode(i); + } + { + int16_t i = 42; + test_encode_and_decode(i); + } + { + bool b = true; + test_encode_and_decode(b); + } + { + bool b = false; + test_encode_and_decode(b); + } + // raw encoder + { + ceph_le64 i; + i = 42; + test_encode_and_decode(i); + } + // enum + { + test_encode_and_decode(Colour::R); + // this should not build, as the size of unsigned is not the same on + // different archs, that's why denc_traits<> intentionally leaves + // `int` and `unsigned int` out of supported types. + // + // enum E { R, G, B }; + // test_encode_and_decode(R); + } +} + +const char* expected_what[] = { + "void lame_decoder(int) no longer understand old encoding version 100 < 200: Malformed input", + "void lame_decoder(int) decode past end of struct encoding: Malformed input" +}; + +void lame_decoder(int which) { + switch (which) { + case 0: + throw buffer::malformed_input(DECODE_ERR_OLDVERSION(__PRETTY_FUNCTION__, 100, 200)); + case 1: + throw buffer::malformed_input(DECODE_ERR_PAST(__PRETTY_FUNCTION__)); + } +} + +TEST(EncodingException, Macros) { + for (unsigned i = 0; i < std::size(expected_what); i++) { + try { + lame_decoder(i); + } catch (const exception& e) { + ASSERT_NE(string(e.what()).find(expected_what[i]), string::npos); + } + } +} + + +TEST(small_encoding, varint) { + uint32_t v[][4] = { + /* value, varint bytes, signed varint bytes, signed varint bytes (neg) */ + {0, 1, 1, 1}, + {1, 1, 1, 1}, + {2, 1, 1, 1}, + {31, 1, 1, 1}, + {32, 1, 1, 1}, + {0xff, 2, 2, 2}, + {0x100, 2, 2, 2}, + {0xfff, 2, 2, 2}, + {0x1000, 2, 2, 2}, + {0x2000, 2, 3, 3}, + {0x3fff, 2, 3, 3}, + {0x4000, 3, 3, 3}, + {0x4001, 3, 3, 3}, + {0x10001, 3, 3, 3}, + {0x20001, 3, 3, 3}, + {0x40001, 3, 3, 3}, + {0x80001, 3, 3, 3}, + {0x7f0001, 4, 4, 4}, + {0xff00001, 4, 5, 5}, + {0x1ff00001, 5, 5, 5}, + {0xffff0001, 5, 5, 5}, + {0xffffffff, 5, 5, 5}, + {1074790401, 5, 5, 5}, + {0, 0, 0, 0} + }; + for (unsigned i=0; v[i][1]; ++i) { + { + bufferlist bl; + { + auto app = bl.get_contiguous_appender(16, true); + denc_varint(v[i][0], app); + } + cout << std::hex << v[i][0] << "\t" << v[i][1] << "\t"; + bl.hexdump(cout, false); + cout << std::endl; + ASSERT_EQ(bl.length(), v[i][1]); + uint32_t u; + auto p = bl.begin().get_current_ptr().cbegin(); + denc_varint(u, p); + ASSERT_EQ(v[i][0], u); + } + { + bufferlist bl; + { + auto app = bl.get_contiguous_appender(16, true); + denc_signed_varint(v[i][0], app); + } + cout << std::hex << v[i][0] << "\t" << v[i][2] << "\t"; + bl.hexdump(cout, false); + cout << std::endl; + ASSERT_EQ(bl.length(), v[i][2]); + int32_t u; + auto p = bl.begin().get_current_ptr().cbegin(); + denc_signed_varint(u, p); + ASSERT_EQ((int32_t)v[i][0], u); + } + { + bufferlist bl; + int64_t x = -(int64_t)v[i][0]; + { + auto app = bl.get_contiguous_appender(16, true); + denc_signed_varint(x, app); + } + cout << std::dec << x << std::hex << "\t" << v[i][3] << "\t"; + bl.hexdump(cout, false); + cout << std::endl; + ASSERT_EQ(bl.length(), v[i][3]); + int64_t u; + auto p = bl.begin().get_current_ptr().cbegin(); + denc_signed_varint(u, p); + ASSERT_EQ(x, u); + } + } +} + +TEST(small_encoding, varint_lowz) { + uint32_t v[][4] = { + /* value, bytes encoded */ + {0, 1, 1, 1}, + {1, 1, 1, 1}, + {2, 1, 1, 1}, + {15, 1, 1, 1}, + {16, 1, 1, 1}, + {31, 1, 2, 2}, + {63, 2, 2, 2}, + {64, 1, 1, 1}, + {0xff, 2, 2, 2}, + {0x100, 1, 1, 1}, + {0x7ff, 2, 2, 2}, + {0xfff, 2, 3, 3}, + {0x1000, 1, 1, 1}, + {0x4000, 1, 1, 1}, + {0x8000, 1, 1, 1}, + {0x10000, 1, 2, 2}, + {0x20000, 2, 2, 2}, + {0x40000, 2, 2, 2}, + {0x80000, 2, 2, 2}, + {0x7f0000, 2, 2, 2}, + {0xffff0000, 4, 4, 4}, + {0xffffffff, 5, 5, 5}, + {0x41000000, 3, 4, 4}, + {0, 0, 0, 0} + }; + for (unsigned i=0; v[i][1]; ++i) { + { + bufferlist bl; + { + auto app = bl.get_contiguous_appender(16, true); + denc_varint_lowz(v[i][0], app); + } + cout << std::hex << v[i][0] << "\t" << v[i][1] << "\t"; + bl.hexdump(cout, false); + cout << std::endl; + ASSERT_EQ(bl.length(), v[i][1]); + uint32_t u; + auto p = bl.begin().get_current_ptr().cbegin(); + denc_varint_lowz(u, p); + ASSERT_EQ(v[i][0], u); + } + { + bufferlist bl; + int64_t x = v[i][0]; + { + auto app = bl.get_contiguous_appender(16, true); + denc_signed_varint_lowz(x, app); + } + cout << std::hex << x << "\t" << v[i][1] << "\t"; + bl.hexdump(cout, false); + cout << std::endl; + ASSERT_EQ(bl.length(), v[i][2]); + int64_t u; + auto p = bl.begin().get_current_ptr().cbegin(); + denc_signed_varint_lowz(u, p); + ASSERT_EQ(x, u); + } + { + bufferlist bl; + int64_t x = -(int64_t)v[i][0]; + { + auto app = bl.get_contiguous_appender(16, true); + denc_signed_varint_lowz(x, app); + } + cout << std::dec << x << "\t" << v[i][1] << "\t"; + bl.hexdump(cout, false); + cout << std::endl; + ASSERT_EQ(bl.length(), v[i][3]); + int64_t u; + auto p = bl.begin().get_current_ptr().cbegin(); + denc_signed_varint_lowz(u, p); + ASSERT_EQ(x, u); + } + } +} + +TEST(small_encoding, lba) { + uint64_t v[][2] = { + /* value, bytes encoded */ + {0, 4}, + {1, 4}, + {0xff, 4}, + {0x10000, 4}, + {0x7f0000, 4}, + {0xffff0000, 4}, + {0x0fffffff, 4}, + {0x1fffffff, 5}, + {0xffffffff, 5}, + {0x3fffffff000, 4}, + {0x7fffffff000, 5}, + {0x1fffffff0000, 4}, + {0x3fffffff0000, 5}, + {0xfffffff00000, 4}, + {0x1fffffff00000, 5}, + {0x41000000, 4}, + {0, 0} + }; + for (unsigned i=0; v[i][1]; ++i) { + bufferlist bl; + { + auto app = bl.get_contiguous_appender(16, true); + denc_lba(v[i][0], app); + } + cout << std::hex << v[i][0] << "\t" << v[i][1] << "\t"; + bl.hexdump(cout, false); + cout << std::endl; + ASSERT_EQ(bl.length(), v[i][1]); + uint64_t u; + auto p = bl.begin().get_current_ptr().cbegin(); + denc_lba(u, p); + ASSERT_EQ(v[i][0], u); + } + +} diff --git a/src/test/encoding/CMakeLists.txt b/src/test/encoding/CMakeLists.txt new file mode 100644 index 000000000..e8c57a042 --- /dev/null +++ b/src/test/encoding/CMakeLists.txt @@ -0,0 +1,3 @@ +# scripts +add_ceph_test(check-generated.sh ${CMAKE_CURRENT_SOURCE_DIR}/check-generated.sh) +add_ceph_test(readable.sh ${CMAKE_CURRENT_SOURCE_DIR}/readable.sh) diff --git a/src/test/encoding/check-generated.sh b/src/test/encoding/check-generated.sh new file mode 100755 index 000000000..2569bc1a5 --- /dev/null +++ b/src/test/encoding/check-generated.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -e + +source $(dirname $0)/../detect-build-env-vars.sh +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +dir=$1 + +tmp1=`mktemp /tmp/typ-XXXXXXXXX` +tmp2=`mktemp /tmp/typ-XXXXXXXXX` +tmp3=`mktemp /tmp/typ-XXXXXXXXX` +tmp4=`mktemp /tmp/typ-XXXXXXXXX` + +failed=0 +numtests=0 +echo "checking ceph-dencoder generated test instances..." +echo "numgen type" +while read type; do + num=`ceph-dencoder type $type count_tests` + echo "$num $type" + for n in `seq 1 1 $num 2>/dev/null`; do + + pids="" + run_in_background pids save_stdout "$tmp1" ceph-dencoder type "$type" select_test "$n" dump_json + run_in_background pids save_stdout "$tmp2" ceph-dencoder type "$type" select_test "$n" encode decode dump_json + run_in_background pids save_stdout "$tmp3" ceph-dencoder type "$type" select_test "$n" copy dump_json + run_in_background pids save_stdout "$tmp4" ceph-dencoder type "$type" select_test "$n" copy_ctor dump_json + wait_background pids + + if [ $? -ne 0 ]; then + echo "**** $type test $n encode+decode check failed ****" + echo " ceph-dencoder type $type select_test $n encode decode" + failed=$(($failed + 3)) + continue + fi + + # nondeterministic classes may dump nondeterministically. compare + # the sorted json output. this is a weaker test, but is better + # than nothing. + deterministic=0 + if ceph-dencoder type "$type" is_deterministic; then + deterministic=1 + fi + + if [ $deterministic -eq 0 ]; then + echo " sorting json output for nondeterministic object" + for f in $tmp1 $tmp2 $tmp3 $tmp4; do + sort $f | sed 's/,$//' > $f.new + mv $f.new $f + done + fi + + if ! cmp $tmp1 $tmp2; then + echo "**** $type test $n dump_json check failed ****" + echo " ceph-dencoder type $type select_test $n dump_json > $tmp1" + echo " ceph-dencoder type $type select_test $n encode decode dump_json > $tmp2" + diff $tmp1 $tmp2 + failed=$(($failed + 1)) + fi + + if ! cmp $tmp1 $tmp3; then + echo "**** $type test $n copy dump_json check failed ****" + echo " ceph-dencoder type $type select_test $n dump_json > $tmp1" + echo " ceph-dencoder type $type select_test $n copy dump_json > $tmp2" + diff $tmp1 $tmp2 + failed=$(($failed + 1)) + fi + + if ! cmp $tmp1 $tmp4; then + echo "**** $type test $n copy_ctor dump_json check failed ****" + echo " ceph-dencoder type $type select_test $n dump_json > $tmp1" + echo " ceph-dencoder type $type select_test $n copy_ctor dump_json > $tmp2" + diff $tmp1 $tmp2 + failed=$(($failed + 1)) + fi + + if [ $deterministic -ne 0 ]; then + run_in_background pids ceph-dencoder type "$type" select_test $n encode export "$tmp1" + run_in_background pids ceph-dencoder type "$type" select_test $n encode decode encode export "$tmp2" + wait_background pids + + if ! cmp $tmp1 $tmp2; then + echo "**** $type test $n binary reencode check failed ****" + echo " ceph-dencoder type $type select_test $n encode export $tmp1" + echo " ceph-dencoder type $type select_test $n encode decode encode export $tmp2" + diff <(hexdump -C $tmp1) <(hexdump -C $tmp2) + failed=$(($failed + 1)) + fi + fi + + numtests=$(($numtests + 3)) + done +done < <(ceph-dencoder list_types) + +rm -f $tmp1 $tmp2 $tmp3 $tmp4 + +if [ $failed -gt 0 ]; then + echo "FAILED $failed / $numtests tests." + exit 1 +fi +echo "passed $numtests tests." diff --git a/src/test/encoding/generate-corpus-objects.sh b/src/test/encoding/generate-corpus-objects.sh new file mode 100755 index 000000000..559ac524d --- /dev/null +++ b/src/test/encoding/generate-corpus-objects.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -ex + +BDIR=`pwd` + +p=$1 +echo path $p +test ! -d $p +mkdir $p +strings bin/ceph-osd | grep "^$p/%s__%d.%x" + +v=`git describe | cut -c 2-` +echo version $v + +echo 'binaries look ok, vstarting' +echo + +MON=3 MDS=3 OSD=5 MDS=3 MGR=2 RGW=1 ../src/vstart.sh -x -n -l --bluestore -e + +export PATH=bin:$PATH + +# do some work to generate a hopefully braod set of object instances + +echo 'starting some background work' +../qa/workunits/rados/test.sh & +../qa/workunits/rbd/test_librbd.sh & +../qa/workunits/libcephfs/test.sh & +../qa/workunits/rgw/run-s3tests.sh & +ceph-syn --syn makedirs 3 3 3 & + +echo 'waiting a bit' + +sleep 10 +echo 'triggering some recovery' + +kill -9 `cat out/osd.0.pid` +sleep 10 +ceph osd out 0 +sleep 10 +init-ceph start osd.0 +ceph osd in 0 + +sleep 5 +echo 'triggering mds work' +bin/ceph mds fail 0 + +echo 'waiting for worker to join (and ignoring errors)' +wait || true + +echo 'importing' +../src/test/encoding/import.sh $p $v ../ceph-object-corpus/archive + +for d in ../ceph-object-corpus/archive/$v/objects/* +do + echo prune $d + ../ceph-object-corpus/bin/prune.sh $d 25 +done + +echo 'done' diff --git a/src/test/encoding/identity.sh b/src/test/encoding/identity.sh new file mode 100755 index 000000000..67c803c9d --- /dev/null +++ b/src/test/encoding/identity.sh @@ -0,0 +1,32 @@ +#!/bin/sh -e + +dir=$1 + +set -e + +tmp1=`mktemp /tmp/typ-XXXXXXXXX` +tmp2=`mktemp /tmp/typ-XXXXXXXXX` + +for type in `ls $dir` +do + if ./ceph-dencoder type $type 2>/dev/null; then + echo "type $type" + for o in `ls $dir/$type`; do + f="$dir/$type/$o" + echo "\t$f" + + ./ceph-dencoder type $type import $f decode dump_json > $tmp1 + ./ceph-dencoder type $type import $f decode encode decode dump_json > $tmp2 + cmp $tmp1 $tmp2 || exit 1 + + ./ceph-dencoder type $type import $f decode encode export $tmp1 + cmp $tmp1 $f || exit 1 + done + else + echo "skip $type" + fi +done + +rm -f $tmp1 $tmp2 + +echo OK diff --git a/src/test/encoding/import-generated.sh b/src/test/encoding/import-generated.sh new file mode 100755 index 000000000..c328e6fd8 --- /dev/null +++ b/src/test/encoding/import-generated.sh @@ -0,0 +1,30 @@ +#!/bin/sh -e + +archive=$1 + +[ -d "$archive" ] || echo "usage: $0 <archive>" + +ver=`bin/ceph-dencoder version` +echo "version $ver" + +[ -d "$archive/$ver" ] || mkdir "$archive/$ver" + +tmp1=`mktemp /tmp/typ-XXXXXXXXX` + +echo "numgen\ttype" +for type in `bin/ceph-dencoder list_types`; do + + [ -d "$archive/$ver/objects/$type" ] || mkdir -p "$archive/$ver/objects/$type" + + num=`bin/ceph-dencoder type $type count_tests` + echo "$num\t$type" + max=$(($num - 1)) + for n in `seq 0 $max`; do + bin/ceph-dencoder type $type select_test $n encode export $tmp1 + md=`md5sum $tmp1 | awk '{print $1}'` + echo "\t$md" + [ -e "$archive/$ver/objects/$type/$md" ] || cp $tmp1 $archive/$ver/objects/$type/$md + done +done + +rm $tmp1 diff --git a/src/test/encoding/import.sh b/src/test/encoding/import.sh new file mode 100755 index 000000000..eea96e353 --- /dev/null +++ b/src/test/encoding/import.sh @@ -0,0 +1,23 @@ +#!/bin/sh -e + +src=$1 +ver=$2 +archive=$3 + +[ -d "$archive" ] && [ -d "$src" ] || echo "usage: $0 <srcdir> <version> <archive>" + +[ -d "$archive/$ver" ] || mkdir "$archive/$ver" + +dest_dir="$archive/$ver/objects" + +[ -d "$dest_dir" ] || mkdir "$dest_dir" + +for f in `find $src -type f` +do + n=`basename $f` + type=`echo $n | sed 's/__.*//'` + md=`md5sum $f | awk '{print $1}'` + + [ -d "$dest_dir/$type" ] || mkdir $dest_dir/$type + [ -e "$dest_dir/$type/$md" ] || cp $f $dest_dir/$type/$md +done diff --git a/src/test/encoding/readable.sh b/src/test/encoding/readable.sh new file mode 100755 index 000000000..81545852c --- /dev/null +++ b/src/test/encoding/readable.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -e + +source $(dirname $0)/../detect-build-env-vars.sh + +[ -z "$CEPH_ROOT" ] && CEPH_ROOT=.. + +dir=$CEPH_ROOT/ceph-object-corpus + +failed=0 +numtests=0 +pids="" + +if [ -x ./ceph-dencoder ]; then + CEPH_DENCODER=./ceph-dencoder +else + CEPH_DENCODER=ceph-dencoder +fi + +myversion=`$CEPH_DENCODER version` +DEBUG=0 +WAITALL_DELAY=.1 +debug() { if [ "$DEBUG" -gt 0 ]; then echo "DEBUG: $*" >&2; fi } + +test_object() { + local type=$1 + local output_file=$2 + local failed=0 + local numtests=0 + + tmp1=`mktemp /tmp/test_object_1-XXXXXXXXX` + tmp2=`mktemp /tmp/test_object_2-XXXXXXXXX` + + rm -f $output_file + if $CEPH_DENCODER type $type 2>/dev/null; then + #echo "type $type"; + echo " $vdir/objects/$type" + + # is there a fwd incompat change between $arversion and $version? + incompat="" + incompat_paths="" + sawarversion=0 + for iv in `ls $dir/archive | sort -n`; do + if [ "$iv" = "$arversion" ]; then + sawarversion=1 + fi + + if [ $sawarversion -eq 1 ] && [ -e "$dir/archive/$iv/forward_incompat/$type" ]; then + incompat="$iv" + + # Check if we'll be ignoring only specified objects, not whole type. If so, remember + # all paths for this type into variable. Assuming that this path won't contain any + # whitechars (implication of above for loop). + if [ -d "$dir/archive/$iv/forward_incompat/$type" ]; then + if [ -n "`ls $dir/archive/$iv/forward_incompat/$type/ | sort -n`" ]; then + incompat_paths="$incompat_paths $dir/archive/$iv/forward_incompat/$type" + else + echo "type $type directory empty, ignoring whole type instead of single objects" + fi; + fi + fi + + if [ "$iv" = "$version" ]; then + rm -rf $tmp1 $tmp2 + break + fi + done + + if [ -n "$incompat" ]; then + if [ -z "$incompat_paths" ]; then + echo "skipping incompat $type version $arversion, changed at $incompat < code $myversion" + rm -rf $tmp1 $tmp2 + return + else + # If we are ignoring not whole type, but objects that are in $incompat_path, + # we don't skip here, just give info. + echo "postponed skip one of incompact $type version $arversion, changed at $incompat < code $myversion" + fi; + fi + + for f in `ls $vdir/objects/$type`; do + + skip=0; + # Check if processed object $f of $type should be skipped (postponed skip) + if [ -n "$incompat_paths" ]; then + for i_path in $incompat_paths; do + # Check if $f is a symbolic link and if it's pointing to existing target + if [ -L "$i_path/$f" ]; then + echo "skipping object $f of type $type" + skip=1 + break + fi; + done; + fi; + + if [ $skip -ne 0 ]; then + continue + fi; + + $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode dump_json > $tmp1 & + pid1="$!" + $CEPH_DENCODER type $type import $vdir/objects/$type/$f decode encode decode dump_json > $tmp2 & + pid2="$!" + #echo "\t$vdir/$type/$f" + if ! wait $pid1; then + echo "**** failed to decode $vdir/objects/$type/$f ****" + failed=$(($failed + 1)) + rm -f $tmp1 $tmp2 + continue + fi + if ! wait $pid2; then + echo "**** failed to decode+encode+decode $vdir/objects/$type/$f ****" + failed=$(($failed + 1)) + rm -f $tmp1 $tmp2 + continue + fi + + # nondeterministic classes may dump + # nondeterministically. compare the sorted json + # output. this is a weaker test, but is better than + # nothing. + if ! $CEPH_DENCODER type $type is_deterministic; then + echo " sorting json output for nondeterministic object" + for f in $tmp1 $tmp2; do + sort $f | sed 's/,$//' > $f.new + mv $f.new $f + done + fi + + if ! cmp $tmp1 $tmp2; then + echo "**** reencode of $vdir/objects/$type/$f resulted in a different dump ****" + diff $tmp1 $tmp2 + failed=$(($failed + 1)) + fi + numtests=$(($numtests + 1)) + rm -f $tmp1 $tmp2 + done + else + echo "skipping unrecognized type $type" + rm -f $tmp1 $tmp2 + fi + + echo "failed=$failed" > $output_file + echo "numtests=$numtests" >> $output_file +} + +waitall() { # PID... + ## Wait for children to exit and indicate whether all exited with 0 status. + local errors=0 + while :; do + debug "Processes remaining: $*" + for pid in "$@"; do + shift + if kill -0 "$pid" 2>/dev/null; then + debug "$pid is still alive." + set -- "$@" "$pid" + elif wait "$pid"; then + debug "$pid exited with zero exit status." + else + debug "$pid exited with non-zero exit status." + errors=$(($errors + 1)) + fi + done + [ $# -eq 0 ] && break + sleep ${WAITALL_DELAY:-1} + done + [ $errors -eq 0 ] +} + +###### +# MAIN +###### + +do_join() { + waitall $pids + pids="" + # Reading the output of jobs to compute failed & numtests + # Tests are run in parallel but sum should be done sequentialy to avoid + # races between threads + while [ "$running_jobs" -ge 0 ]; do + if [ -f $output_file.$running_jobs ]; then + read_failed=$(grep "^failed=" $output_file.$running_jobs | cut -d "=" -f 2) + read_numtests=$(grep "^numtests=" $output_file.$running_jobs | cut -d "=" -f 2) + rm -f $output_file.$running_jobs + failed=$(($failed + $read_failed)) + numtests=$(($numtests + $read_numtests)) + fi + running_jobs=$(($running_jobs - 1)) + done + running_jobs=0 +} + +# Using $MAX_PARALLEL_JOBS jobs if defined, unless the number of logical +# processors +if [ `uname` == FreeBSD -o `uname` == Darwin ]; then + NPROC=`sysctl -n hw.ncpu` + max_parallel_jobs=${MAX_PARALLEL_JOBS:-${NPROC}} +else + max_parallel_jobs=${MAX_PARALLEL_JOBS:-$(nproc)} +fi + +output_file=`mktemp /tmp/output_file-XXXXXXXXX` +running_jobs=0 + +for arversion in `ls $dir/archive | sort -n`; do + vdir="$dir/archive/$arversion" + #echo $vdir + + if [ ! -d "$vdir/objects" ]; then + continue; + fi + + for type in `ls $vdir/objects`; do + test_object $type $output_file.$running_jobs & + pids="$pids $!" + running_jobs=$(($running_jobs + 1)) + + # Once we spawned enough jobs, let's wait them to complete + # Every spawned job have almost the same execution time so + # it's not a big deal having them not ending at the same time + if [ "$running_jobs" -eq "$max_parallel_jobs" ]; then + do_join + fi + rm -f ${output_file}* + done +done + +do_join + +if [ $failed -gt 0 ]; then + echo "FAILED $failed / $numtests tests." + exit 1 +fi + +if [ $numtests -eq 0 ]; then + echo "FAILED: no tests found to run!" + exit 1 +fi + +echo "passed $numtests tests." + |