From 8daa83a594a2e98f39d764422bfbdbc62c9efd44 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 19:20:00 +0200 Subject: Adding upstream version 2:4.20.0+dfsg. Signed-off-by: Daniel Baumann --- .../INTEGRATION/database/basics.001.attach.sh | 48 ++++++ .../INTEGRATION/database/basics.002.attach.sh | 116 ++++++++++++++ .../INTEGRATION/database/basics.003.detach.sh | 166 +++++++++++++++++++ ctdb/tests/INTEGRATION/database/basics.004.wipe.sh | 56 +++++++ .../database/basics.010.backup_restore.sh | 97 +++++++++++ ctdb/tests/INTEGRATION/database/fetch.001.ring.sh | 34 ++++ .../INTEGRATION/database/fetch.002.ring-hotkeys.sh | 161 +++++++++++++++++++ .../INTEGRATION/database/readonly.001.basic.sh | 178 +++++++++++++++++++++ .../INTEGRATION/database/recovery.001.volatile.sh | 118 ++++++++++++++ .../INTEGRATION/database/recovery.002.large.sh | 106 ++++++++++++ .../database/recovery.003.no_resurrect.sh | 63 ++++++++ .../database/recovery.010.persistent.sh | 103 ++++++++++++ .../INTEGRATION/database/recovery.011.continue.sh | 73 +++++++++ ctdb/tests/INTEGRATION/database/scripts/local.bash | 116 ++++++++++++++ .../INTEGRATION/database/transaction.001.ptrans.sh | 110 +++++++++++++ .../INTEGRATION/database/transaction.002.loop.sh | 28 ++++ .../database/transaction.003.loop_recovery.sh | 50 ++++++ .../database/transaction.004.update_record.sh | 80 +++++++++ .../database/transaction.010.loop_recovery.sh | 51 ++++++ .../tests/INTEGRATION/database/traverse.001.one.sh | 116 ++++++++++++++ .../INTEGRATION/database/traverse.002.many.sh | 52 ++++++ ctdb/tests/INTEGRATION/database/vacuum.001.fast.sh | 159 ++++++++++++++++++ ctdb/tests/INTEGRATION/database/vacuum.002.full.sh | 96 +++++++++++ .../INTEGRATION/database/vacuum.003.recreate.sh | 139 ++++++++++++++++ .../INTEGRATION/database/vacuum.030.locked.sh | 102 ++++++++++++ .../INTEGRATION/database/vacuum.031.locked.sh | 114 +++++++++++++ .../INTEGRATION/database/vacuum.032.locked.sh | 102 ++++++++++++ .../INTEGRATION/database/vacuum.033.locked.sh | 117 ++++++++++++++ .../INTEGRATION/database/vacuum.034.locked.sh | 129 +++++++++++++++ 29 files changed, 2880 insertions(+) create mode 100755 ctdb/tests/INTEGRATION/database/basics.001.attach.sh create mode 100755 ctdb/tests/INTEGRATION/database/basics.002.attach.sh create mode 100755 ctdb/tests/INTEGRATION/database/basics.003.detach.sh create mode 100755 ctdb/tests/INTEGRATION/database/basics.004.wipe.sh create mode 100755 ctdb/tests/INTEGRATION/database/basics.010.backup_restore.sh create mode 100755 ctdb/tests/INTEGRATION/database/fetch.001.ring.sh create mode 100755 ctdb/tests/INTEGRATION/database/fetch.002.ring-hotkeys.sh create mode 100755 ctdb/tests/INTEGRATION/database/readonly.001.basic.sh create mode 100755 ctdb/tests/INTEGRATION/database/recovery.001.volatile.sh create mode 100755 ctdb/tests/INTEGRATION/database/recovery.002.large.sh create mode 100755 ctdb/tests/INTEGRATION/database/recovery.003.no_resurrect.sh create mode 100755 ctdb/tests/INTEGRATION/database/recovery.010.persistent.sh create mode 100755 ctdb/tests/INTEGRATION/database/recovery.011.continue.sh create mode 100644 ctdb/tests/INTEGRATION/database/scripts/local.bash create mode 100755 ctdb/tests/INTEGRATION/database/transaction.001.ptrans.sh create mode 100755 ctdb/tests/INTEGRATION/database/transaction.002.loop.sh create mode 100755 ctdb/tests/INTEGRATION/database/transaction.003.loop_recovery.sh create mode 100755 ctdb/tests/INTEGRATION/database/transaction.004.update_record.sh create mode 100755 ctdb/tests/INTEGRATION/database/transaction.010.loop_recovery.sh create mode 100755 ctdb/tests/INTEGRATION/database/traverse.001.one.sh create mode 100755 ctdb/tests/INTEGRATION/database/traverse.002.many.sh create mode 100755 ctdb/tests/INTEGRATION/database/vacuum.001.fast.sh create mode 100755 ctdb/tests/INTEGRATION/database/vacuum.002.full.sh create mode 100755 ctdb/tests/INTEGRATION/database/vacuum.003.recreate.sh create mode 100755 ctdb/tests/INTEGRATION/database/vacuum.030.locked.sh create mode 100755 ctdb/tests/INTEGRATION/database/vacuum.031.locked.sh create mode 100755 ctdb/tests/INTEGRATION/database/vacuum.032.locked.sh create mode 100755 ctdb/tests/INTEGRATION/database/vacuum.033.locked.sh create mode 100755 ctdb/tests/INTEGRATION/database/vacuum.034.locked.sh (limited to 'ctdb/tests/INTEGRATION/database') diff --git a/ctdb/tests/INTEGRATION/database/basics.001.attach.sh b/ctdb/tests/INTEGRATION/database/basics.001.attach.sh new file mode 100755 index 0000000..1fbffc5 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/basics.001.attach.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Verify that 'ctdb getdbmap' operates as expected + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +select_test_node + +# test_node set by select_test_node() above +# shellcheck disable=SC2154 +ctdb_onnode -v "$test_node" getdbmap + +dbid='dbid:0x[[:xdigit:]]+' +name='name:[^[:space:]]+' +path='path:[^[:space:]]+' +opts='( (PERSISTENT|STICKY|READONLY|REPLICATED|UNHEALTHY))*' +line="${dbid} ${name} ${path}${opts}" +dbmap_pattern="^(Number of databases:[[:digit:]]+|${line})\$" + +# outfile set by ctdb_onnode() above +# shellcheck disable=SC2154 +num_db_init=$(sed -n -e '1s/.*://p' "$outfile") + +sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" + +for i in $(seq 1 5) ; do + f="attach_test_${i}.tdb" + echo "Creating test database: $f" + ctdb_onnode "$test_node" "attach ${f}" + + ctdb_onnode "$test_node" getdbmap + sanity_check_output $((num_db_init + 1)) "$dbmap_pattern" + num=$(sed -n -e '1s/^.*://p' "$outfile") + if [ "$num" = $((num_db_init + i)) ] ; then + echo "OK: correct number of additional databases" + else + ctdb_test_fail "BAD: no additional database" + fi + if awk '{print $2}' "$outfile" | grep -Fqx "name:$f" ; then + echo "OK: getdbmap knows about \"$f\"" + else + ctdb_test_fail "BAD: getdbmap does not know about \"$f\"" + fi +done diff --git a/ctdb/tests/INTEGRATION/database/basics.002.attach.sh b/ctdb/tests/INTEGRATION/database/basics.002.attach.sh new file mode 100755 index 0000000..6a5c812 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/basics.002.attach.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +# Verify that databases are attached a node joins the cluster: +# 1. Shut down CTDB on one node +# 2. Attach test databases +# 3. Check that databases are attached on all up nodes +# 4. Start CTDB on the node where it is shut down +# 5. Verify that the test databases are attached on this node +# 6. Restart one of the nodes +# 7. Verify that the test databases are attached on this node + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +###################################################################### + +try_command_on_node 0 "$CTDB listnodes -X | wc -l" +numnodes="$out" +lastnode=$(( numnodes - 1 )) + +###################################################################### + +# Confirm that the database is attached with appropriate flags +check_db_once () +{ + local pnn="$1" + local db="$2" + + try_command_on_node "$pnn" $CTDB getdbmap + if grep -qF "name:${db}" "$outfile" >/dev/null ; then + return 0 + else + return 1 + fi +} + +check_db () +{ + local pnn="$1" + local db="$2" + local flag="$3" + + local flags + + echo "Waiting until database ${db} is attached on node ${pnn}" + wait_until 10 check_db_once "$pnn" "$db" + + flags=$(awk -v db="$db" '$2 == "name:" db {print $4}' "$outfile") + if [ "$flags" = "$flag" ]; then + echo "GOOD: db ${db} attached on node ${pnn} with flag $flag" + else + echo "BAD: db ${db} attached on node ${pnn} with wrong flag" + cat "$outfile" + exit 1 + fi +} + +###################################################################### + +testdb1="test_volatile.tdb" +testdb2="test_persistent.tdb" +testdb3="test_replicated.tdb" + +test_node="0" + +echo "Shutting down node $test_node" +ctdb_nodes_stop "$test_node" +sleep 1 +wait_until_node_has_status 1 recovered +try_command_on_node -v 1 $CTDB status + +echo "Create test databases" +try_command_on_node 1 $CTDB attach "$testdb1" +try_command_on_node 1 $CTDB attach "$testdb2" persistent +try_command_on_node 1 $CTDB attach "$testdb3" replicated + +echo +echo "Checking if database is attached with correct flags" +for node in $(seq 0 $lastnode) ; do + if [ $node -ne $test_node ] ; then + check_db $node $testdb1 "" + check_db $node $testdb2 PERSISTENT + check_db $node $testdb3 REPLICATED + fi +done + +###################################################################### + +echo +echo "Start node $test_node" +ctdb_nodes_start "$test_node" +sleep 1 +wait_until_ready + +echo +echo "Checking if database is attached with correct flags" +check_db $test_node $testdb1 "" +check_db $test_node $testdb2 PERSISTENT +check_db $test_node $testdb3 REPLICATED + +###################################################################### + +echo +echo "Restarting node $test_node" +ctdb_nodes_restart "$test_node" +sleep 1 +wait_until_ready + +echo +echo "Checking if database is attached with correct flags" +check_db $test_node $testdb1 "" +check_db $test_node $testdb2 PERSISTENT +check_db $test_node $testdb3 REPLICATED diff --git a/ctdb/tests/INTEGRATION/database/basics.003.detach.sh b/ctdb/tests/INTEGRATION/database/basics.003.detach.sh new file mode 100755 index 0000000..cb44955 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/basics.003.detach.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# Verify that 'ctdb detach' works as expected: +# 1. Attach test databases +# 2. Detach test databases +# 3. Confirm test databases are not attached + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +###################################################################### + +try_command_on_node 0 "$CTDB listnodes -X | wc -l" +numnodes="$out" + +###################################################################### + +# Confirm that the database is attached +check_db_once () +{ + local db="$1" + + local num_db + + try_command_on_node all "$CTDB getdbmap" + num_db=$(grep -cF "name:${db}" "$outfile") || true + if [ "$num_db" -eq "$numnodes" ]; then + return 0 + else + return 1 + fi +} + +check_db () +{ + local db="$1" + + echo "Waiting until database ${db} is attached on all nodes" + wait_until 10 check_db_once "$db" +} + +# Confirm that no nodes have databases attached +check_no_db_once () +{ + local db="$1" + + local num_db + + try_command_on_node all "$CTDB getdbmap" + num_db=$(grep -cF "name:${db}" "$outfile") || true + if [ "$num_db" -eq 0 ]; then + return 0 + else + return 1 + fi +} + +check_no_db () +{ + local db="$1" + + echo "Waiting until database ${db} is detached on all nodes" + wait_until 10 check_no_db_once "$db" +} + +###################################################################### + +testdb1="detach_test1.tdb" +testdb2="detach_test2.tdb" +testdb3="detach_test3.tdb" +testdb4="detach_test4.tdb" + +echo "Create test databases" +for db in "$testdb1" "$testdb2" "$testdb3" "$testdb4" ; do + echo " $db" + try_command_on_node 0 $CTDB attach "$db" +done + +for db in "$testdb1" "$testdb2" "$testdb3" "$testdb4" ; do + check_db "$db" +done + +###################################################################### + +echo +echo "Ensuring AllowClientDBAttach=1 on all nodes" +try_command_on_node all $CTDB setvar AllowClientDBAttach 1 + +echo "Check failure detaching single test database $testdb1" +try_command_on_node 1 "! $CTDB detach $testdb1" +check_db "$testdb1" + +echo +echo "Setting AllowClientDBAttach=0 on node 0" +try_command_on_node 0 $CTDB setvar AllowClientDBAttach 0 + +echo "Check failure detaching single test database $testdb1" +try_command_on_node 1 "! $CTDB detach $testdb1" +check_db "$testdb1" + +echo +echo "Setting AllowClientDBAttach=0 on all nodes" +try_command_on_node all $CTDB setvar AllowClientDBAttach 0 + +echo "Check detaching single test database $testdb1" +try_command_on_node 1 "$CTDB detach $testdb1" +check_no_db "$testdb1" + +###################################################################### + +echo +echo "Detach multiple test databases" +echo " $testdb2, $testdb3, $testdb4" +try_command_on_node 0 $CTDB detach $testdb2 $testdb3 $testdb4 + +for db in "$testdb2" "$testdb3" "$testdb4" ; do + check_no_db "$db" +done + +###################################################################### + +echo +echo "Attach a single test database" +try_command_on_node all $CTDB setvar AllowClientDBAttach 1 +try_command_on_node 0 $CTDB attach $testdb1 +check_db "$testdb1" + +echo +echo "Write a key to database" +try_command_on_node 0 $CTDB writekey $testdb1 foo bar +try_command_on_node 0 $CTDB catdb $testdb1 +num_keys=$(sed -n -e 's/Dumped \([0-9]*\) records/\1/p' "$outfile") || true +if [ -n "$num_keys" -a $num_keys -eq 1 ]; then + echo "GOOD: Key added to database" +else + echo "BAD: Key did not get added to database" + cat "$outfile" + exit 1 +fi + +echo +echo "Detach test database" +try_command_on_node all $CTDB setvar AllowClientDBAttach 0 +try_command_on_node 0 $CTDB detach $testdb1 +check_no_db "$testdb1" + +echo +echo "Re-attach test database" +try_command_on_node all $CTDB setvar AllowClientDBAttach 1 +try_command_on_node 0 $CTDB attach $testdb1 +check_db "$testdb1" + +echo +echo "Check if the database is empty" +try_command_on_node 0 $CTDB catdb $testdb1 +num_keys=$(sed -n -e 's/Dumped \([0-9]*\) records/\1/p' "$outfile") || true +if [ -n "$num_keys" -a $num_keys -eq 0 ]; then + echo "GOOD: Database $testdb1 is empty" +else + echo "BAD: Database $testdb1 is not empty" + cat "$outfile" + exit 1 +fi diff --git a/ctdb/tests/INTEGRATION/database/basics.004.wipe.sh b/ctdb/tests/INTEGRATION/database/basics.004.wipe.sh new file mode 100755 index 0000000..115d64c --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/basics.004.wipe.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Verify that 'ctdb wipedb' can clear a persistent database: +# 1. Verify that the status on all of the ctdb nodes is 'OK'. +# 2. Create a persistent test database +# 3. Add some records to node 0 and node 1 +# 4. Run wipedb on node 0 +# 5. verify the database is empty on both node 0 and 1 + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" + +# 2. +test_db="persistent_test.tdb" +echo "Create persistent test database \"$test_db\"" +try_command_on_node 0 $CTDB attach "$test_db" persistent + +# 3. +# add one record to node 0 key==ABC data==ABC +echo "Store key(ABC) data(ABC) on node 0" +db_ctdb_tstore 0 "$test_db" "ABC" "ABC" + +# add one record to node 1 key==DEF data==DEF +echo "Store key(DEF) data(DEF) on node 1" +db_ctdb_tstore 1 "$test_db" "DEF" "DEF" + +# 4. +echo "Wipe database" +try_command_on_node 0 $CTDB wipedb "$test_db" + +# check that the database is wiped +num_records=$(db_ctdb_cattdb_count_records 1 "$test_db") +if [ $num_records = "0" ] ; then + echo "OK: Database was wiped" +else + echo "BAD: We did not end up with an empty database" + exit 1 +fi + +echo "Force a recovery" +try_command_on_node 0 $CTDB recover + +# check that the database is wiped +num_records=$(db_ctdb_cattdb_count_records 1 "$test_db") +if [ $num_records = "0" ] ; then + echo "OK: Database was wiped" +else + echo "BAD: We did not end up with an empty database" + exit 1 +fi diff --git a/ctdb/tests/INTEGRATION/database/basics.010.backup_restore.sh b/ctdb/tests/INTEGRATION/database/basics.010.backup_restore.sh new file mode 100755 index 0000000..8c469d4 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/basics.010.backup_restore.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Confirm that 'ctdb restoredb' works correctly: +# 1. Create a persistent test database +# 2. Add some records to test database +# 3. Backup database +# 4. Wipe database and verify the database is empty on all nodes +# 5. Restore database and make sure all the records are restored +# 6. Make sure no recovery has been triggered + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +try_command_on_node 0 $CTDB status +generation=$(sed -n -e 's/^Generation:\([0-9]*\)/\1/p' "$outfile") + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" + +# 2. +test_db="restoredb_test.tdb" +test_dump=$(mktemp) +echo $test_dump +echo "Create persistent test database \"$test_db\"" +try_command_on_node 0 $CTDB attach "$test_db" persistent +try_command_on_node 0 $CTDB wipedb "$test_db" + +# 3. +# add 10,000 records to database +echo "Adding 10000 records to database" +( +for i in $(seq 1 10000) ; do + echo "\"key$i\" \"value$i\"" +done +) | try_command_on_node -i 0 $CTDB ptrans "$test_db" + +num_records=$(db_ctdb_cattdb_count_records 1 "$test_db") +if [ $num_records = "10000" ] ; then + echo "OK: Records added" +else + echo "BAD: We did not end up with 10000 records" + echo "num records = $num_records" + exit 1 +fi + +ctdb_test_exit_hook_add "rm -f $test_dump" + +# 4. +echo "Backup database" +try_command_on_node 0 $CTDB backupdb "$test_db" "$test_dump" + +# 5. +echo "Wipe database" +try_command_on_node 0 $CTDB wipedb "$test_db" + +# check that the database is restored +num_records=$(db_ctdb_cattdb_count_records 1 "$test_db") +if [ $num_records = "0" ] ; then + echo "OK: Database was wiped" +else + echo "BAD: We did not end up with an empty database" + echo "num records = $num_records" + exit 1 +fi + +# 6. +echo "Restore database" +try_command_on_node 0 $CTDB restoredb "$test_dump" "$test_db" + +# check that the database is restored +num_records=$(db_ctdb_cattdb_count_records 1 "$test_db") +if [ $num_records = "10000" ] ; then + echo "OK: Database was restored" +else + echo "BAD: We did not end up with 10000 records" + echo "num records = $num_records" + exit 1 +fi + +# 7. +wait_until_ready + +try_command_on_node 0 $CTDB status +new_generation=$(sed -n -e 's/^Generation:\([0-9]*\)/\1/p' "$outfile") + +echo "Old generation = $generation" +echo "New generation = $new_generation" + +if [ "$generation" = "$new_generation" ]; then + echo "OK: Database recovery not triggered." +else + echo "BAD: Database recovery triggered." + exit 1 +fi diff --git a/ctdb/tests/INTEGRATION/database/fetch.001.ring.sh b/ctdb/tests/INTEGRATION/database/fetch.001.ring.sh new file mode 100755 index 0000000..4d7d392 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/fetch.001.ring.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Run the fetch_ring test and sanity check the output + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" + +echo "Running fetch_ring on all $num_nodes nodes." +testprog_onnode -v -p all \ + fetch_ring -n "$num_nodes" -D "fetch_ring.tdb" -k "testkey" + +pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$' +sanity_check_output 1 "$pat" + +# Get the last line of output. +last=$(tail -n 1 "$outfile") + +# $last should look like this: +# Fetch[1]: 10670.93 msgs/sec +stuff="${last##*Fetch\[*\]: }" +mps="${stuff% msgs/sec*}" + +if [ ${mps%.*} -ge 10 ] ; then + echo "OK: $mps msgs/sec >= 10 msgs/sec" +else + echo "BAD: $mps msgs/sec < 10 msgs/sec" + exit 1 +fi diff --git a/ctdb/tests/INTEGRATION/database/fetch.002.ring-hotkeys.sh b/ctdb/tests/INTEGRATION/database/fetch.002.ring-hotkeys.sh new file mode 100755 index 0000000..6d44253 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/fetch.002.ring-hotkeys.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +# Run the fetch_ring test, sanity check the output and check hot keys +# statistics + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +testdb="fetch_ring.tdb" + +ctdb_get_all_pnns +# $all_pnns is set above +# shellcheck disable=SC2154 +num_nodes=$(echo "$all_pnns" | wc -w | tr -d '[:space:]') +first=$(echo "$all_pnns" | sed -n -e '1p') + +get_key () +{ + _n="$1" + + echo "testkey${_n}" +} + +run_fetch_ring () +{ + _timelimit="$1" + _key_num="$2" + + _key=$(get_key "$_key_num") + _base_cmd="fetch_ring -n ${num_nodes} -D ${testdb}" + _cmd="${_base_cmd} -t ${_timelimit} -k ${_key}" + echo "Running \"${_cmd}\" on all $num_nodes nodes." + testprog_onnode -v -p all "$_cmd" + + _pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$' + sanity_check_output 1 "$_pat" + + # Get the last line of output. + # $outfile is set above by testprog_onnode() + # shellcheck disable=SC2154 + _last=$(tail -n 1 "$outfile") + + # $last should look like this: + # Fetch[1]: 10670.93 msgs/sec + _stuff="${_last##*Fetch\[*\]: }" + _mps="${_stuff% msgs/sec*}" + + if [ "${_mps%.*}" -ge 10 ] ; then + echo "OK: ${_mps} msgs/sec >= 10 msgs/sec" + else + ctdb_test_fail "BAD: ${_mps} msgs/sec < 10 msgs/sec" + fi +} + +check_hot_keys () +{ + _pnn="$1" + _first_key="$2" + _num_keys="$3" + + echo + echo "Checking hot keys on node ${_pnn}" + + ctdb_onnode "$_pnn" dbstatistics "$testdb" + + # Get hot keys with a non-empty key + _hotkeys=$(grep -Ex '[[:space:]]+Count:[[:digit:]]+ Key:[[:xdigit:]]+' \ + "$outfile") || true + + # Check that there are the right number of non-empty slots + if [ -z "$_hotkeys" ] ; then + _num=0 + else + _num=$(echo "$_hotkeys" | wc -l | tr -d '[:space:]') + fi + _msg="hot key slots in use = ${_num}" + if [ "$_num_keys" -ne "$_num" ] ; then + echo + cat "$outfile" + ctdb_test_fail "BAD: ${_msg} (expected ${_num_keys})" + fi + echo "GOOD: ${_msg}" + + # No hot keys? Done... + if [ "$_num" = 0 ] ; then + return + fi + + # Check that hot key counts are correctly sorted + # + # Try to be as POSIX as possible + # shellcheck disable=SC2001 + _counts=$(echo "$_hotkeys" | \ + sed -e 's|.*Count:\([[:digit:]][[:digit:]]*\).*|\1|') + _counts_sorted=$(echo "$_counts" | sort -n) + if [ "$_counts" != "$_counts_sorted" ] ; then + echo + cat "$outfile" + ctdb_test_fail "BAD: hot keys not sorted" + fi + echo "GOOD: hot key counts are correctly sorted" + + # Check that all keys are considered hot + for _j in $(seq "$_first_key" $((_first_key + _num_keys - 1))) ; do + _key=$(get_key "$_j") + _key_hex=$(printf '%s' "$_key" | \ + od -A n -t x1 | \ + tr -d '[:space:]') + if ! echo "$_hotkeys" | grep -q "Key:${_key_hex}\$" ; then + echo + cat "$outfile" + ctdb_test_fail "BAD: key \"${_key}\" is not a hot key" + fi + done + echo "GOOD: all keys are listed as hot keys" +} + +# Run fetch_ring for each of 10 keys. After each run confirm that all +# keys used so far are considered hot keys (and do other hot key +# sanity checks) on all nodes. +for i in $(seq 1 10) ; do + run_fetch_ring 5 "$i" + + for pnn in $all_pnns ; do + check_hot_keys "$pnn" 1 "$i" + done + + echo +done + +echo +echo "Resetting statistics on node ${first}" +ctdb_onnode "$first" statisticsreset + +# Ensure that only node $first has had statistics reset +for pnn in $all_pnns ; do + if [ "$pnn" = "$first" ] ; then + check_hot_keys "$pnn" 1 0 + else + check_hot_keys "$pnn" 1 10 + fi +done + +echo + +# Run fetch_ring for each of 3 new keys. After each run confirm that +# the new keys used so far are considered hot keys (and do other hot +# key sanity checks) on node $first. +# +# Note that nothing can be said about hot keys on other nodes, since +# they may be an arbitrary blend of old and new keys. +for i in $(seq 1 3) ; do + run_fetch_ring 5 $((100 + i)) + + check_hot_keys 0 101 "$i" + + echo +done diff --git a/ctdb/tests/INTEGRATION/database/readonly.001.basic.sh b/ctdb/tests/INTEGRATION/database/readonly.001.basic.sh new file mode 100755 index 0000000..aeb9740 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/readonly.001.basic.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash + +# Test support for read-only records + +# Read-only records can be activated at runtime using a ctdb command. +# If read-only records are not activated, then any attempt to fetch a +# read-only copy should be automatically upgraded to a read-write +# fetch_locked(). + +# If read-only delegations are present, then any attempt to acquire a +# read-write fetch_lock will trigger revocation of all delegations +# before the fetch_locked(). + +# 1. Create a test database and some records +# 2. Try to fetch read-only records, this should not result in any delegations +# 3. Activate read-only support +# 4. Try to fetch read-only records, this should result in delegations +# 5. Do a fetchlock and the delegations should be revoked +# 6. Try to fetch read-only records, this should result in delegations + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +###################################################################### + +# Confirm that no nodes have databases with read-only delegations +check_no_readonly () +{ + try_command_on_node all $CTDB cattdb $testdb + local ro_flags="RO_HAVE_READONLY|RO_HAVE_DELEGATIONS" + local numreadonly=$(grep -c -E "$ro_flags" "$outfile") || true + if [ $numreadonly -eq 0 ] ; then + echo "GOOD: no read-only delegations" + else + echo "BAD: there are read-only delegations" + cat "$outfile" + exit 1 + fi +} + +# Check that the test record has the correct read-only flags on the +# given nodes. The first node is the dmaster, which should know there +# are delegations but should not be flagged as having a read-only +# copy. Subsequent nodes should have a read-only copy but not know +# about any (other) delegations. +check_readonly () +{ + local dmaster="$1" ; shift + local others="$*" + + local count + + try_command_on_node $dmaster $CTDB cattdb $testdb + count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true + if [ $count -eq 1 ] ; then + echo "GOOD: dmaster ${dmaster} has read-only delegations" + else + echo "BAD: dmaster ${dmaster} has no read-only delegations" + cat "$outfile" + exit 1 + fi + count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true + if [ $count -ne 0 ] ; then + echo "BAD: dmaster ${dmaster} has a read-only copy" + cat "$outfile" + exit 1 + fi + + local o + for o in $others ; do + try_command_on_node $o $CTDB cattdb $testdb + count=$(grep -c -E "RO_HAVE_READONLY" "$outfile") || true + if [ $count -eq 1 ] ; then + echo "GOOD: node ${o} has a read-only copy" + else + echo "BAD: node ${o} has no read-only copy" + cat "$outfile" + exit 1 + fi + count=$(grep -c -E "RO_HAVE_DELEGATIONS" "$outfile") || true + if [ $count -ne 0 ] ; then + echo "BAD: other node ${o} has read-only delegations" + cat "$outfile" + exit 1 + fi + done +} + +###################################################################### + +echo "Get list of nodes..." +ctdb_onnode 0 "-X listnodes" +all_nodes=$(awk -F'|' '{print $2}' "$outfile") + +###################################################################### + +testdb="test.tdb" +echo "Create test database \"${testdb}\"" +try_command_on_node 0 $CTDB attach $testdb + +echo "Create some records..." +try_command_on_node all $CTDB_TEST_WRAPPER $VALGRIND update_record \ + -D ${testdb} -k testkey + +###################################################################### + +echo "Try some readonly fetches, these should all be upgraded to full fetchlocks..." +try_command_on_node all $CTDB_TEST_WRAPPER $VALGRIND fetch_readonly \ + -D ${testdb} -k testkey + +check_no_readonly + +###################################################################### + +echo "Activate read-only record support for \"$testdb\"..." +try_command_on_node all $CTDB setdbreadonly $testdb + +# Database should be tagged as READONLY +try_command_on_node 0 $CTDB getdbmap +db_details=$(awk -v db="$testdb" '$2 == foo="name:" db { print }' "$outfile") +if grep -q "READONLY" <<<"$db_details" ; then + echo "GOOD: read-only record support is enabled" +else + echo "BAD: could not activate read-only support" + echo "$db_details" + exit 1 +fi + +###################################################################### + +echo "Create 1 read-only delegation ..." +# dmaster=1 +try_command_on_node 1 $CTDB_TEST_WRAPPER $VALGRIND update_record \ + -D ${testdb} -k testkey + +# Fetch read-only to node 0 +try_command_on_node 0 $CTDB_TEST_WRAPPER $VALGRIND fetch_readonly \ + -D ${testdb} -k testkey + +check_readonly 1 0 + +###################################################################### + +echo "Verify that a fetchlock revokes read-only delegations..." +# Node 1 becomes dmaster +try_command_on_node 1 $CTDB_TEST_WRAPPER $VALGRIND update_record \ + -D ${testdb} -k testkey + +check_no_readonly + +###################################################################### + +echo "Create more read-only delegations..." +dmaster=1 +try_command_on_node $dmaster $CTDB_TEST_WRAPPER $VALGRIND update_record \ + -D ${testdb} -k testkey + +others="" +for n in $all_nodes ; do + if [ "$n" != "$dmaster" ] ; then + # Fetch read-only copy to this node + try_command_on_node $n $CTDB_TEST_WRAPPER $VALGRIND fetch_readonly \ + -D ${testdb} -k testkey + others="${others} ${n}" + fi +done + +check_readonly $dmaster $others + +###################################################################### + +echo "Verify that a recovery will revoke the delegations..." +try_command_on_node 0 $CTDB recover + +check_no_readonly diff --git a/ctdb/tests/INTEGRATION/database/recovery.001.volatile.sh b/ctdb/tests/INTEGRATION/database/recovery.001.volatile.sh new file mode 100755 index 0000000..d7aaa3b --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/recovery.001.volatile.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +# Test that recovery correctly handles RSNs + +# Recovery can under certain circumstances lead to old record copies +# resurrecting: Recovery selects the newest record copy purely by RSN. At +# the end of the recovery, the leader is the dmaster for all +# records in all (non-persistent) databases. And the other nodes locally +# hold the complete copy of the databases. The bug is that the recovery +# process does not increment the RSN on the leader at the end of +# the recovery. Now clients acting directly on the leader will +# directly change a record's content on the leader without migration +# and hence without RSN bump. So a subsequent recovery can not tell that +# the leader's copy is newer than the copies on the other nodes, since +# their RSN is the same. Hence, if the leader is not node 0 (or more +# precisely not the active node with the lowest node number), the recovery +# will choose copies from nodes with lower number and stick to these. + +# 1. Create a test database +# 2. Add a record with value value1 on leader +# 3. Force a recovery +# 4. Update the record with value value2 on leader +# 5. Force a recovery +# 6. Confirm that the value is value2 + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +# +# Main test +# +TESTDB="rec_test.tdb" + +status=0 + +# Make sure node 0 is not the leader +echo "find out which node is leader" +ctdb_onnode 0 leader +leader="$out" +if [ "$leader" = "0" ]; then + echo "node 0 is leader, disable leader role on node 0" + # + # Note: + # It should be sufficient to run "ctdb setleaderrole off" + # on node 0 and wait for election and recovery to finish. + # But there were problems related to this in this automatic + # test, so for now use "ctdb stop" and "ctdb continue". + # + echo "stop node 0" + try_command_on_node 0 $CTDB stop + wait_until_node_has_status 0 stopped + echo "continue node 0" + try_command_on_node 0 $CTDB continue + wait_until_node_has_status 0 notstopped + + ctdb_onnode 0 leader + leader="$out" + if [ "$leader" = "0" ]; then + echo "failed to move leader to different node" + exit 1 + fi +fi + +echo "Leader:${leader}" + +# Create a temporary non-persistent database to test with +echo "create test database $TESTDB" +ctdb_onnode "$leader" attach "$TESTDB" + +# Wipe Test database +echo "wipe test database" +ctdb_onnode "$leader" wipedb "$TESTDB" + +# Add a record key=test1 data=value1 +echo "store key(test1) data(value1)" +ctdb_onnode "$leader" writekey "$TESTDB" test1 value1 + +# Fetch a record key=test1 +echo "read key(test1)" +ctdb_onnode "$leader" readkey "$TESTDB" test1 +cat "$outfile" + +# Do a recovery +echo "force recovery" +ctdb_onnode "$leader" recover + +wait_until_node_has_status "$leader" recovered + +# Add a record key=test1 data=value2 +echo "store key(test1) data(value2)" +ctdb_onnode "$leader" writekey "$TESTDB" test1 value2 + +# Fetch a record key=test1 +echo "read key(test1)" +ctdb_onnode "$leader" readkey "$TESTDB" test1 +cat "$outfile" + +# Do a recovery +echo "force recovery" +ctdb_onnode "$leader" recover + +wait_until_node_has_status "$leader" recovered + +# Verify record key=test1 +echo "read key(test1)" +ctdb_onnode "$leader" readkey "$TESTDB" test1 +cat "$outfile" +if [ "$out" = "Data: size:6 ptr:[value2]" ]; then + echo "GOOD: Recovery did not corrupt database" +else + echo "BAD: Recovery corrupted database" + status=1 +fi + +exit $status diff --git a/ctdb/tests/INTEGRATION/database/recovery.002.large.sh b/ctdb/tests/INTEGRATION/database/recovery.002.large.sh new file mode 100755 index 0000000..4736071 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/recovery.002.large.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# Test recovery of large volatile and persistent databases + +# Recovery now uses DB_PULL and DB_PUSH_START/DB_PUSH_CONFIRM +# controls. This sends the records in batches of ~RecBufferSizeLimit +# in size at a time. Test that large databases are re-assembled +# correctly. + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +# +# Main test +# +TEST1DB="large_persistent_db.tdb" +TEST2DB="large_volatile_db.tdb" +RECDATA=$(onnode 0 mktemp) + +# Create a persistent database to test +echo "create persistent test database $TEST1DB" +try_command_on_node 0 $CTDB attach $TEST1DB persistent + +# Wipe Test database +echo "wipe test database $TEST1DB" +try_command_on_node 0 $CTDB wipedb $TEST1DB + +# Create dummy record data +echo "creating dummy record data" +onnode 0 dd if=/dev/urandom of=$RECDATA bs=10K count=1 + +# Add 345 records +echo "Adding 345 records" +for i in $(seq 1 345) ; do + try_command_on_node 0 $CTDB pstore $TEST1DB record$i $RECDATA || exit 1 +done + +num_records=$(db_ctdb_cattdb_count_records 0 $TEST1DB) +if [ $num_records = "345" ] ; then + echo "OK: records added correctly" +else + echo "BAD: persistent database has $num_records of 345 records" + try_command_on_node -v 0 "$CTDB cattdb $TEST1DB | tail -n 1" + exit 1 +fi + +# Create a volatile database to test +echo "create volatile test database $TEST2DB" +try_command_on_node 0 $CTDB attach $TEST2DB + +# Wipe Test database +echo "wipe test database $TEST2DB" +try_command_on_node 0 $CTDB wipedb $TEST2DB + +# Create dummy record data +v1="1234567890" +v2="$v1$v1$v1$v1$v1$v1$v1$v1$v1$v1" +v3="$v2$v2$v2$v2$v2$v2$v2$v2$v2$v2" + +# Add 1234 records +echo "Adding 1234 records" +for i in $(seq 1 1234) ; do + try_command_on_node 0 $CTDB writekey $TEST2DB record$i $v3 || exit 1 +done + +num_records=$(db_ctdb_cattdb_count_records 0 $TEST2DB) +if [ $num_records = "1234" ] ; then + echo "OK: records added correctly" +else + echo "BAD: volatile database has $num_records of 1234 records" + try_command_on_node -v 0 "$CTDB cattdb $TEST2DB | tail -n 1" + exit 1 +fi + +echo +leader_get 0 +# Set RecBufferSizeLimit to 10000 +ctdb_onnode "$leader" setvar RecBufferSizeLimit 10000 + +# Do a recovery +echo "force recovery" +try_command_on_node 0 $CTDB recover + +wait_until_node_has_status 0 recovered 30 + +# check that there are correct number of records +num_records=$(db_ctdb_cattdb_count_records 0 $TEST1DB) +if [ $num_records = "345" ] ; then + echo "OK: persistent database recovered correctly" +else + echo "BAD: persistent database has $num_records of 345 records" + try_command_on_node -v 0 "$CTDB cattdb $TEST1DB | tail -n 1" + exit 1 +fi + +num_records=$(db_ctdb_cattdb_count_records 0 $TEST2DB) +if [ $num_records = "1234" ] ; then + echo "OK: volatile database recovered correctly" +else + echo "BAD: volatile database has $num_records of 1234 records" + try_command_on_node -v 0 "$CTDB cattdb $TEST2DB | tail -n 1" + exit 1 +fi diff --git a/ctdb/tests/INTEGRATION/database/recovery.003.no_resurrect.sh b/ctdb/tests/INTEGRATION/database/recovery.003.no_resurrect.sh new file mode 100755 index 0000000..b314d4d --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/recovery.003.no_resurrect.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +# Ensure recovery doesn't resurrect deleted records from recently +# inactive nodes + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +testdb="rec_test.tdb" + +echo "Getting list of nodes..." +ctdb_get_all_pnns + +first=$(echo "$all_pnns" | sed -n -e '1p') +second=$(echo "$all_pnns" | sed -n -e '2p') +notfirst=$(echo "$all_pnns" | tail -n +2) + +echo "Create/wipe test database ${testdb}" +try_command_on_node $first $CTDB attach "$testdb" +try_command_on_node $first $CTDB wipedb "$testdb" + +echo "store key(test1) data(value1)" +try_command_on_node $first $CTDB writekey "$testdb" test1 value1 + +echo "Migrate key(test1) to all nodes" +try_command_on_node all $CTDB readkey "$testdb" test1 + +echo "Stop node ${first}" +try_command_on_node $first $CTDB stop +wait_until_node_has_status $first stopped + +echo "Delete key(test1)" +try_command_on_node $second $CTDB deletekey "$testdb" test1 + +database_has_zero_records () +{ + # shellcheck disable=SC2086 + # $notfirst can be multi-word + check_cattdb_num_records "$testdb" 0 "$notfirst" +} + +echo "Trigger a recovery" +try_command_on_node "$second" $CTDB recover + +echo "Checking that database has 0 records" +database_has_zero_records + +echo "Continue node ${first}" +try_command_on_node $first $CTDB continue +wait_until_node_has_status $first notstopped + +echo "Get database contents" +try_command_on_node -v $first $CTDB catdb "$testdb" + +if grep -q '^key(' "$outfile" ; then + echo "BAD: Deleted record has been resurrected" + exit 1 +fi + +echo "GOOD: Deleted record is still gone" diff --git a/ctdb/tests/INTEGRATION/database/recovery.010.persistent.sh b/ctdb/tests/INTEGRATION/database/recovery.010.persistent.sh new file mode 100755 index 0000000..d13a9a5 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/recovery.010.persistent.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +# Ensure that persistent databases are correctly recovered by database +# sequence number +# +# 1. Create and wipe a persistent test database +# 2. Directly add a single record to the database on each node +# 3. Trigger a recover +# 4. Ensure that the database contains only a single record +# +# Repeat but with sequence numbers set by hand on each node + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" + +add_record_per_node () +{ + _i=0 + while [ $_i -lt $num_nodes ] ; do + _k="KEY${_i}" + _d="DATA${_i}" + echo "Store key(${_k}) data(${_d}) on node ${_i}" + db_ctdb_tstore $_i "$test_db" "$_k" "$_d" + _i=$(($_i + 1)) + done +} + +test_db="persistent_test.tdb" +echo "Create persistent test database \"$test_db\"" +try_command_on_node 0 $CTDB attach "$test_db" persistent + +# 3, +# If no __db_sequence_number__ recover whole database +# + +echo +echo "Test that no __db_sequence_number__ does not blend the database during recovery" + +# wipe database +echo "Wipe the test database" +try_command_on_node 0 $CTDB wipedb "$test_db" + +add_record_per_node + +# force a recovery +echo force a recovery +try_command_on_node 0 $CTDB recover + +# Check that we now have 1 record on node 0 +num_records=$(db_ctdb_cattdb_count_records 0 "$test_db") +if [ $num_records = "1" ] ; then + echo "OK: databases were not blended" +else + echo "BAD: we did not end up with the expected single record after the recovery" + exit 1 +fi + + +# 4, +# If __db_sequence_number__ recover whole database +# + +echo +echo test that __db_sequence_number__ does not blend the database during recovery + +# wipe database +echo wipe the test database +try_command_on_node 0 $CTDB wipedb persistent_test.tdb + +add_record_per_node + +echo "Add __db_sequence_number__==5 record to all nodes" +pnn=0 +while [ $pnn -lt $num_nodes ] ; do + db_ctdb_tstore_dbseqnum $pnn "$test_db" 5 + pnn=$(($pnn + 1)) +done + +echo "Set __db_sequence_number__ to 7 on node 0" +db_ctdb_tstore_dbseqnum 0 "$test_db" 7 + +echo "Set __db_sequence_number__ to 8 on node 1" +db_ctdb_tstore_dbseqnum 1 "$test_db" 8 + + +# force a recovery +echo force a recovery +try_command_on_node 0 $CTDB recover + +# check that we now have both records on node 0 +num_records=$(db_ctdb_cattdb_count_records 0 "$test_db") +if [ $num_records = "1" ] ; then + echo "OK: databases were not blended" +else + echo "BAD: we did not end up with the expected single record after the recovery" + exit 1 +fi diff --git a/ctdb/tests/INTEGRATION/database/recovery.011.continue.sh b/ctdb/tests/INTEGRATION/database/recovery.011.continue.sh new file mode 100755 index 0000000..995b282 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/recovery.011.continue.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# Confirm that the deleted records are not resurrected after recovery +# +# 1. Create a persistent database +# 2. Add a record and update it few times. +# 3. Delete the record +# 4. Use "ctdb stop" to stop one of the nodes +# 5. Add a record with same key. +# 6. Continue on the stopped node +# 7. Confirm that the record still exists + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +do_test() +{ +# Wipe Test database +echo "wipe test database" +try_command_on_node 0 $CTDB wipedb $TESTDB + +# Add a record key=test1 data=value1 +# and update values +for value in value1 value2 value3 value4 value5 ; do + echo "store key(test1) data($value)" + echo "\"test1\" \"$value\"" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB" +done + +# Delete record +echo "delete key(test1)" +try_command_on_node 0 $CTDB pdelete $TESTDB test1 + +# Stop a node +echo "stop node 1" +try_command_on_node 1 $CTDB stop + +wait_until_node_has_status 1 stopped + +# Add a record key=test1 data=value2 +echo "store key(test1) data(newvalue1)" +echo '"test1" "newvalue1"' | try_command_on_node -i 0 $CTDB ptrans "$TESTDB" + +# Continue node +echo "continue node 1" +try_command_on_node 1 $CTDB continue + +wait_until_node_has_status 1 notstopped + +} + +# +# Main test +# +TESTDB="persistent_test.tdb" + +status=0 + +# Create a temporary persistent database to test with +echo "create persistent test database $TESTDB" +try_command_on_node 0 $CTDB attach $TESTDB persistent + +do_test +if try_command_on_node 0 $CTDB pfetch $TESTDB test1 ; then + echo "GOOD: Record was not deleted (recovery by sequence number worked)" +else + echo "BAD: Record was deleted" + status=1 +fi + +exit $status diff --git a/ctdb/tests/INTEGRATION/database/scripts/local.bash b/ctdb/tests/INTEGRATION/database/scripts/local.bash new file mode 100644 index 0000000..ae2e0d5 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/scripts/local.bash @@ -0,0 +1,116 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +check_cattdb_num_records () +{ + local db="$1" + local num="$2" + local nodes="$3" + + # $nodes has embedded newlines - put list on 1 line for printing + local t + t=$(echo "$nodes" | xargs) + echo "Confirm that ${db} has ${num} record(s) on node(s): ${t}" + + local ret=0 + local node + for node in $nodes ; do + local num_found + + num_found=$(db_ctdb_cattdb_count_records "$node" "$db") + if [ "$num_found" = "$num" ] ; then + continue + fi + + printf 'BAD: %s on node %d has %d record(s), expected %d\n' \ + "$db" "$node" "$num_found" "$num" + ctdb_onnode -v "$node" "cattdb $db" + ret=1 + done + + return $ret +} + +_key_dmaster_check () +{ + local node="$1" + local db="$2" + local key="$3" + local dmaster="${4:-${node}}" + + testprog_onnode "$node" "ctdb-db-test local-read ${db} ${key}" + + # shellcheck disable=SC2154 + # $outfile is set above by try_command_on_node() + grep -Fqx "dmaster: ${dmaster}" "$outfile" +} + +_key_dmaster_fail () +{ + local dmaster="$1" + + echo "BAD: node ${dmaster} is not dmaster" + # shellcheck disable=SC2154 + # $outfile is set by the caller via _key_dmaster_check() + cat "$outfile" + ctdb_test_fail +} + +vacuum_test_key_dmaster () +{ + local node="$1" + local db="$2" + local key="$3" + local dmaster="${4:-${node}}" + + if ! _key_dmaster_check "$node" "$db" "$key" "$dmaster" ; then + _key_dmaster_fail "$dmaster" + fi +} + +vacuum_test_wait_key_dmaster () +{ + local node="$1" + local db="$2" + local key="$3" + local dmaster="${4:-${node}}" + + if ! wait_until 30 \ + _key_dmaster_check "$node" "$db" "$key" "$dmaster" ; then + _key_dmaster_fail "$dmaster" + fi +} + +vacuum_confirm_key_empty_dmaster () +{ + local node="$1" + local db="$2" + local key="$3" + local dmaster="${4:-${node}}" + + echo "Confirm record key=\"${key}\" is empty and dmaster=${dmaster}" + + vacuum_test_key_dmaster "$node" "$db" "$key" "$dmaster" + + if ! grep -Fqx 'data(0) = ""' "$outfile" ; then + echo "BAD: record not empty" + cat "$outfile" + ctdb_test_fail + fi +} + +db_confirm_key_has_value () +{ + local node="$1" + local db="$2" + local key="$3" + local val="$4" + + local out + + ctdb_onnode "$node" "readkey ${db} ${key}" + outv=$(echo "$out" | sed -n 's|^Data: size:.* ptr:\[\(.*\)\]$|\1|p') + if [ "$val" != "$outv" ] ; then + ctdb_test_fail \ + "BAD: value for \"${key}\"=\"${outv}\" (not \"${val}\")" + fi +} diff --git a/ctdb/tests/INTEGRATION/database/transaction.001.ptrans.sh b/ctdb/tests/INTEGRATION/database/transaction.001.ptrans.sh new file mode 100755 index 0000000..556e523 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/transaction.001.ptrans.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +# Verify that the 'ctdb ptrans' works as expected +# +# Pipe some operation to ctdb ptrans and validate the TDB contents +# with ctdb catdb + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +TESTDB="ptrans_test.tdb" + +# Create a temporary persistent database to test with +echo "create persistent test database $TESTDB" +try_command_on_node 0 $CTDB attach $TESTDB persistent + +# Wipe Test database +echo "wipe test database" +try_command_on_node 0 $CTDB wipedb $TESTDB + +########## + +echo "Adding 3 records" + +items=' +"key1" "value1" +"key2" "value1" +"key3" "value1"' + +echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB" + +try_command_on_node 0 $CTDB catdb "$TESTDB" + +n=$(grep -c '^key.*= "key.*"' "$outfile" || true) + +if [ $n -ne 3 ] ; then + echo "BAD: expected 3 keys in..." + cat "$outfile" + exit 1 +else + echo "GOOD: 3 records were inserted" +fi + +########## + +echo "Deleting 1 record, updating 1, adding 1 new record, 1 bogus input line" + +items=' +"key1" "" +"key2" "value2" +"key3" +"key4" "value1"' + +echo "$items" | try_command_on_node -i 0 $CTDB ptrans "$TESTDB" + +try_command_on_node 0 $CTDB catdb "$TESTDB" + +n=$(grep -c '^key.*= "key.*"' "$outfile" || true) + +if [ $n -ne 3 ] ; then + echo "BAD: expected 3 keys in..." + cat "$outfile" + exit 1 +else + echo "GOOD: 3 records found" +fi + +########## + +echo "Verifying records" + +while read key value ; do + try_command_on_node 0 $CTDB pfetch "$TESTDB" "$key" + if [ "$value" != "$out" ] ; then + echo "BAD: for key \"$key\" expected \"$value\" but got \"$out\"" + exit 1 + else + echo "GOOD: for key \"$key\" got \"$out\"" + fi +done </dev/null & + RECLOOP_PID=$! + ctdb_test_exit_hook_add "kill $RECLOOP_PID >/dev/null 2>&1" +} + +TESTDB="persistent_trans.tdb" + +try_command_on_node 0 "$CTDB attach $TESTDB persistent" +try_command_on_node 0 "$CTDB wipedb $TESTDB" + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" + +if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then + CTDB_TEST_TIMELIMIT=30 +fi + +t="$CTDB_TEST_WRAPPER $VALGRIND transaction_loop \ + -n ${num_nodes} -t ${CTDB_TEST_TIMELIMIT} \ + -D ${TESTDB} -T persistent -k testkey" + +echo "Starting recovery loop" +recovery_loop_start + +echo "Running transaction_loop on all $num_nodes nodes." +try_command_on_node -v -p all "$t" diff --git a/ctdb/tests/INTEGRATION/database/transaction.004.update_record.sh b/ctdb/tests/INTEGRATION/database/transaction.004.update_record.sh new file mode 100755 index 0000000..528303a --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/transaction.004.update_record.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Verify that "ctdb update_record_persistent" creates new records and +# updates existing records in a persistent database +# +# 1. Create and wipe a persistent test database +# 2. Do a recovery +# 3. Confirm that the database is empty +# 4. Create a new record using "ctdb update_record_persistent" +# 5. Confirm the record exists in the database using "ctdb cattdb" +# 6. Update the record's value using "ctdb update_record_persistent" +# 7. Confirm that the original value no longer exists using "ctdb cattdb" + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" + +test_db="persistent_test.tdb" + +# create a temporary persistent database to test with +echo "Create persistent test database \"$test_db\"" +try_command_on_node 0 $CTDB attach "$test_db" persistent + + +# 3. +echo "Wipe the persistent test database" +try_command_on_node 0 $CTDB wipedb "$test_db" +echo "Force a recovery" +try_command_on_node 0 $CTDB recover + +# check that the database is wiped +num_records=$(db_ctdb_cattdb_count_records 1 "$test_db") +if [ $num_records = "0" ] ; then + echo "OK: database was wiped" +else + echo "BAD: we did not end up with an empty database" + exit 1 +fi + +# 4. +echo "Create a new record in the persistent database using UPDATE_RECORD" +try_command_on_node 0 $CTDB_TEST_WRAPPER $VALGRIND update_record_persistent \ + -D "$test_db" -k "Update_Record_Persistent" -v "FirstValue" + +try_command_on_node 0 "$CTDB cattdb "$test_db" | grep 'FirstValue' | wc -l" +if [ "$out" = 1 ] ; then + echo "GOOD: we did not find the record after the create/update" +else + echo "BAD: we did find the record after the create/update" + exit 1 +fi + +# 5. +echo Modify an existing record in the persistent database using UPDATE_RECORD +try_command_on_node 0 $CTDB_TEST_WRAPPER $VALGRIND update_record_persistent \ + -D "$test_db" -k "Update_Record_Persistent" -v "SecondValue" + +try_command_on_node 0 "$CTDB cattdb "$test_db" | grep 'FirstValue' | wc -l" +if [ "$out" = 0 ] ; then + echo "GOOD: did not find old record after the modify/update" +else + echo "BAD: we still found the old record after the modify/update" + exit 1 +fi + +try_command_on_node 0 "$CTDB cattdb "$test_db" | grep 'SecondValue' | wc -l" +if [ "$out" = 1 ] ; then + echo "GOOD: found the record after the modify/update" +else + echo "BAD: could not find the record after the modify/update" + exit 1 +fi + +echo "Wipe the persistent test databases and clean up" +try_command_on_node 0 $CTDB wipedb "$test_db" diff --git a/ctdb/tests/INTEGRATION/database/transaction.010.loop_recovery.sh b/ctdb/tests/INTEGRATION/database/transaction.010.loop_recovery.sh new file mode 100755 index 0000000..9de6c34 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/transaction.010.loop_recovery.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Verify that the transaction_loop test succeeds with recoveries for +# replicated databases + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +recovery_loop() +{ + local COUNT=1 + + while true ; do + echo Recovery $COUNT + try_command_on_node 0 $CTDB recover + sleep 2 + COUNT=$((COUNT + 1)) + done +} + +recovery_loop_start() +{ + recovery_loop >/dev/null & + RECLOOP_PID=$! + ctdb_test_exit_hook_add "kill $RECLOOP_PID >/dev/null 2>&1" +} + +TESTDB="replicated_trans.tdb" + +try_command_on_node 0 "$CTDB attach $TESTDB replicated" +try_command_on_node 0 "$CTDB wipedb $TESTDB" + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" + +if [ -z "$CTDB_TEST_TIMELIMIT" ] ; then + CTDB_TEST_TIMELIMIT=30 +fi + +t="$CTDB_TEST_WRAPPER $VALGRIND transaction_loop \ + -n ${num_nodes} -t ${CTDB_TEST_TIMELIMIT} \ + -D ${TESTDB} -T replicated -k testkey" + +echo "Starting recovery loop" +recovery_loop_start + +echo "Running transaction_loop on all $num_nodes nodes." +try_command_on_node -v -p all "$t" diff --git a/ctdb/tests/INTEGRATION/database/traverse.001.one.sh b/ctdb/tests/INTEGRATION/database/traverse.001.one.sh new file mode 100755 index 0000000..1b3b7c2 --- /dev/null +++ b/ctdb/tests/INTEGRATION/database/traverse.001.one.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +# Confirm that traverses of volatile databases work as expected + +# This is a very simple example. It writes a single record, updates it +# on another node and then confirms that the correct value is found when +# traversing. It then repeats this after removing the LMASTER role from +# the node where the value is updated. + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +# +# Main test +# +TESTDB="traverse_db.tdb" + +echo "create volatile test database $TESTDB" +try_command_on_node 0 $CTDB attach "$TESTDB" + +echo "wipe test database $TESTDB" +try_command_on_node 0 $CTDB wipedb "$TESTDB" + +echo "write foo=bar0 on node 0" +try_command_on_node 0 $CTDB writekey "$TESTDB" "foo" "bar0" + +echo "write foo=bar1 on node 1" +try_command_on_node 1 $CTDB writekey "$TESTDB" "foo" "bar1" + +echo + +check_db_num_records () +{ + local node="$1" + local db="$2" + local n="$3" + + echo "Checking on node ${node} to ensure ${db} has ${n} records..." + try_command_on_node "$node" "${CTDB} catdb ${db}" + + num=$(sed -n -e 's|^Dumped \(.*\) records$|\1|p' "$outfile") + if [ "$num" = "$n" ] ; then + echo "OK: Number of records=${num}" + echo + else + echo "BAD: There were ${num} (!= ${n}) records" + cat "$outfile" + exit 1 + fi +} + +check_db_num_records 0 "$TESTDB" 1 +check_db_num_records 1 "$TESTDB" 1 + +cat <