diff options
Diffstat (limited to 'ctdb/tests/INTEGRATION/failover')
11 files changed, 563 insertions, 0 deletions
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.001.list.sh b/ctdb/tests/INTEGRATION/failover/pubips.001.list.sh new file mode 100755 index 0000000..2fc75b7 --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.001.list.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Verify that 'ctdb ip' shows the correct output + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +echo "Getting list of public IPs..." +try_command_on_node -v 1 "$CTDB ip all | tail -n +2" +ips=$(sed \ + -e 's@ node\[@ @' \ + -e 's@\].*$@@' \ + "$outfile") +machineout=$(sed -r \ + -e 's@^| |$@\|@g' \ + -e 's@[[:alpha:]]+\[@@g' \ + -e 's@\]@@g' \ + "$outfile") + +if ctdb_test_on_cluster ; then + while read ip pnn ; do + try_command_on_node $pnn "ip addr show to ${ip}" + if [ -n "$out" ] ; then + echo "GOOD: node $pnn appears to have $ip assigned" + else + die "BAD: node $pnn does not appear to have $ip assigned" + fi + done <<<"$ips" # bashism to avoid problem setting variable in pipeline. +fi + +echo "Looks good!" + +cmd="$CTDB -X ip all | tail -n +2" +echo "Checking that \"$cmd\" produces expected output..." + +try_command_on_node 1 "$cmd" +if [ "$out" = "$machineout" ] ; then + echo "Yep, looks good!" +else + echo "Nope, it looks like this:" + echo "$out" + echo "Should be like this:" + echo "$machineout" + exit 1 +fi diff --git a/ctdb/tests/INTEGRATION/failover/pubips.010.addip.sh b/ctdb/tests/INTEGRATION/failover/pubips.010.addip.sh new file mode 100755 index 0000000..aba85dd --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.010.addip.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Verify that an IP address can be added to a node using 'ctdb addip' + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +select_test_node_and_ips +get_test_ip_mask_and_iface + +echo "Deleting IP $test_ip from all nodes" +delete_ip_from_all_nodes $test_ip +try_command_on_node -v $test_node $CTDB ipreallocate +wait_until_ips_are_on_node '!' $test_node $test_ip + +# Debugging... +try_command_on_node -v all $CTDB ip + +echo "Adding IP ${test_ip}/${mask} on ${iface}, node ${test_node}" +try_command_on_node $test_node $CTDB addip ${test_ip}/${mask} $iface +try_command_on_node $test_node $CTDB ipreallocate +wait_until_ips_are_on_node $test_node $test_ip diff --git a/ctdb/tests/INTEGRATION/failover/pubips.011.delip.sh b/ctdb/tests/INTEGRATION/failover/pubips.011.delip.sh new file mode 100755 index 0000000..5235a9d --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.011.delip.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Verify that a node's public IP address can be deleted using 'ctdb deleteip' + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +select_test_node_and_ips + +echo "Deleting IP ${test_ip} from node ${test_node}" +try_command_on_node $test_node $CTDB delip $test_ip +try_command_on_node $test_node $CTDB ipreallocate +wait_until_ips_are_on_node '!' $test_node $test_ip diff --git a/ctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh b/ctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh new file mode 100755 index 0000000..a3bb3af --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +# Verify that IPs can be reconfigured using 'ctdb reloadips' + +# Various sub-tests that remove addresses from the public_addresses file +# on a node or delete the entire contents of the public_addresses file. + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +select_test_node_and_ips + +try_command_on_node $test_node $CTDB_TEST_WRAPPER ctdb_base_show +addresses="${out}/public_addresses" +echo "Public addresses file on node $test_node is \"$addresses\"" +backup="${addresses}.$$" + +restore_public_addresses () +{ + try_command_on_node $test_node "mv $backup $addresses >/dev/null 2>&1 || true" +} +ctdb_test_exit_hook_add restore_public_addresses + +# ctdb reloadips will fail if it can't disable takover runs. The most +# likely reason for this is that there is already a takeover run in +# progress. We can't predict when this will happen, so retry if this +# occurs. +do_ctdb_reloadips () +{ + local retry_max=10 + local retry_count=0 + while : ; do + if ctdb_onnode "$test_node" "reloadips all" ; then + return 0 + fi + + if [ "$out" != "Failed to disable takeover runs" ] ; then + return 1 + fi + + if [ $retry_count -ge $retry_max ] ; then + return 1 + fi + + retry_count=$((retry_count + 1)) + echo "Retrying..." + sleep_for 1 + done +} + + +echo "Removing IP $test_ip from node $test_node" + +try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses" + +do_ctdb_reloadips + +try_command_on_node $test_node $CTDB ip + +if grep "^${test_ip} " <<<"$out" ; then + cat <<EOF +BAD: node $test_node can still host IP $test_ip: +$out +EOF + exit 1 +fi + +cat <<EOF +GOOD: node $test_node is no longer hosting IP $test_ip: +$out +EOF + +ctdb_onnode "$test_node" sync + + +echo "Restoring addresses" +restore_public_addresses + +do_ctdb_reloadips + +echo "Getting list of public IPs on node $test_node" +try_command_on_node $test_node "$CTDB ip | tail -n +2" + +if [ -z "$out" ] ; then + echo "BAD: node $test_node has no ips" + exit 1 +fi + +cat <<EOF +GOOD: node $test_node has these addresses: +$out +EOF + +ctdb_onnode "$test_node" sync + + +echo "Emptying public addresses file on $test_node" + +try_command_on_node $test_node "mv $addresses $backup && touch $addresses" + +do_ctdb_reloadips + +echo "Getting list of public IPs on node $test_node" +try_command_on_node $test_node "$CTDB ip | tail -n +2" + +if [ -n "$out" ] ; then + cat <<EOF +BAD: node $test_node still has ips: +$out +EOF + exit 1 +fi + +echo "GOOD: no IPs left on node $test_node" diff --git a/ctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh b/ctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh new file mode 100755 index 0000000..77f9a63 --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Check that CTDB operates correctly if: + +# * failover is disabled; or +# * there are 0 public IPs configured + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_skip_on_cluster + +ctdb_test_init -n + +echo "Starting CTDB with failover disabled..." +ctdb_nodes_start_custom -F + +select_test_node + +echo "Getting IP allocation..." + +# $test_node set above by select_test_node() +# shellcheck disable=SC2154 +try_command_on_node -v "$test_node" "$CTDB ip all | tail -n +2" + +while read ip pnn ; do + if [ "$pnn" != "-1" ] ; then + die "BAD: IP address ${ip} is assigned to node ${pnn}" + fi +done <"$outfile" + +echo "GOOD: All IP addresses are unassigned" + +echo "----------------------------------------" + +echo "Starting CTDB with an empty public addresses configuration..." +ctdb_nodes_start_custom -P /dev/null + +echo "Trying explicit ipreallocate..." +ctdb_onnode "$test_node" ipreallocate + +echo "Good, that seems to work!" +echo diff --git a/ctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh b/ctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh new file mode 100755 index 0000000..845b4b5 --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Verify that an interface is deleted when all IPs on it are deleted + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +select_test_node_and_ips + +# Find interfaces on test node +try_command_on_node $test_node "$CTDB ifaces -X" +ifaces=$(awk -F'|' 'NR > 1 { print $2 }' "$outfile") +echo "Node ${test_node} has interfaces: ${ifaces}" + +# Delete all IPs on each interface... deleting IPs from one interface +# can cause other interfaces to disappear, so we need to be careful... +for i in $ifaces ; do + try_command_on_node $test_node "$CTDB ifaces -X" + info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile") + + if [ -z "$info" ] ; then + echo "Interface ${i} missing... assuming already deleted!" + continue + fi + + echo "Deleting IPs on interface ${i}, with this information:" + echo " $info" + + try_command_on_node $test_node "$CTDB ip -v -X | tail -n +2" + awk -F'|' -v i="$i" \ + '$6 == i { print $2 }' "$outfile" | + while read ip ; do + echo " $ip" + try_command_on_node $test_node "$CTDB delip $ip" + done + try_command_on_node $test_node "$CTDB ipreallocate" + + try_command_on_node $test_node "$CTDB ifaces -X" + info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile") + + if [ -z "$info" ] ; then + echo "GOOD: Interface ${i} has been garbage collected" + else + echo "BAD: Interface ${i} still exists" + echo "$out" + exit 1 + fi +done diff --git a/ctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh b/ctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh new file mode 100755 index 0000000..8daf3f5 --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# Verify that 'ctdb moveip' allows movement of public IPs between nodes + +# This test does not do any network level checks to make sure IP +# addresses are actually on interfaces. It just consults "ctdb ip". + +# To work, this test ensures that IPAllocAlgorithm is not set to 0 +# (Deterministic IPs) and sets NoIPFailback. + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +select_test_node_and_ips + +sanity_check_ips () +{ + echo "Sanity checking IPs..." + + local x ipp prev + prev="" + while read x ipp ; do + [ "$ipp" = "-1" ] && break + if [ -n "$prev" -a "$ipp" != "$prev" ] ; then + echo "OK" + return 0 + fi + prev="$ipp" + done <"$outfile" + + echo "BAD: a node was -1 or IPs are only assigned to one node:" + cat "$outfile" + echo "Are you running an old version of CTDB?" + return 1 +} + +sanity_check_ips + +# Find a target node - it must be willing to host $test_ip + +# $test_node set above by select_test_node_and_ips() +# shellcheck disable=SC2154 +try_command_on_node "$test_node" "$CTDB listnodes | wc -l" +num_nodes="$out" +to_node="" +for i in $(seq 0 $(($num_nodes - 1)) ) ; do + [ $i -ne $test_node ] || continue + all_ips_on_node $i + while read ip x ; do + if [ "$ip" = "$test_ip" ] ; then + to_node="$i" + break 2 + fi + done <"$outfile" +done + +if [ -z "$to_node" ] ; then + echo "Unable to find target node" + exit 1 +fi + +echo "Target node is ${to_node}" + +echo "Setting IPAllocAlgorithm=2 to avoid Deterministic IPs..." +try_command_on_node -q all $CTDB setvar IPAllocAlgorithm 2 + +echo "Turning on NoIPFailback..." +try_command_on_node -q all $CTDB setvar NoIPFailback 1 + +echo "Attempting to move ${test_ip} from node ${test_node} to node ${to_node}" +try_command_on_node $test_node $CTDB moveip $test_ip $to_node +wait_until_ips_are_on_node '!' $test_node $test_ip +wait_until_ips_are_on_node $to_node $test_ip diff --git a/ctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh b/ctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh new file mode 100755 index 0000000..3f40097 --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Verify the operation of "ctdb disable" and "ctdb enable" + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +######################################## + +select_test_node_and_ips + +echo "Disabling node $test_node" +try_command_on_node 1 $CTDB disable -n $test_node +wait_until_node_has_status $test_node disabled 30 all +wait_until_node_has_no_ips "$test_node" + +echo "Re-enabling node $test_node" +try_command_on_node 1 $CTDB enable -n $test_node +wait_until_node_has_status $test_node enabled 30 all +wait_until_node_has_some_ips "$test_node" diff --git a/ctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh b/ctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh new file mode 100755 index 0000000..f5936b0 --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Verify the operation of "ctdb stop" and "ctdb continue" + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +select_test_node_and_ips + +echo "Stopping node ${test_node}..." +try_command_on_node 1 $CTDB stop -n $test_node +wait_until_node_has_status $test_node stopped +wait_until_node_has_no_ips "$test_node" + +echo "Continuing node $test_node" +try_command_on_node 1 $CTDB continue -n $test_node +wait_until_node_has_status $test_node notstopped +wait_until_node_has_some_ips "$test_node" diff --git a/ctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh b/ctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh new file mode 100755 index 0000000..e99a265 --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Verify that 'ctdb setvar NoIPTakeover 1' stops IP addresses being taken over + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +ctdb_get_all_pnns +# out is set above +# shellcheck disable=SC2154 +num_nodes=$(echo "$out" | wc -l | tr -d '[:space:]') +echo "There are $num_nodes nodes..." + +if [ "$num_nodes" -lt 2 ] ; then + echo "Less than 2 nodes!" + exit 1 +fi + +select_test_node_and_ips + + +# sets: num +count_ips_on_node () +{ + local node="$1" + + ctdb_onnode "$node" ip + # outfile is set by ctdb_onnode() above + # shellcheck disable=SC2154,SC2126 + # * || true is needed to avoid command failure when there are no matches + # * Using "wc -l | tr -d '[:space:]'" is our standard + # pattern... and "grep -c" requires handling of special case + # for no match + num=$(grep -v 'Public' "$outfile" | \ + grep " ${node}\$" | \ + wc -l | \ + tr -d '[:space:]') + echo "Number of addresses on node ${node}: ${num}" +} + + +# test_node is set by select_test_node_and_ips() above +# shellcheck disable=SC2154 +count_ips_on_node "$test_node" + +echo "Turning on NoIPTakeover on all nodes" +ctdb_onnode all "setvar NoIPTakeover 1" +ctdb_onnode "$test_node" ipreallocate + +echo "Disable node ${test_node}" +ctdb_onnode "$test_node" disable + +count_ips_on_node "$test_node" +if [ "$num" != "0" ] ; then + test_fail "BAD: node 1 still hosts IP addresses" +fi + + +echo "Enable node 1 again" +ctdb_onnode "$test_node" enable + +count_ips_on_node "$test_node" +if [ "$num" != "0" ] ; then + test_fail "BAD: node 1 took over IP addresses" +fi + + +echo "OK: IP addresses were not taken over" diff --git a/ctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh b/ctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh new file mode 100755 index 0000000..543f9a9 --- /dev/null +++ b/ctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Verify that the recovery daemon handles unhosted IPs properly + +# This test does not do any network level checks to make sure the IP +# address is actually on an interface. It just consults "ctdb ip". + +# This is a variation of the "addip" test. + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init + +select_test_node_and_ips + +echo "Running test against node $test_node and IP $test_ip" + +get_test_ip_mask_and_iface + +echo "Deleting IP $test_ip from all nodes" +delete_ip_from_all_nodes $test_ip +try_command_on_node -v $test_node $CTDB ipreallocate +wait_until_ips_are_on_node ! $test_node $test_ip + +try_command_on_node -v all $CTDB ip + +my_exit_hook () +{ + if ctdb_test_on_cluster ; then + onnode -q all $CTDB event script enable legacy "10.interface" + fi +} + +ctdb_test_exit_hook_add my_exit_hook + +# This forces us to wait until the ipreallocated associated with the +# delips is complete. +try_command_on_node $test_node $CTDB sync + +# Wait for a monitor event. Then the next steps are unlikely to occur +# in the middle of a monitor event and will have the expected effect. +wait_for_monitor_event $test_node + +if ctdb_test_on_cluster ; then + # Stop monitor events from bringing up the link status of an interface + try_command_on_node $test_node $CTDB event script disable legacy 10.interface +fi + +echo "Marking interface $iface down on node $test_node" +try_command_on_node $test_node $CTDB setifacelink $iface down + +echo "Adding IP $test_ip to node $test_node" +try_command_on_node $test_node $CTDB addip $test_ip/$mask $iface +try_command_on_node $test_node $CTDB ipreallocate + +echo "Wait long enough for IP verification to have taken place" +sleep_for 15 + +echo "Ensuring that IP ${test_ip} is not hosted on node ${test_node} when interface is down" +if ips_are_on_node '!' $test_node $test_ip; then + echo "GOOD: the IP has not been hosted while the interface is down" +else + echo "BAD: the IP is hosted but the interface is down" + exit 1 +fi + +echo "Marking interface $iface up on node $test_node" +try_command_on_node $test_node $CTDB setifacelink $iface up +wait_until_ips_are_on_node $test_node $test_ip |