summaryrefslogtreecommitdiffstats
path: root/ctdb/tests/INTEGRATION/failover
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-05 17:47:29 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-05 17:47:29 +0000
commit4f5791ebd03eaec1c7da0865a383175b05102712 (patch)
tree8ce7b00f7a76baa386372422adebbe64510812d4 /ctdb/tests/INTEGRATION/failover
parentInitial commit. (diff)
downloadsamba-4f5791ebd03eaec1c7da0865a383175b05102712.tar.xz
samba-4f5791ebd03eaec1c7da0865a383175b05102712.zip
Adding upstream version 2:4.17.12+dfsg.upstream/2%4.17.12+dfsgupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'ctdb/tests/INTEGRATION/failover')
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.001.list.sh48
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.010.addip.sh25
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.011.delip.sh16
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh117
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh44
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh51
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh76
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh23
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh21
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh71
-rwxr-xr-xctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh71
11 files changed, 563 insertions, 0 deletions
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.001.list.sh b/ctdb/tests/INTEGRATION/failover/pubips.001.list.sh
new file mode 100755
index 0000000..2fc75b7
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.001.list.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Verify that 'ctdb ip' shows the correct output
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+echo "Getting list of public IPs..."
+try_command_on_node -v 1 "$CTDB ip all | tail -n +2"
+ips=$(sed \
+ -e 's@ node\[@ @' \
+ -e 's@\].*$@@' \
+ "$outfile")
+machineout=$(sed -r \
+ -e 's@^| |$@\|@g' \
+ -e 's@[[:alpha:]]+\[@@g' \
+ -e 's@\]@@g' \
+ "$outfile")
+
+if ctdb_test_on_cluster ; then
+ while read ip pnn ; do
+ try_command_on_node $pnn "ip addr show to ${ip}"
+ if [ -n "$out" ] ; then
+ echo "GOOD: node $pnn appears to have $ip assigned"
+ else
+ die "BAD: node $pnn does not appear to have $ip assigned"
+ fi
+ done <<<"$ips" # bashism to avoid problem setting variable in pipeline.
+fi
+
+echo "Looks good!"
+
+cmd="$CTDB -X ip all | tail -n +2"
+echo "Checking that \"$cmd\" produces expected output..."
+
+try_command_on_node 1 "$cmd"
+if [ "$out" = "$machineout" ] ; then
+ echo "Yep, looks good!"
+else
+ echo "Nope, it looks like this:"
+ echo "$out"
+ echo "Should be like this:"
+ echo "$machineout"
+ exit 1
+fi
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.010.addip.sh b/ctdb/tests/INTEGRATION/failover/pubips.010.addip.sh
new file mode 100755
index 0000000..aba85dd
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.010.addip.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Verify that an IP address can be added to a node using 'ctdb addip'
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+select_test_node_and_ips
+get_test_ip_mask_and_iface
+
+echo "Deleting IP $test_ip from all nodes"
+delete_ip_from_all_nodes $test_ip
+try_command_on_node -v $test_node $CTDB ipreallocate
+wait_until_ips_are_on_node '!' $test_node $test_ip
+
+# Debugging...
+try_command_on_node -v all $CTDB ip
+
+echo "Adding IP ${test_ip}/${mask} on ${iface}, node ${test_node}"
+try_command_on_node $test_node $CTDB addip ${test_ip}/${mask} $iface
+try_command_on_node $test_node $CTDB ipreallocate
+wait_until_ips_are_on_node $test_node $test_ip
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.011.delip.sh b/ctdb/tests/INTEGRATION/failover/pubips.011.delip.sh
new file mode 100755
index 0000000..5235a9d
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.011.delip.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Verify that a node's public IP address can be deleted using 'ctdb deleteip'
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+select_test_node_and_ips
+
+echo "Deleting IP ${test_ip} from node ${test_node}"
+try_command_on_node $test_node $CTDB delip $test_ip
+try_command_on_node $test_node $CTDB ipreallocate
+wait_until_ips_are_on_node '!' $test_node $test_ip
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh b/ctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh
new file mode 100755
index 0000000..a3bb3af
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.012.reloadips.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+# Verify that IPs can be reconfigured using 'ctdb reloadips'
+
+# Various sub-tests that remove addresses from the public_addresses file
+# on a node or delete the entire contents of the public_addresses file.
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+select_test_node_and_ips
+
+try_command_on_node $test_node $CTDB_TEST_WRAPPER ctdb_base_show
+addresses="${out}/public_addresses"
+echo "Public addresses file on node $test_node is \"$addresses\""
+backup="${addresses}.$$"
+
+restore_public_addresses ()
+{
+ try_command_on_node $test_node "mv $backup $addresses >/dev/null 2>&1 || true"
+}
+ctdb_test_exit_hook_add restore_public_addresses
+
+# ctdb reloadips will fail if it can't disable takover runs. The most
+# likely reason for this is that there is already a takeover run in
+# progress. We can't predict when this will happen, so retry if this
+# occurs.
+do_ctdb_reloadips ()
+{
+ local retry_max=10
+ local retry_count=0
+ while : ; do
+ if ctdb_onnode "$test_node" "reloadips all" ; then
+ return 0
+ fi
+
+ if [ "$out" != "Failed to disable takeover runs" ] ; then
+ return 1
+ fi
+
+ if [ $retry_count -ge $retry_max ] ; then
+ return 1
+ fi
+
+ retry_count=$((retry_count + 1))
+ echo "Retrying..."
+ sleep_for 1
+ done
+}
+
+
+echo "Removing IP $test_ip from node $test_node"
+
+try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses"
+
+do_ctdb_reloadips
+
+try_command_on_node $test_node $CTDB ip
+
+if grep "^${test_ip} " <<<"$out" ; then
+ cat <<EOF
+BAD: node $test_node can still host IP $test_ip:
+$out
+EOF
+ exit 1
+fi
+
+cat <<EOF
+GOOD: node $test_node is no longer hosting IP $test_ip:
+$out
+EOF
+
+ctdb_onnode "$test_node" sync
+
+
+echo "Restoring addresses"
+restore_public_addresses
+
+do_ctdb_reloadips
+
+echo "Getting list of public IPs on node $test_node"
+try_command_on_node $test_node "$CTDB ip | tail -n +2"
+
+if [ -z "$out" ] ; then
+ echo "BAD: node $test_node has no ips"
+ exit 1
+fi
+
+cat <<EOF
+GOOD: node $test_node has these addresses:
+$out
+EOF
+
+ctdb_onnode "$test_node" sync
+
+
+echo "Emptying public addresses file on $test_node"
+
+try_command_on_node $test_node "mv $addresses $backup && touch $addresses"
+
+do_ctdb_reloadips
+
+echo "Getting list of public IPs on node $test_node"
+try_command_on_node $test_node "$CTDB ip | tail -n +2"
+
+if [ -n "$out" ] ; then
+ cat <<EOF
+BAD: node $test_node still has ips:
+$out
+EOF
+ exit 1
+fi
+
+echo "GOOD: no IPs left on node $test_node"
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh b/ctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh
new file mode 100755
index 0000000..77f9a63
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.013.failover_noop.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Check that CTDB operates correctly if:
+
+# * failover is disabled; or
+# * there are 0 public IPs configured
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_skip_on_cluster
+
+ctdb_test_init -n
+
+echo "Starting CTDB with failover disabled..."
+ctdb_nodes_start_custom -F
+
+select_test_node
+
+echo "Getting IP allocation..."
+
+# $test_node set above by select_test_node()
+# shellcheck disable=SC2154
+try_command_on_node -v "$test_node" "$CTDB ip all | tail -n +2"
+
+while read ip pnn ; do
+ if [ "$pnn" != "-1" ] ; then
+ die "BAD: IP address ${ip} is assigned to node ${pnn}"
+ fi
+done <"$outfile"
+
+echo "GOOD: All IP addresses are unassigned"
+
+echo "----------------------------------------"
+
+echo "Starting CTDB with an empty public addresses configuration..."
+ctdb_nodes_start_custom -P /dev/null
+
+echo "Trying explicit ipreallocate..."
+ctdb_onnode "$test_node" ipreallocate
+
+echo "Good, that seems to work!"
+echo
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh b/ctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh
new file mode 100755
index 0000000..845b4b5
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.014.iface_gc.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Verify that an interface is deleted when all IPs on it are deleted
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+select_test_node_and_ips
+
+# Find interfaces on test node
+try_command_on_node $test_node "$CTDB ifaces -X"
+ifaces=$(awk -F'|' 'NR > 1 { print $2 }' "$outfile")
+echo "Node ${test_node} has interfaces: ${ifaces}"
+
+# Delete all IPs on each interface... deleting IPs from one interface
+# can cause other interfaces to disappear, so we need to be careful...
+for i in $ifaces ; do
+ try_command_on_node $test_node "$CTDB ifaces -X"
+ info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
+
+ if [ -z "$info" ] ; then
+ echo "Interface ${i} missing... assuming already deleted!"
+ continue
+ fi
+
+ echo "Deleting IPs on interface ${i}, with this information:"
+ echo " $info"
+
+ try_command_on_node $test_node "$CTDB ip -v -X | tail -n +2"
+ awk -F'|' -v i="$i" \
+ '$6 == i { print $2 }' "$outfile" |
+ while read ip ; do
+ echo " $ip"
+ try_command_on_node $test_node "$CTDB delip $ip"
+ done
+ try_command_on_node $test_node "$CTDB ipreallocate"
+
+ try_command_on_node $test_node "$CTDB ifaces -X"
+ info=$(awk -F'|' -v iface="$i" '$2 == iface { print $0 }' "$outfile")
+
+ if [ -z "$info" ] ; then
+ echo "GOOD: Interface ${i} has been garbage collected"
+ else
+ echo "BAD: Interface ${i} still exists"
+ echo "$out"
+ exit 1
+ fi
+done
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh b/ctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh
new file mode 100755
index 0000000..8daf3f5
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.020.moveip.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Verify that 'ctdb moveip' allows movement of public IPs between nodes
+
+# This test does not do any network level checks to make sure IP
+# addresses are actually on interfaces. It just consults "ctdb ip".
+
+# To work, this test ensures that IPAllocAlgorithm is not set to 0
+# (Deterministic IPs) and sets NoIPFailback.
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+select_test_node_and_ips
+
+sanity_check_ips ()
+{
+ echo "Sanity checking IPs..."
+
+ local x ipp prev
+ prev=""
+ while read x ipp ; do
+ [ "$ipp" = "-1" ] && break
+ if [ -n "$prev" -a "$ipp" != "$prev" ] ; then
+ echo "OK"
+ return 0
+ fi
+ prev="$ipp"
+ done <"$outfile"
+
+ echo "BAD: a node was -1 or IPs are only assigned to one node:"
+ cat "$outfile"
+ echo "Are you running an old version of CTDB?"
+ return 1
+}
+
+sanity_check_ips
+
+# Find a target node - it must be willing to host $test_ip
+
+# $test_node set above by select_test_node_and_ips()
+# shellcheck disable=SC2154
+try_command_on_node "$test_node" "$CTDB listnodes | wc -l"
+num_nodes="$out"
+to_node=""
+for i in $(seq 0 $(($num_nodes - 1)) ) ; do
+ [ $i -ne $test_node ] || continue
+ all_ips_on_node $i
+ while read ip x ; do
+ if [ "$ip" = "$test_ip" ] ; then
+ to_node="$i"
+ break 2
+ fi
+ done <"$outfile"
+done
+
+if [ -z "$to_node" ] ; then
+ echo "Unable to find target node"
+ exit 1
+fi
+
+echo "Target node is ${to_node}"
+
+echo "Setting IPAllocAlgorithm=2 to avoid Deterministic IPs..."
+try_command_on_node -q all $CTDB setvar IPAllocAlgorithm 2
+
+echo "Turning on NoIPFailback..."
+try_command_on_node -q all $CTDB setvar NoIPFailback 1
+
+echo "Attempting to move ${test_ip} from node ${test_node} to node ${to_node}"
+try_command_on_node $test_node $CTDB moveip $test_ip $to_node
+wait_until_ips_are_on_node '!' $test_node $test_ip
+wait_until_ips_are_on_node $to_node $test_ip
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh b/ctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh
new file mode 100755
index 0000000..3f40097
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.030.disable_enable.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Verify the operation of "ctdb disable" and "ctdb enable"
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+########################################
+
+select_test_node_and_ips
+
+echo "Disabling node $test_node"
+try_command_on_node 1 $CTDB disable -n $test_node
+wait_until_node_has_status $test_node disabled 30 all
+wait_until_node_has_no_ips "$test_node"
+
+echo "Re-enabling node $test_node"
+try_command_on_node 1 $CTDB enable -n $test_node
+wait_until_node_has_status $test_node enabled 30 all
+wait_until_node_has_some_ips "$test_node"
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh b/ctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh
new file mode 100755
index 0000000..f5936b0
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.032.stop_continue.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Verify the operation of "ctdb stop" and "ctdb continue"
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+select_test_node_and_ips
+
+echo "Stopping node ${test_node}..."
+try_command_on_node 1 $CTDB stop -n $test_node
+wait_until_node_has_status $test_node stopped
+wait_until_node_has_no_ips "$test_node"
+
+echo "Continuing node $test_node"
+try_command_on_node 1 $CTDB continue -n $test_node
+wait_until_node_has_status $test_node notstopped
+wait_until_node_has_some_ips "$test_node"
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh b/ctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh
new file mode 100755
index 0000000..e99a265
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.040.NoIPTakeover.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Verify that 'ctdb setvar NoIPTakeover 1' stops IP addresses being taken over
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+ctdb_get_all_pnns
+# out is set above
+# shellcheck disable=SC2154
+num_nodes=$(echo "$out" | wc -l | tr -d '[:space:]')
+echo "There are $num_nodes nodes..."
+
+if [ "$num_nodes" -lt 2 ] ; then
+ echo "Less than 2 nodes!"
+ exit 1
+fi
+
+select_test_node_and_ips
+
+
+# sets: num
+count_ips_on_node ()
+{
+ local node="$1"
+
+ ctdb_onnode "$node" ip
+ # outfile is set by ctdb_onnode() above
+ # shellcheck disable=SC2154,SC2126
+ # * || true is needed to avoid command failure when there are no matches
+ # * Using "wc -l | tr -d '[:space:]'" is our standard
+ # pattern... and "grep -c" requires handling of special case
+ # for no match
+ num=$(grep -v 'Public' "$outfile" | \
+ grep " ${node}\$" | \
+ wc -l | \
+ tr -d '[:space:]')
+ echo "Number of addresses on node ${node}: ${num}"
+}
+
+
+# test_node is set by select_test_node_and_ips() above
+# shellcheck disable=SC2154
+count_ips_on_node "$test_node"
+
+echo "Turning on NoIPTakeover on all nodes"
+ctdb_onnode all "setvar NoIPTakeover 1"
+ctdb_onnode "$test_node" ipreallocate
+
+echo "Disable node ${test_node}"
+ctdb_onnode "$test_node" disable
+
+count_ips_on_node "$test_node"
+if [ "$num" != "0" ] ; then
+ test_fail "BAD: node 1 still hosts IP addresses"
+fi
+
+
+echo "Enable node 1 again"
+ctdb_onnode "$test_node" enable
+
+count_ips_on_node "$test_node"
+if [ "$num" != "0" ] ; then
+ test_fail "BAD: node 1 took over IP addresses"
+fi
+
+
+echo "OK: IP addresses were not taken over"
diff --git a/ctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh b/ctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh
new file mode 100755
index 0000000..543f9a9
--- /dev/null
+++ b/ctdb/tests/INTEGRATION/failover/pubips.050.missing_ip.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Verify that the recovery daemon handles unhosted IPs properly
+
+# This test does not do any network level checks to make sure the IP
+# address is actually on an interface. It just consults "ctdb ip".
+
+# This is a variation of the "addip" test.
+
+. "${TEST_SCRIPTS_DIR}/integration.bash"
+
+set -e
+
+ctdb_test_init
+
+select_test_node_and_ips
+
+echo "Running test against node $test_node and IP $test_ip"
+
+get_test_ip_mask_and_iface
+
+echo "Deleting IP $test_ip from all nodes"
+delete_ip_from_all_nodes $test_ip
+try_command_on_node -v $test_node $CTDB ipreallocate
+wait_until_ips_are_on_node ! $test_node $test_ip
+
+try_command_on_node -v all $CTDB ip
+
+my_exit_hook ()
+{
+ if ctdb_test_on_cluster ; then
+ onnode -q all $CTDB event script enable legacy "10.interface"
+ fi
+}
+
+ctdb_test_exit_hook_add my_exit_hook
+
+# This forces us to wait until the ipreallocated associated with the
+# delips is complete.
+try_command_on_node $test_node $CTDB sync
+
+# Wait for a monitor event. Then the next steps are unlikely to occur
+# in the middle of a monitor event and will have the expected effect.
+wait_for_monitor_event $test_node
+
+if ctdb_test_on_cluster ; then
+ # Stop monitor events from bringing up the link status of an interface
+ try_command_on_node $test_node $CTDB event script disable legacy 10.interface
+fi
+
+echo "Marking interface $iface down on node $test_node"
+try_command_on_node $test_node $CTDB setifacelink $iface down
+
+echo "Adding IP $test_ip to node $test_node"
+try_command_on_node $test_node $CTDB addip $test_ip/$mask $iface
+try_command_on_node $test_node $CTDB ipreallocate
+
+echo "Wait long enough for IP verification to have taken place"
+sleep_for 15
+
+echo "Ensuring that IP ${test_ip} is not hosted on node ${test_node} when interface is down"
+if ips_are_on_node '!' $test_node $test_ip; then
+ echo "GOOD: the IP has not been hosted while the interface is down"
+else
+ echo "BAD: the IP is hosted but the interface is down"
+ exit 1
+fi
+
+echo "Marking interface $iface up on node $test_node"
+try_command_on_node $test_node $CTDB setifacelink $iface up
+wait_until_ips_are_on_node $test_node $test_ip