diff options
Diffstat (limited to 'ctdb/tests/UNIT/eventscripts/scripts')
19 files changed, 1917 insertions, 0 deletions
diff --git a/ctdb/tests/UNIT/eventscripts/scripts/00.ctdb.sh b/ctdb/tests/UNIT/eventscripts/scripts/00.ctdb.sh new file mode 100644 index 0000000..a192e05 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/00.ctdb.sh @@ -0,0 +1,24 @@ +setup() +{ + setup_dbdir + setup_date + + export FAKE_TDBTOOL_SUPPORTS_CHECK="yes" + export FAKE_TDB_IS_OK="yes" + + export FAKE_CTDB_TUNABLES_OK=" + MonitorInterval + DatabaseHashSize + " + export FAKE_CTDB_TUNABLES_OBSOLETE=" + EventScriptUnhealthyOnTimeout + " +} + +result_filter() +{ + _date="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" + _time="[0-9][0-9][0-9][0-9][0-9][0-9]" + _date_time="${_date}\.${_time}" + sed -e "s|\.${_date_time}\.|.DATE.TIME.|" +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/01.reclock.sh b/ctdb/tests/UNIT/eventscripts/scripts/01.reclock.sh new file mode 100644 index 0000000..7365dd8 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/01.reclock.sh @@ -0,0 +1,16 @@ +setup() +{ + if [ $# -eq 1 ]; then + reclock="$1" + else + reclock="${CTDB_TEST_TMP_DIR}/reclock_subdir/rec.lock" + fi + CTDB_RECOVERY_LOCK="$reclock" + + if [ -n "$CTDB_RECOVERY_LOCK" ]; then + cat >>"${CTDB_BASE}/ctdb.conf" <<EOF +[cluster] + recovery lock = $CTDB_RECOVERY_LOCK +EOF + fi +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/05.system.sh b/ctdb/tests/UNIT/eventscripts/scripts/05.system.sh new file mode 100644 index 0000000..0191e55 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/05.system.sh @@ -0,0 +1,48 @@ +# shellcheck disable=SC2120 +# Arguments used in testcases +set_mem_usage() +{ + _mem_usage="${1:-10}" # Default is 10% + _swap_usage="${2:-0}" # Default is 0% + + _swap_total=5857276 + _swap_free=$(((100 - _swap_usage) * _swap_total / 100)) + + _mem_total=3940712 + _mem_free=225268 + _mem_buffers=146120 + _mem_cached=$((_mem_total * (100 - _mem_usage) / 100 - \ + _mem_free - _mem_buffers)) + + export FAKE_PROC_MEMINFO="\ +MemTotal: ${_mem_total} kB +MemFree: ${_mem_free} kB +Buffers: ${_mem_buffers} kB +Cached: ${_mem_cached} kB +SwapCached: 56016 kB +Active: 2422104 kB +Inactive: 1019928 kB +Active(anon): 1917580 kB +Inactive(anon): 523080 kB +Active(file): 504524 kB +Inactive(file): 496848 kB +Unevictable: 4844 kB +Mlocked: 4844 kB +SwapTotal: ${_swap_total} kB +SwapFree: ${_swap_free} kB +..." +} + +set_fs_usage() +{ + export FAKE_FS_USE="${1:-10}" # Default is 10% usage +} + +setup() +{ + setup_dbdir + + # Tests use default unless explicitly set + set_mem_usage + set_fs_usage +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/06.nfs.sh b/ctdb/tests/UNIT/eventscripts/scripts/06.nfs.sh new file mode 100644 index 0000000..5912a4b --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/06.nfs.sh @@ -0,0 +1,4 @@ +setup() +{ + : +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/10.interface.sh b/ctdb/tests/UNIT/eventscripts/scripts/10.interface.sh new file mode 100644 index 0000000..579f3ee --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/10.interface.sh @@ -0,0 +1,72 @@ +setup() +{ + setup_public_addresses +} + +_tcp_connections() +{ + _count="$1" + _sip="$2" + _sport="$3" + _cip_base="$4" + _cport_base="$5" + + _cip_prefix="${_cip_base%.*}" + _cip_suffix="${_cip_base##*.}" + + for _i in $(seq 1 "$_count"); do + _cip_last=$((_cip_suffix + _i)) + _cip="${_cip_prefix}.${_cip_last}" + _cport=$((_cport_base + _i)) + echo "${_sip}:${_sport} ${_cip}:${_cport}" + done +} + +setup_tcp_connections() +{ + _t="${FAKE_NETWORK_STATE}/tcp-established" + export FAKE_NETSTAT_TCP_ESTABLISHED_FILE="$_t" + _tcp_connections "$@" >"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" +} + +setup_tcp_connections_unkillable() +{ + # These connections are listed by the "ss" stub but are not + # killed by the "ctdb killtcp" stub. So killing these + # connections will never succeed... and will look like a time + # out. + _t=$(_tcp_connections "$@" | sed -e 's/ /|/g') + export FAKE_NETSTAT_TCP_ESTABLISHED="$_t" +} + +# Setup some fake /proc/net/bonding files with just enough info for +# the eventscripts. + +# arg1 is interface name, arg2 is currently active slave (use "None" +# if none), arg3 is MII status ("up" or "down"). +setup_bond() +{ + _iface="$1" + _slave="${2:-${_iface}_sl_0}" + _mii_s="${3:-up}" + _mii_subs="${4:-${_mii_s:-up}}" + + cat <<EOF +Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s +EOF + + _t="${FAKE_NETWORK_STATE}/proc-net-bonding" + export FAKE_PROC_NET_BONDING="$_t" + mkdir -p "$FAKE_PROC_NET_BONDING" + + cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF +Bonding Mode: IEEE 802.3ad Dynamic link aggregation +Currently Active Slave: $_slave +# Status of the bond +MII Status: $_mii_s +# Status of 1st pretend adapter +MII Status: $_mii_subs +# Status of 2nd pretend adapter +MII Status: $_mii_subs +EOF +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/11.natgw.sh b/ctdb/tests/UNIT/eventscripts/scripts/11.natgw.sh new file mode 100644 index 0000000..3b19895 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/11.natgw.sh @@ -0,0 +1,120 @@ +setup() +{ + debug "Setting up NAT gateway" + + natgw_nodes="${CTDB_BASE}/natgw_nodes" + + ctdb_set_pnn +} + +# A separate function for this makes sense because it can be done +# multiple times per test +setup_ctdb_natgw() +{ + # Read from stdin + while read -r _ip _opts; do + case "$_opts" in + leader) + export FAKE_CTDB_NATGW_LEADER="$_ip" + echo "$_ip" + ;; + follower-only) + printf "%s\tfollower-only\n" "$_ip" + ;; + *) + echo "$_ip" + ;; + esac + done >"$natgw_nodes" + + # Assume all of the nodes are on a /24 network and have IPv4 + # addresses: + read -r _ip <"$natgw_nodes" + + setup_script_options <<EOF +CTDB_NATGW_NODES="$natgw_nodes" +CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24" +# These are fixed. Probably don't use the same network for the +# private node IPs. To unset the default gateway just set it to +# "". :-) +CTDB_NATGW_PUBLIC_IP="10.1.1.121/24" +CTDB_NATGW_PUBLIC_IFACE="eth1" +CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254" +EOF +} + +ok_natgw_leader_ip_addr_show() +{ + _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | + cksum | + sed -r -e 's@(..)(..)(..).*@fe:fe:fe:\1:\2:\3@') + + # This is based on CTDB_NATGW_PUBLIC_IP + _brd="10.1.1.255" + + ok <<EOF +1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether ${_mac} brd ff:ff:ff:ff:ff:ff + inet ${CTDB_NATGW_PUBLIC_IP} brd ${_brd} scope global ${CTDB_NATGW_PUBLIC_IFACE} + valid_lft forever preferred_lft forever +EOF +} + +ok_natgw_follower_ip_addr_show() +{ + _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | + cksum | + sed -r -e 's@(..)(..)(..).*@fe:fe:fe:\1:\2:\3@') + + ok <<EOF +1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether ${_mac} brd ff:ff:ff:ff:ff:ff +EOF +} + +ok_natgw_leader_static_routes() +{ + _nl=" +" + _t="" + for _i in $CTDB_NATGW_STATIC_ROUTES; do + # This is intentionally different to the code in 11.natgw ;-) + case "$_i" in + *@*) + _net=$(echo "$_i" | sed -e 's|@.*||') + _gw=$(echo "$_i" | sed -e 's|.*@||') + ;; + *) + _net="$_i" + _gw="$CTDB_NATGW_DEFAULT_GATEWAY" + ;; + esac + + [ -n "$_gw" ] || continue + _t="${_t}${_t:+${_nl}}" + _t="${_t}${_net} via ${_gw} dev ethXXX metric 10 " + done + _t=$(echo "$_t" | sort) + ok "$_t" +} + +ok_natgw_follower_static_routes() +{ + _nl=" +" + _t="" + for _i in $CTDB_NATGW_STATIC_ROUTES; do + # This is intentionally different to the code in 11.natgw ;-) + _net=$(echo "$_i" | sed -e 's|@.*||') + + # The interface for the private network isn't + # specified as part of the NATGW configuration and + # isn't part of the command to add the route. It is + # implicitly added by "ip route" but our stub doesn't + # do this and adds "ethXXX". + _t="${_t}${_t:+${_nl}}" + _t="${_t}${_net} via ${FAKE_CTDB_NATGW_LEADER} dev ethXXX metric 10 " + done + _t=$(echo "$_t" | sort) + ok "$_t" +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/13.per_ip_routing.sh b/ctdb/tests/UNIT/eventscripts/scripts/13.per_ip_routing.sh new file mode 100644 index 0000000..aac2c3d --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/13.per_ip_routing.sh @@ -0,0 +1,47 @@ +setup() +{ + setup_public_addresses + + # shellcheck disable=SC2034 + # Used in expected output + service_name="per_ip_routing" + + setup_script_options <<EOF +CTDB_PER_IP_ROUTING_CONF="${CTDB_BASE}/policy_routing" +CTDB_PER_IP_ROUTING_RULE_PREF=100 +CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000 +CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000 +EOF + + # Tests need to create and populate this file + rm -f "$CTDB_PER_IP_ROUTING_CONF" +} + +# Create policy routing configuration in $CTDB_PER_IP_ROUTING_CONF. +# $1 is the number of assigned IPs to use (<num>, all), defaulting to +# 1. If $2 is "default" then a default route is also added. +create_policy_routing_config() +{ + _num_ips="${1:-1}" + _should_add_default="$2" + + ctdb_get_my_public_addresses | + if [ "$_num_ips" = "all" ]; then + cat + else + { + head -n "$_num_ips" + cat >/dev/null + } + fi | + while read -r _dev _ip _bits; do + _net=$(ipv4_host_addr_to_net "$_ip" "$_bits") + _gw="${_net%.*}.254" # a dumb, calculated default + + echo "$_ip $_net" + + if [ "$_should_add_default" = "default" ]; then + echo "$_ip 0.0.0.0/0 $_gw" + fi + done >"$CTDB_PER_IP_ROUTING_CONF" +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/20.multipathd.sh b/ctdb/tests/UNIT/eventscripts/scripts/20.multipathd.sh new file mode 100644 index 0000000..9add0bc --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/20.multipathd.sh @@ -0,0 +1,25 @@ +setup() +{ + _failures="" + _devices="" + for i; do + case "$i" in + \!*) + _t="${i#!}" + echo "Marking ${_t} as having no active paths" + _failures="${_failures}${_failures:+ }${_t}" + ;; + *) + _t="$i" + ;; + esac + _devices="${_devices}${_devices:+ }${_t}" + done + + setup_script_options <<EOF +CTDB_MONITOR_MPDEVICES="$_devices" +EOF + + export FAKE_MULTIPATH_FAILURES="$_failures" + export FAKE_SLEEP_FORCE=0.1 +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/31.clamd.sh b/ctdb/tests/UNIT/eventscripts/scripts/31.clamd.sh new file mode 100644 index 0000000..27016cb --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/31.clamd.sh @@ -0,0 +1,8 @@ +setup() +{ + setup_script_options <<EOF +CTDB_CLAMD_SOCKET="/var/run/clamd.sock" +EOF + + setup_unix_listen +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/40.vsftpd.sh b/ctdb/tests/UNIT/eventscripts/scripts/40.vsftpd.sh new file mode 100644 index 0000000..236d130 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/40.vsftpd.sh @@ -0,0 +1,14 @@ +setup() +{ + debug "Setting up VSFTPD environment: service $1, not managed by CTDB" + + _service_name="vsftpd" + + if [ "$1" != "down" ]; then + service "$_service_name" start + setup_tcp_listen 21 + else + service "$_service_name" force-stopped + setup_tcp_listen "" + fi +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/41.httpd.sh b/ctdb/tests/UNIT/eventscripts/scripts/41.httpd.sh new file mode 100644 index 0000000..3fac4f0 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/41.httpd.sh @@ -0,0 +1,14 @@ +setup() +{ + debug "Setting up HTTPD environment: service $1, not managed by CTDB" + + if [ "$1" != "down" ]; then + for _service_name in "apache2" "httpd"; do + service "$_service_name" start + done + else + for _service_name in "apache2" "httpd"; do + service "$_service_name" force-stopped + done + fi +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/48.netbios.sh b/ctdb/tests/UNIT/eventscripts/scripts/48.netbios.sh new file mode 100644 index 0000000..6efcd8a --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/48.netbios.sh @@ -0,0 +1,23 @@ +setup() +{ + # shellcheck disable=SC2034 + # Used in expected output + service_name="netbios" + + if [ "$1" != "down" ]; then + + debug "Marking Netbios name services as up, listening and managed by CTDB" + + # All possible service names for all known distros. + for i in "nmb" "nmbd"; do + service "$i" force-started + done + else + debug "Marking Netbios name services as down, not listening and not managed by CTDB" + + # All possible service names for all known distros. + for i in "nmb" "nmbd"; do + service "$i" force-stopped + done + fi +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/49.winbind.sh b/ctdb/tests/UNIT/eventscripts/scripts/49.winbind.sh new file mode 100644 index 0000000..bbe1de2 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/49.winbind.sh @@ -0,0 +1,28 @@ +setup() +{ + # shellcheck disable=SC2034 + # Used in expected output + service_name="winbind" + + if [ "$1" != "down" ]; then + + debug "Marking Winbind service as up and managed by CTDB" + + service "winbind" force-started + + export FAKE_WBINFO_FAIL="no" + + else + debug "Marking Winbind service as down and not managed by CTDB" + + service "winbind" force-stopped + + export FAKE_WBINFO_FAIL="yes" + fi +} + +wbinfo_down() +{ + debug "Making wbinfo commands fail" + FAKE_WBINFO_FAIL="yes" +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/50.samba.sh b/ctdb/tests/UNIT/eventscripts/scripts/50.samba.sh new file mode 100644 index 0000000..88af69b --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/50.samba.sh @@ -0,0 +1,58 @@ +setup() +{ + # shellcheck disable=SC2034 + # Used in expected output + service_name="samba" + + if [ "$1" != "down" ]; then + + debug "Marking Samba services as up, listening and managed by CTDB" + + # All possible service names for all known distros. + for i in "smb" "samba" "smbd"; do + service "$i" force-started + done + + setup_tcp_listen 445 139 + + # Some things in 50.samba are backgrounded and waited + # for. If we don't sleep at all then timeouts can + # happen. This avoids that... :-) + export FAKE_SLEEP_FORCE=0.1 + else + debug "Marking Samba services as down, not listening and not managed by CTDB" + + # All possible service names for all known distros. + for i in "smb" "samba" "smbd"; do + service "$i" force-stopped + done + + setup_tcp_listen + fi + + setup_script_options <<EOF +CTDB_SAMBA_SKIP_SHARE_CHECK="no" +EOF + + setup_shares + +} + +samba_setup_fake_threads() +{ + export FAKE_SMBD_THREAD_PIDS="$*" + + _nl=" +" + _out="" + _count=0 + for _pid; do + [ "$_count" -lt 5 ] || break + _t=$(program_stack_trace "smbd" "$_pid") + _out="${_out:+${_out}${_nl}}${_t}" + _count=$((_count + 1)) + done + # shellcheck disable=SC2034 + # Used in expected output + SAMBA_STACK_TRACES="$_out" +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh b/ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh new file mode 100644 index 0000000..9c614c7 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh @@ -0,0 +1,435 @@ +setup() +{ + setup_public_addresses + setup_shares + + # shellcheck disable=SC2034 + # Used in expected output + service_name="nfs" + + if [ -z "$CTDB_NFS_DISTRO_STYLE" ]; then + # Currently supported: sysvinit-redhat, systemd-redhat + CTDB_NFS_DISTRO_STYLE="systemd-redhat" + fi + + export FAKE_RPCINFO_SERVICES="" + + setup_script_options <<EOF +CTDB_NFS_SKIP_SHARE_CHECK="no" +# This doesn't even need to exist +CTDB_NFS_EXPORTS_FILE="${CTDB_TEST_TMP_DIR}/etc-exports" +EOF + + export RPCNFSDCOUNT + + if [ "$1" != "down" ]; then + debug <<EOF +Setting up NFS environment: all RPC services up, NFS managed by CTDB +EOF + + case "$CTDB_NFS_DISTRO_STYLE" in + sysvinit-*) + service "nfs" force-started + service "nfslock" force-started + ;; + systemd-*) + service "nfs-service" force-started + service "nfs-mountd" force-started + service "rpc-rquotad" force-started + service "rpc-statd" force-started + ;; + esac + + rpc_services_up \ + "portmapper" "nfs" "mountd" "rquotad" \ + "nlockmgr" "status" + + nfs_setup_fake_threads "nfsd" + nfs_setup_fake_threads "rpc.foobar" # Set the variable to empty + else + debug <<EOF +Setting up NFS environment: all RPC services down, NFS not managed by CTDB +EOF + + case "$CTDB_NFS_DISTRO_STYLE" in + sysvinit-*) + service "nfs" force-stopped + service "nfslock" force-stopped + service "nfs-kernel-server" force-stopped + ;; + systemd-*) + service "nfs-server" force-stopped + service "nfs-mountd" force-stopped + service "rpc-quotad" force-stopped + service "rpc-statd" force-stopped + ;; + esac + fi + + # This is really nasty. However, when we test NFS we don't + # actually test statd-callout. If we leave it there then left + # over, backgrounded instances of statd-callout will do + # horrible things with the "ctdb ip" stub and cause the actual + # statd-callout tests that follow to fail. + rm "${CTDB_BASE}/statd-callout" +} + +rpc_services_down() +{ + _out="" + for _s in $FAKE_RPCINFO_SERVICES; do + for _i; do + if [ "$_i" = "${_s%%:*}" ]; then + debug "Marking RPC service \"${_i}\" as UNAVAILABLE" + continue 2 + fi + done + _out="${_out}${_out:+ }${_s}" + done + FAKE_RPCINFO_SERVICES="$_out" +} + +rpc_services_up() +{ + _out="$FAKE_RPCINFO_SERVICES" + for _i; do + debug "Marking RPC service \"${_i}\" as available" + case "$_i" in + portmapper) _t="2:4" ;; + nfs) _t="2:3" ;; + mountd) _t="1:3" ;; + rquotad) _t="1:2" ;; + nlockmgr) _t="3:4" ;; + status) _t="1:1" ;; + *) die "Internal error - unsupported RPC service \"${_i}\"" ;; + esac + + _out="${_out}${_out:+ }${_i}:${_t}" + done + export FAKE_RPCINFO_SERVICES="$_out" +} + +nfs_setup_fake_threads() +{ + _prog="$1" + shift + + case "$_prog" in + nfsd) + export PROCFS_PATH="${CTDB_TEST_TMP_DIR}/proc" + _threads="${PROCFS_PATH}/fs/nfsd/threads" + mkdir -p "$(dirname "$_threads")" + echo $# >"$_threads" + export FAKE_NFSD_THREAD_PIDS="$*" + ;; + *) + export FAKE_RPC_THREAD_PIDS="$*" + ;; + esac +} + +guess_output() +{ + case "$1" in + "${CTDB_NFS_CALLOUT} start nlockmgr") + case "$CTDB_NFS_DISTRO_STYLE" in + sysvinit-redhat) + echo "&Starting nfslock: OK" + ;; + sysvinit-debian) + cat <<EOF +&Starting nfs-kernel-server: OK +EOF + ;; + systemd-*) + echo "&Starting rpc-statd: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} start nfs") + case "$CTDB_NFS_DISTRO_STYLE" in + sysvinit-redhat) + cat <<EOF +&Starting nfslock: OK +&Starting nfs: OK +EOF + ;; + sysvinit-debian) + cat <<EOF +&Starting nfs-kernel-server: OK +EOF + ;; + systemd-redhat) + cat <<EOF +&Starting rpc-statd: OK +&Starting nfs-server: OK +&Starting rpc-rquotad: OK +EOF + ;; + systemd-debian) + cat <<EOF +&Starting rpc-statd: OK +&Starting nfs-server: OK +&Starting quotarpc: OK +EOF + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} stop mountd") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-*) + echo "Stopping nfs-mountd: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} stop rquotad") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-redhat) + echo "Stopping rpc-rquotad: OK" + ;; + systemd-debian) + if service "quotarpc" status >/dev/null; then + echo "Stopping quotarpc: OK" + else + echo "service: can't stop quotarpc - not running" + fi + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} stop status") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-*) + echo "Stopping rpc-statd: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} start mountd") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-*) + echo "&Starting nfs-mountd: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} start rquotad") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-redhat) + echo "&Starting rpc-rquotad: OK" + ;; + systemd-debian) + echo "&Starting quotarpc: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} start status") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-*) + echo "&Starting rpc-statd: OK" + ;; + esac + ;; + *) + : # Nothing + ;; + esac +} + +# Set the required result for a particular RPC program having failed +# for a certain number of iterations. This is probably still a work +# in progress. Note that we could hook aggressively +# nfs_check_rpc_service() to try to implement this but we're better +# off testing nfs_check_rpc_service() using independent code... even +# if it is incomplete and hacky. So, if the 60.nfs eventscript +# changes and the tests start to fail then it may be due to this +# function being incomplete. +rpc_set_service_failure_response() +{ + _rpc_service="$1" + _numfails="${2:-1}" # default 1 + + # Default + ok_null + if [ "$_numfails" -eq 0 ]; then + return + fi + + nfs_load_config + + # A handy newline. :-) + _nl=" +" + + _dir="${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}" + + _file=$(ls "$_dir"/[0-9][0-9]."${_rpc_service}.check") + [ -r "$_file" ] || + die "RPC check file \"$_file\" does not exist or is not unique" + + _out="${CTDB_TEST_TMP_DIR}/rpc_failure_output" + : >"$_out" + _rc_file="${CTDB_TEST_TMP_DIR}/rpc_result" + + ( + # Subshell to restrict scope variables... + + # Defaults + # shellcheck disable=SC2034 + # Unused, but for completeness, possible future use + family="tcp" + version="" + unhealthy_after=1 + restart_every=0 + service_stop_cmd="" + service_start_cmd="" + # shellcheck disable=SC2034 + # Unused, but for completeness, possible future use + service_check_cmd="" + service_debug_cmd="" + + # Don't bother syntax checking, eventscript does that... + . "$_file" + + # Just use the first version, or use default. This is + # dumb but handles all the cases that we care about + # now... + if [ -n "$version" ]; then + _ver="${version%% *}" + else + case "$_rpc_service" in + portmapper) _ver="" ;; + *) _ver=1 ;; + esac + fi + _rpc_check_out="\ +$_rpc_service failed RPC check: +rpcinfo: RPC: Program not registered +program $_rpc_service${_ver:+ version }${_ver} is not available" + + if [ $unhealthy_after -gt 0 ] && + [ "$_numfails" -ge $unhealthy_after ]; then + _unhealthy=true + echo 1 >"$_rc_file" + echo "ERROR: ${_rpc_check_out}" >>"$_out" + else + _unhealthy=false + echo 0 >"$_rc_file" + fi + + if [ $restart_every -gt 0 ] && + [ $((_numfails % restart_every)) -eq 0 ]; then + if ! $_unhealthy; then + echo "WARNING: ${_rpc_check_out}" >>"$_out" + fi + + echo "Trying to restart service \"${_rpc_service}\"..." \ + >>"$_out" + + guess_output "$service_stop_cmd" >>"$_out" + + if [ -n "$service_debug_cmd" ]; then + $service_debug_cmd >>"$_out" 2>&1 + fi + + guess_output "$service_start_cmd" >>"$_out" + fi + ) + + read -r _rc <"$_rc_file" + required_result "$_rc" <"$_out" + + rm -f "$_out" "$_rc_file" +} + +program_stack_traces() +{ + _prog="$1" + _max="${2:-1}" + + _count=1 + if [ "$_prog" = "nfsd" ]; then + _pids="$FAKE_NFSD_THREAD_PIDS" + else + _pids="$FAKE_RPC_THREAD_PIDS" + fi + for _pid in $_pids; do + [ $_count -le "$_max" ] || break + + program_stack_trace "$_prog" "$_pid" + _count=$((_count + 1)) + done +} + +# Run an NFS eventscript iteratively. +# +# - 1st argument is the number of iterations. +# +# - 2nd argument is the NFS/RPC service being tested +# +# rpcinfo is used on each iteration to test the availability of the +# service +# +# If this is not set or null then no RPC service is checked and the +# required output is not reset on each iteration. This is useful in +# baseline tests to confirm that the eventscript and test +# infrastructure is working correctly. +# +# - Subsequent arguments come in pairs: an iteration number and +# something to eval before that iteration. Each time an iteration +# number is matched the associated argument is given to eval after +# the default setup is done. The iteration numbers need to be given +# in ascending order. +# +# These arguments can allow a service to be started or stopped +# before a particular iteration. +# +nfs_iterate_test() +{ + _repeats="$1" + _rpc_service="$2" + if [ -n "$2" ]; then + shift 2 + else + shift + fi + + # shellcheck disable=SC2154 + # Variables defined in define_test() + echo "Running $_repeats iterations of \"$script $event\" $args" + + _iterate_failcount=0 + for _iteration in $(seq 1 "$_repeats"); do + # This is not a numerical comparison because $1 will + # often not be set. + if [ "$_iteration" = "$1" ]; then + debug <<EOF +################################################## +EOF + eval "$2" + debug <<EOF +################################################## +EOF + shift 2 + fi + if [ -n "$_rpc_service" ]; then + if rpcinfo -T tcp localhost "$_rpc_service" \ + >/dev/null 2>&1 ; then + _iterate_failcount=0 + else + _iterate_failcount=$((_iterate_failcount + 1)) + fi + rpc_set_service_failure_response \ + "$_rpc_service" $_iterate_failcount + fi + _out=$(simple_test 2>&1) + _ret=$? + if "$CTDB_TEST_VERBOSE" || [ $_ret -ne 0 ]; then + cat <<EOF +################################################## +Iteration ${_iteration}: +$_out +EOF + fi + if [ $_ret -ne 0 ]; then + exit $_ret + fi + done +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/91.lvs.sh b/ctdb/tests/UNIT/eventscripts/scripts/91.lvs.sh new file mode 100644 index 0000000..1e307c5 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/91.lvs.sh @@ -0,0 +1,76 @@ +setup() +{ + _ip="$1" + _iface="$2" + + export FAKE_LVS_STATE_DIR="${FAKE_NETWORK_STATE}/lvs" + mkdir -p "$FAKE_LVS_STATE_DIR" + + lvs_header=$(ipvsadm -l -n) + + [ -n "$_ip" ] || return 0 + [ -n "$_iface" ] || return 0 + + setup_script_options <<EOF +CTDB_LVS_NODES="${CTDB_BASE}/lvs_nodes" +CTDB_LVS_PUBLIC_IP="$_ip" +CTDB_LVS_PUBLIC_IFACE="$_iface" +EOF + + export FAKE_CTDB_LVS_LEADER="" + + # Read from stdin + _pnn=0 + while read -r _ip _opts; do + case "$_opts" in + leader) + FAKE_CTDB_LVS_LEADER="$_pnn" + echo "$_ip" + ;; + follower-only) + printf "%s\tfollower-only\n" "$_ip" + ;; + *) + echo "$_ip" + ;; + esac + _pnn=$((_pnn + 1)) + done >"$CTDB_LVS_NODES" +} + +check_ipvsadm() +{ + if [ "$1" = "NULL" ]; then + required_result 0 <<EOF +$lvs_header +EOF + else + required_result 0 <<EOF +$lvs_header +$(cat) +EOF + fi + + simple_test_command ipvsadm -l -n +} + +check_lvs_ip() +{ + _scope="$1" + + if [ "$_scope" = "NULL" ]; then + required_result 0 <<EOF +1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 +EOF + else + required_result 0 <<EOF +1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet ${CTDB_LVS_PUBLIC_IP}/32 scope ${_scope} lo + valid_lft forever preferred_lft forever +EOF + fi + + simple_test_command ip addr show dev lo +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/debug_locks.sh b/ctdb/tests/UNIT/eventscripts/scripts/debug_locks.sh new file mode 100644 index 0000000..b0cd039 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/debug_locks.sh @@ -0,0 +1,272 @@ +setup() +{ + setup_dbdir +} + +result_filter() +{ + sed -e 's|\( of debug locks PID=\)[0-9]*|\1PID|' +} + +tdb_path() +{ + echo "${CTDB_DBDIR}/${1}.${FAKE_CTDB_PNN}" +} + +fake_file_id() +{ + _path="$1" + + echo "$FAKE_FILE_ID_MAP" | + awk -v path="$_path" '$1 == path { print $2 }' +} + +fake_stack_trace() +{ + _pid="$1" + _command="${2:-smbd}" + _state="$3" + + echo "----- Stack trace for PID=${_pid} -----" + + case "$_state" in + D*) + cat <<EOF +----- Process in D state, printing kernel stack only +[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff +EOF + ;; + *) + cat <<EOF +Thread 1 (Thread 0x7f688fbfb180 (LWP ${_pid}) "${_command}"): +#0 0x00007f688ff7a076 in open (FAKE ARGS...) at FAKE PLACE +.... +#3 0x000055cd368ead72 in main (argc=<optimized out>, argv=<optimized out>) at ${_command}.c +EOF + ;; + esac +} + +do_test() +{ + _holder_scope="$1" + _holder_state="$2" + _helper_scope="$3" + _lock_type="${4:-FCNTL}" + + _lock_helper_pid="4132032" + + FAKE_PS_MAP=$( + cat <<EOF +1234567 ctdbd S +2345678 smbd S +4131931 smbd ${_holder_state} +${_lock_helper_pid} ctdb_lock_helpe S+ +EOF + ) + export FAKE_PS_MAP + + FAKE_FILE_ID_MAP="" + _tdbs="locking.tdb brlock.tdb test.tdb foo.tdb" + _n=1 + for _t in $_tdbs; do + _path=$(tdb_path "$_t") + _inode=$((19690818 + _n)) + FAKE_FILE_ID_MAP=$( + cat <<EOF +${FAKE_FILE_ID_MAP} +${_path} 103:04:${_inode} +EOF + ) + rm -f "$_path" + touch "$_path" + _n=$((_n + 1)) + done + export FAKE_FILE_ID_MAP + + _path=$(tdb_path "locking.tdb") + _locking_tdb_id=$(fake_file_id "$_path") + + _t=$( + cat <<EOF +POSIX ADVISORY WRITE 3769740 103:04:24380821 1073741826 1073742335 +FLOCK ADVISORY WRITE 3632524 103:02:1059266 0 EOF +FLOCK ADVISORY WRITE 4060231 00:17:17184 0 EOF +POSIX ADVISORY READ 1234567 ${_locking_tdb_id} 4 4 +POSIX ADVISORY WRITE 59178 103:04:24380821 1073741826 1073742335 +POSIX ADVISORY READ 4427 103:04:22152234 1073741826 1073742335 +POSIX ADVISORY WRITE 4427 103:04:22152494 0 EOF +POSIX ADVISORY READ 4427 103:04:22152702 1073741826 1073742335 +EOF + ) + + _holder_lock="" + if [ "$_holder_scope" = "DB" ]; then + if [ "$_lock_type" = "FCNTL" ]; then + _holder_lock=$( + cat <<EOF +POSIX ADVISORY WRITE 4131931 ${_locking_tdb_id} 168 EOF +EOF + ) + elif [ "$_lock_type" = "MUTEX" ]; then + _holder_lock=$( + cat <<EOF +POSIX ADVISORY WRITE 4131931 ${_locking_tdb_id} 400172 EOF +EOF + ) + fi + elif [ "$_holder_scope" = "RECORD" ] && + [ "$_lock_type" = "FCNTL" ]; then + _holder_lock=$( + cat <<EOF +POSIX ADVISORY WRITE 2345678 ${_locking_tdb_id} 112736 112736 +POSIX ADVISORY WRITE 4131931 ${_locking_tdb_id} 225472 225472 +EOF + ) + fi + + _t=$( + cat <<EOF +$_t +$_holder_lock +EOF + ) + + _helper_lock="" + if [ "$_helper_scope" = "DB" ] && + [ "$_lock_type" = "FCNTL" ]; then + _helper_lock=$( + cat <<EOF +-> POSIX ADVISORY WRITE ${_lock_helper_pid} ${_locking_tdb_id} 168 170 +EOF + ) + elif [ "$_helper_scope" = "RECORD" ] && + [ "$_lock_type" = "FCNTL" ]; then + _helper_lock=$( + cat <<EOF +-> POSIX ADVISORY WRITE ${_lock_helper_pid} ${_locking_tdb_id} 112736 112736 +EOF + ) + fi + _t=$( + cat <<EOF +$_t +$_helper_lock +EOF + ) + + if [ "$_holder_scope" = "DB" ]; then + _t=$( + cat <<EOF +$_t +POSIX ADVISORY READ 4131931 ${_locking_tdb_id} 4 4 +EOF + ) + elif [ "$_holder_scope" = "RECORD" ] && + [ "$_lock_type" = "FCNTL" ]; then + _t=$( + cat <<EOF +$_t +POSIX ADVISORY READ 2345678 ${_locking_tdb_id} 4 4 +POSIX ADVISORY READ 4131931 ${_locking_tdb_id} 4 4 +EOF + ) + fi + + _t=$( + cat <<EOF +$_t +POSIX ADVISORY READ 3769740 103:04:24390149 1073741826 1073742335 +POSIX ADVISORY WRITE 3769740 103:04:24380839 1073741826 1073742335 +FLOCK ADVISORY WRITE 3769302 103:02:1180313 0 EOF +FLOCK ADVISORY WRITE 3769302 103:02:1177487 0 EOF +FLOCK ADVISORY WRITE 3769302 103:02:1180308 0 EOF +OFDLCK ADVISORY READ -1 00:05:6 0 EOF +EOF + ) + + FAKE_PROC_LOCKS=$(echo "$_t" | awk '{ printf "%d: %s\n", NR, $0 }') + export FAKE_PROC_LOCKS + + _holder_mutex_lock="" + if [ "$_lock_type" = "MUTEX" ]; then + if [ "$_holder_scope" = "RECORD" ]; then + _holder_mutex_lock=$( + cat <<EOF +2345678 28142 +4131931 56284 +EOF + ) + fi + fi + + FAKE_TDB_MUTEX_CHECK="$_holder_mutex_lock" + export FAKE_TDB_MUTEX_CHECK + + _out='' + _nl=' +' + _db="locking.tdb.${FAKE_CTDB_PNN}" + + if [ -n "$_helper_lock" ]; then + read -r _ _ _ _ _pid _ _start _end <<EOF +$_helper_lock +EOF + _out="Waiter:${_nl}" + _out="${_out}${_pid} ctdb_lock_helpe ${_db} ${_start} ${_end}" + fi + + # fake lock info + _pids='' + _out="${_out:+${_out}${_nl}}Lock holders:" + if [ -n "$_holder_mutex_lock" ]; then + while read -r _pid _chain; do + _comm="smbd" + _out="${_out}${_nl}" + _out="${_out}${_pid} smbd ${_db} ${_chain}" + _pids="${_pids:+${_pids} }${_pid}" + done <<EOF +$_holder_mutex_lock +EOF + else + while read -r _ _ _ _pid _ _start _end; do + _comm="smbd" + _out="${_out}${_nl}" + _out="${_out}${_pid} smbd ${_db} ${_start} ${_end}" + _pids="${_pids:+${_pids} }${_pid}" + done <<EOF +$_holder_lock +EOF + fi + + # fake stack traces + for _pid in $_pids; do + _comm="smbd" + if [ "$_pid" = "4131931" ]; then + _state="$_holder_state" + else + _state="S" + fi + _out=$( + cat <<EOF +$_out +$(fake_stack_trace "$_pid" "$_comm" "$_state") +EOF + ) + done + + ok <<EOF +===== Start of debug locks PID=PID ===== +$_out +===== End of debug locks PID=PID ===== +EOF + + # shellcheck disable=SC2154 + # script_dir and script set in define_test() + script_test "${script_dir}/${script}" \ + "$_lock_helper_pid" \ + "$_helper_scope" \ + "$_path" \ + "$_lock_type" + +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/local.sh b/ctdb/tests/UNIT/eventscripts/scripts/local.sh new file mode 100644 index 0000000..3c28181 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/local.sh @@ -0,0 +1,568 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# +# Augment PATH with relevant stubs/ directories. +# + +stubs_dir="${CTDB_TEST_SUITE_DIR}/stubs" +[ -d "${stubs_dir}" ] || die "Failed to locate stubs/ subdirectory" + +# Make the path absolute for tests that change directory +case "$stubs_dir" in +/*) : ;; +*) stubs_dir="${PWD}/${stubs_dir}" ;; +esac + +# Use stubs as helpers +export CTDB_HELPER_BINDIR="$stubs_dir" + +PATH="${stubs_dir}:${PATH}" + +export CTDB="ctdb" + +# Force this to be absolute - event scripts can change directory +CTDB_TEST_TMP_DIR=$(cd "$CTDB_TEST_TMP_DIR" && echo "$PWD") + +export CTDB_LOGGING="file:" + +if [ -d "${CTDB_TEST_SUITE_DIR}/etc" ]; then + cp -a "${CTDB_TEST_SUITE_DIR}/etc" "$CTDB_TEST_TMP_DIR" + export CTDB_SYS_ETCDIR="${CTDB_TEST_TMP_DIR}/etc" +else + die "Unable to setup \$CTDB_SYS_ETCDIR" +fi + +setup_ctdb_base "$CTDB_TEST_TMP_DIR" "etc-ctdb" \ + debug_locks.sh \ + functions \ + nfs-checks.d \ + nfs-linux-kernel-callout \ + statd-callout + +export FAKE_CTDB_STATE="${CTDB_TEST_TMP_DIR}/fake-ctdb" +mkdir -p "$FAKE_CTDB_STATE" + +export FAKE_NETWORK_STATE="${CTDB_TEST_TMP_DIR}/fake-network-state" +mkdir -p "$FAKE_NETWORK_STATE" + +###################################################################### + +if "$CTDB_TEST_VERBOSE"; then + debug() + { + if [ -n "$1" ]; then + echo "$@" >&2 + else + cat >&2 + fi + } +else + debug() + { + : + } +fi + +###################################################################### + +# General setup fakery + +# Default is to use script name with ".options" appended. With +# arguments, this can specify an alternate script name (and +# component). +setup_script_options() +{ + if [ $# -eq 2 ]; then + _script="$2" + elif [ $# -eq 0 ]; then + _script="" + else + die "usage: setup_script_options [ component script ]" + fi + + if [ -n "$_script" ]; then + _options="${CTDB_BASE}/events/legacy/${_script}.options" + else + _options="${script_dir}/${script%.script}.options" + fi + + cat >>"$_options" + + # Source the options so that tests can use the variables + . "$_options" +} + +setup_dbdir() +{ + export CTDB_DBDIR_BASE="${CTDB_TEST_TMP_DIR}/db" + CTDB_DBDIR="${CTDB_DBDIR_BASE}/volatile" + CTDB_DBDIR_PERSISTENT="${CTDB_DBDIR_BASE}/persistent" + CTDB_DBDIR_STATE="${CTDB_DBDIR_BASE}/state" + cat >>"${CTDB_BASE}/ctdb.conf" <<EOF +[database] + volatile database directory = ${CTDB_DBDIR} + persistent database directory = ${CTDB_DBDIR_PERSISTENT} + state database directory = ${CTDB_DBDIR_STATE} +EOF + mkdir -p "$CTDB_DBDIR" + mkdir -p "$CTDB_DBDIR_PERSISTENT" + mkdir -p "$CTDB_DBDIR_STATE" +} + +setup_date() +{ + export FAKE_DATE_OUTPUT="$1" +} + +setup_tcp_listen() +{ + export FAKE_TCP_LISTEN="$*" +} + +tcp_port_listening() +{ + for _i; do + FAKE_TCP_LISTEN="${FAKE_TCP_LISTEN} ${_i}" + done +} + +tcp_port_down() +{ + _port="$1" + debug "Marking TCP port \"${_port}\" as not listening" + + _t="" + for _i in $FAKE_TCP_LISTEN; do + if [ "$_i" = "$_port" ]; then + continue + fi + _t="${_t} ${_i}" + done + + FAKE_TCP_LISTEN="$_t" +} + +setup_unix_listen() +{ + export FAKE_NETSTAT_UNIX_LISTEN="$*" +} + +unix_socket_listening() +{ + _s="$1" + + FAKE_NETSTAT_UNIX_LISTEN="${FAKE_NETSTAT_UNIX_LISTEN} ${_s}" +} + +setup_shares() +{ + debug "Setting up shares (3 existing shares)" + # Create 3 fake shares/exports. + export FAKE_SHARES="" + for i in $(seq 1 3); do + _s="${CTDB_TEST_TMP_DIR}/shares/share${i}" + mkdir -p "$_s" + FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}" + done +} + +shares_missing() +{ + # Mark some shares as non-existent + _type="$1" + shift + + _out="" + _nl=" +" + + _n=1 + for _i in $FAKE_SHARES; do + for _j; do + if [ $_n -ne "$_j" ]; then + continue + fi + + debug "Mark share $_n as missing share \"$_i\"" + rmdir "$_i" + _t=$(printf "ERROR: %s directory \"%s\" not available" \ + "$_type" "${_i}") + _out="${_out}${_out:+${_nl}}${_t}" + done + _n=$((_n + 1)) + done + + echo "$_out" +} + +_ethtool_setup() +{ + FAKE_ETHTOOL_LINK_DOWN="${FAKE_NETWORK_STATE}/ethtool-link-down" + export FAKE_ETHTOOL_LINK_DOWN + mkdir -p "$FAKE_ETHTOOL_LINK_DOWN" +} + +ethtool_interfaces_down() +{ + _ethtool_setup + + for _i; do + echo "Marking interface $_i DOWN for ethtool" + touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}" + done +} + +ethtool_interfaces_up() +{ + _ethtool_setup + + for _i; do + echo "Marking interface $_i UP for ethtool" + rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}" + done +} + +dump_routes() +{ + echo "# ip rule show" + ip rule show + + ip rule show | + while read -r _p _ _i _ _t; do + # Remove trailing colon after priority/preference. + _p="${_p%:}" + # Only remove rules that match our priority/preference. + [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue + + echo "# ip route show table $_t" + ip route show table "$_t" + done +} + +# Copied from 13.per_ip_routing for now... so this is lazy testing :-( +ipv4_host_addr_to_net() +{ + _host="$1" + _maskbits="$2" + + # Convert the host address to an unsigned long by splitting out + # the octets and doing the math. + _host_ul=0 + for _o in $( + export IFS="." + # shellcheck disable=SC2086 + # Intentional word splitting + echo $_host + ); do + _host_ul=$(((_host_ul << 8) + _o)) # work around Emacs color bug + done + + # Calculate the mask and apply it. + _mask_ul=$((0xffffffff << (32 - _maskbits))) + _net_ul=$((_host_ul & _mask_ul)) + + # Now convert to a network address one byte at a time. + _net="" + for _o in $(seq 1 4); do + _net="$((_net_ul & 255))${_net:+.}${_net}" + _net_ul=$((_net_ul >> 8)) + done + + echo "${_net}/${_maskbits}" +} + +###################################################################### + +# CTDB fakery + +# shellcheck disable=SC2120 +# Argument can be used in testcases +setup_numnodes() +{ + export FAKE_CTDB_NUMNODES="${1:-3}" + echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes" +} + +# For now this creates the same public addresses each time. However, +# it could be made more flexible. +setup_public_addresses() +{ + _f="${CTDB_BASE}/public_addresses" + + echo "Setting up public addresses in ${_f}" + cat >"$_f" <<EOF +# This is a comment +10.0.0.1/24 dev123 +10.0.0.2/24 dev123 +10.0.0.3/24 dev123 +10.0.0.4/24 dev123 +10.0.0.5/24 dev123 +10.0.0.6/24 dev123 +10.0.1.1/24 dev456 +10.0.1.2/24 dev456 +10.0.1.3/24 dev456 +EOF + + # Needed for IP allocation + setup_numnodes +} + +# Need to cope with ctdb_get_pnn(). If a test changes PNN then it +# needs to be using a different state directory, otherwise the wrong +# PNN can already be cached in the state directory. +ctdb_set_pnn() +{ + export FAKE_CTDB_PNN="$1" + echo "Setting up PNN ${FAKE_CTDB_PNN}" + + CTDB_SCRIPT_VARDIR="${CTDB_TEST_TMP_DIR}/scripts/${FAKE_CTDB_PNN}" + export CTDB_SCRIPT_VARDIR + mkdir -p "$CTDB_SCRIPT_VARDIR" +} + +ctdb_get_interfaces() +{ + ctdb ifaces -X | awk -F'|' 'FNR > 1 {print $2}' | xargs +} + +ctdb_get_1_interface() +{ + _t=$(ctdb_get_interfaces) + echo "${_t%% *}" +} + +# Print public addresses on this node as: interface IP maskbits +# Each line is suitable for passing to takeip/releaseip +ctdb_get_my_public_addresses() +{ + ctdb ip -v -X | { + read -r _ # skip header line + + while IFS="|" read -r _ _ip _ _iface _; do + [ -n "$_iface" ] || continue + while IFS="/$IFS" read -r _i _maskbits _; do + if [ "$_ip" = "$_i" ]; then + echo "$_iface $_ip $_maskbits" + break + fi + done <"${CTDB_BASE}/public_addresses" + done + } +} + +# Prints the 1st public address as: interface IP maskbits +# This is suitable for passing to takeip/releaseip +ctdb_get_1_public_address() +{ + ctdb_get_my_public_addresses | { + head -n 1 + cat >/dev/null + } +} + +# Check the routes against those that are expected. $1 is the number +# of assigned IPs to use (<num>, all), defaulting to 1. If $2 is +# "default" then expect default routes to have been added. +check_routes() +{ + _num_ips="${1:-1}" + _should_add_default="$2" + + _policy_rules="" + _policy_routes="" + + ctdb_get_my_public_addresses | + if [ "$_num_ips" = "all" ]; then + cat + else + { + head -n "$_num_ips" + cat >/dev/null + } + fi | { + while read -r _dev _ip _bits; do + _net=$(ipv4_host_addr_to_net "$_ip" "$_bits") + _gw="${_net%.*}.254" # a dumb, calculated default + + _policy_rules="${_policy_rules} +${CTDB_PER_IP_ROUTING_RULE_PREF}: from $_ip lookup ctdb.$_ip " + _policy_routes="${_policy_routes} +# ip route show table ctdb.$_ip +$_net dev $_dev scope link " + + if [ "$_should_add_default" = "default" ]; then + _policy_routes="${_policy_routes} +default via $_gw dev $_dev " + fi + done + + ok <<EOF +# ip rule show +0: from all lookup local ${_policy_rules} +32766: from all lookup main +32767: from all lookup default ${_policy_routes} +EOF + + simple_test_command dump_routes + } || test_fail +} + +###################################################################### + +nfs_load_config() +{ + _etc="$CTDB_SYS_ETCDIR" # shortcut for readability + for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs"; do + if [ -r "$_c" ]; then + . "$_c" + break + fi + done +} + +setup_nfs_callout() +{ + export CTDB_NFS_CALLOUT="${CTDB_HELPER_BINDIR}/nfs-fake-callout" + export NFS_FAKE_CALLOUT_MAGIC="$1" +} + +program_stack_trace() +{ + _prog="$1" + _pid="$2" + + cat <<EOF +Stack trace for ${_prog}[${_pid}]: +[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff +EOF +} + +###################################################################### + +# Result and test functions + +############################################################ + +setup() +{ + die "setup() is not defined" +} + +# Set some globals and print the summary. +define_test() +{ + desc="$1" + + _f=$(basename "$0" ".sh") + + # Remaining format should be NN.script.event.NUM or + # NN.script.NUM or script.NUM: + _num="${_f##*.}" + _f="${_f%.*}" + + case "$_f" in + [0-9][0-9].*) + case "$_f" in + [0-9][0-9].*.*) + script="${_f%.*}.script" + event="${_f##*.}" + ;; + [0-9][0-9].*) + script="${_f}.script" + unset event + ;; + esac + # "Enable" the script + _subdir="events/legacy" + script_dir="${CTDB_BASE}/${_subdir}" + # Symlink target needs to be absolute + case "$CTDB_SCRIPTS_DATA_DIR" in + /*) _data_dir="${CTDB_SCRIPTS_DATA_DIR}/${_subdir}" ;; + *) _data_dir="${PWD}/${CTDB_SCRIPTS_DATA_DIR}/${_subdir}" ;; + esac + mkdir -p "$script_dir" + ln -s "${_data_dir}/${script}" "$script_dir" + ;; + *) + script="${_f%.*}" + script="$_f" + unset event + script_dir="${CTDB_BASE}" + ;; + esac + + _s="${script_dir}/${script}" + [ -r "$_s" ] || + die "Internal error - unable to find script \"${_s}\"" + + case "$script" in + *.script) script_short="${script%.script}" ;; + *.sh) script_short="${script%.sh}" ;; + *) script_short="$script" ;; + esac + + printf "%-17s %-10s %-4s - %s\n\n" \ + "$script_short" "$event" "$_num" "$desc" + + _f="${CTDB_TEST_SUITE_DIR}/scripts/${script_short}.sh" + if [ -r "$_f" ]; then + . "$_f" + fi + + ctdb_set_pnn 0 +} + +# Run an eventscript once. The test passes if the return code and +# output match those required. + +# Any args are passed to the eventscript. + +simple_test() +{ + [ -n "$event" ] || die 'simple_test: event not set' + + args="$*" + + # shellcheck disable=SC2317 + # used in unit_test(), etc. + test_header() + { + echo "Running script \"$script $event${args:+ }$args\"" + } + + # shellcheck disable=SC2317 + # used in unit_test(), etc. + extra_header() + { + cat <<EOF + +################################################## +CTDB_BASE="$CTDB_BASE" +CTDB_SYS_ETCDIR="$CTDB_SYS_ETCDIR" +ctdb client is "$(which ctdb)" +ip command is "$(which ip)" +EOF + } + + script_test "${script_dir}/${script}" "$event" "$@" + + reset_test_header + reset_extra_header +} + +simple_test_event() +{ + # If something has previously failed then don't continue. + : "${_passed:=true}" + $_passed || return 1 + + event="$1" + shift + echo "==================================================" + simple_test "$@" +} + +simple_test_command() +{ + unit_test_notrace "$@" +} diff --git a/ctdb/tests/UNIT/eventscripts/scripts/statd-callout.sh b/ctdb/tests/UNIT/eventscripts/scripts/statd-callout.sh new file mode 100644 index 0000000..e966cb4 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/statd-callout.sh @@ -0,0 +1,65 @@ +setup() +{ + ctdb_set_pnn + setup_public_addresses + setup_date "123456789" +} + +ctdb_catdb_format_pairs() +{ + _count=0 + + while read -r _k _v; do + _kn=$(printf '%s' "$_k" | wc -c) + _vn=$(printf '%s' "$_v" | wc -c) + cat <<EOF +key(${_kn}) = "${_k}" +dmaster: 0 +rsn: 1 +data(${_vn}) = "${_v}" + +EOF + _count=$((_count + 1)) + done + + echo "Dumped ${_count} records" +} + +check_ctdb_tdb_statd_state() +{ + ctdb_get_my_public_addresses | + while read -r _ _sip _; do + for _cip; do + cat <<EOF +statd-state@${_sip}@${_cip} $(date) +EOF + done + done | + ctdb_catdb_format_pairs | { + ok + simple_test_command ctdb catdb ctdb.tdb + } || exit $? +} + +check_statd_callout_smnotify() +{ + _state_even=$(( $(date '+%s') / 2 * 2)) + _state_odd=$((_state_even + 1)) + + nfs_load_config + + ctdb_get_my_public_addresses | + while read -r _ _sip _; do + for _cip; do + cat <<EOF +SM_NOTIFY: ${_sip} -> ${_cip}, MON_NAME=${_sip}, STATE=${_state_even} +SM_NOTIFY: ${_sip} -> ${_cip}, MON_NAME=${NFS_HOSTNAME}, STATE=${_state_even} +SM_NOTIFY: ${_sip} -> ${_cip}, MON_NAME=${_sip}, STATE=${_state_odd} +SM_NOTIFY: ${_sip} -> ${_cip}, MON_NAME=${NFS_HOSTNAME}, STATE=${_state_odd} +EOF + done + done | { + ok + simple_test_event "notify" + } || exit $? +} |