diff options
Diffstat (limited to 'ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh')
| -rw-r--r-- | ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh | 435 |
1 files changed, 435 insertions, 0 deletions
diff --git a/ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh b/ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh new file mode 100644 index 0000000..9c614c7 --- /dev/null +++ b/ctdb/tests/UNIT/eventscripts/scripts/60.nfs.sh @@ -0,0 +1,435 @@ +setup() +{ + setup_public_addresses + setup_shares + + # shellcheck disable=SC2034 + # Used in expected output + service_name="nfs" + + if [ -z "$CTDB_NFS_DISTRO_STYLE" ]; then + # Currently supported: sysvinit-redhat, systemd-redhat + CTDB_NFS_DISTRO_STYLE="systemd-redhat" + fi + + export FAKE_RPCINFO_SERVICES="" + + setup_script_options <<EOF +CTDB_NFS_SKIP_SHARE_CHECK="no" +# This doesn't even need to exist +CTDB_NFS_EXPORTS_FILE="${CTDB_TEST_TMP_DIR}/etc-exports" +EOF + + export RPCNFSDCOUNT + + if [ "$1" != "down" ]; then + debug <<EOF +Setting up NFS environment: all RPC services up, NFS managed by CTDB +EOF + + case "$CTDB_NFS_DISTRO_STYLE" in + sysvinit-*) + service "nfs" force-started + service "nfslock" force-started + ;; + systemd-*) + service "nfs-service" force-started + service "nfs-mountd" force-started + service "rpc-rquotad" force-started + service "rpc-statd" force-started + ;; + esac + + rpc_services_up \ + "portmapper" "nfs" "mountd" "rquotad" \ + "nlockmgr" "status" + + nfs_setup_fake_threads "nfsd" + nfs_setup_fake_threads "rpc.foobar" # Set the variable to empty + else + debug <<EOF +Setting up NFS environment: all RPC services down, NFS not managed by CTDB +EOF + + case "$CTDB_NFS_DISTRO_STYLE" in + sysvinit-*) + service "nfs" force-stopped + service "nfslock" force-stopped + service "nfs-kernel-server" force-stopped + ;; + systemd-*) + service "nfs-server" force-stopped + service "nfs-mountd" force-stopped + service "rpc-quotad" force-stopped + service "rpc-statd" force-stopped + ;; + esac + fi + + # This is really nasty. However, when we test NFS we don't + # actually test statd-callout. If we leave it there then left + # over, backgrounded instances of statd-callout will do + # horrible things with the "ctdb ip" stub and cause the actual + # statd-callout tests that follow to fail. + rm "${CTDB_BASE}/statd-callout" +} + +rpc_services_down() +{ + _out="" + for _s in $FAKE_RPCINFO_SERVICES; do + for _i; do + if [ "$_i" = "${_s%%:*}" ]; then + debug "Marking RPC service \"${_i}\" as UNAVAILABLE" + continue 2 + fi + done + _out="${_out}${_out:+ }${_s}" + done + FAKE_RPCINFO_SERVICES="$_out" +} + +rpc_services_up() +{ + _out="$FAKE_RPCINFO_SERVICES" + for _i; do + debug "Marking RPC service \"${_i}\" as available" + case "$_i" in + portmapper) _t="2:4" ;; + nfs) _t="2:3" ;; + mountd) _t="1:3" ;; + rquotad) _t="1:2" ;; + nlockmgr) _t="3:4" ;; + status) _t="1:1" ;; + *) die "Internal error - unsupported RPC service \"${_i}\"" ;; + esac + + _out="${_out}${_out:+ }${_i}:${_t}" + done + export FAKE_RPCINFO_SERVICES="$_out" +} + +nfs_setup_fake_threads() +{ + _prog="$1" + shift + + case "$_prog" in + nfsd) + export PROCFS_PATH="${CTDB_TEST_TMP_DIR}/proc" + _threads="${PROCFS_PATH}/fs/nfsd/threads" + mkdir -p "$(dirname "$_threads")" + echo $# >"$_threads" + export FAKE_NFSD_THREAD_PIDS="$*" + ;; + *) + export FAKE_RPC_THREAD_PIDS="$*" + ;; + esac +} + +guess_output() +{ + case "$1" in + "${CTDB_NFS_CALLOUT} start nlockmgr") + case "$CTDB_NFS_DISTRO_STYLE" in + sysvinit-redhat) + echo "&Starting nfslock: OK" + ;; + sysvinit-debian) + cat <<EOF +&Starting nfs-kernel-server: OK +EOF + ;; + systemd-*) + echo "&Starting rpc-statd: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} start nfs") + case "$CTDB_NFS_DISTRO_STYLE" in + sysvinit-redhat) + cat <<EOF +&Starting nfslock: OK +&Starting nfs: OK +EOF + ;; + sysvinit-debian) + cat <<EOF +&Starting nfs-kernel-server: OK +EOF + ;; + systemd-redhat) + cat <<EOF +&Starting rpc-statd: OK +&Starting nfs-server: OK +&Starting rpc-rquotad: OK +EOF + ;; + systemd-debian) + cat <<EOF +&Starting rpc-statd: OK +&Starting nfs-server: OK +&Starting quotarpc: OK +EOF + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} stop mountd") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-*) + echo "Stopping nfs-mountd: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} stop rquotad") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-redhat) + echo "Stopping rpc-rquotad: OK" + ;; + systemd-debian) + if service "quotarpc" status >/dev/null; then + echo "Stopping quotarpc: OK" + else + echo "service: can't stop quotarpc - not running" + fi + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} stop status") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-*) + echo "Stopping rpc-statd: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} start mountd") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-*) + echo "&Starting nfs-mountd: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} start rquotad") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-redhat) + echo "&Starting rpc-rquotad: OK" + ;; + systemd-debian) + echo "&Starting quotarpc: OK" + ;; + esac + ;; + "${CTDB_NFS_CALLOUT} start status") + case "$CTDB_NFS_DISTRO_STYLE" in + systemd-*) + echo "&Starting rpc-statd: OK" + ;; + esac + ;; + *) + : # Nothing + ;; + esac +} + +# Set the required result for a particular RPC program having failed +# for a certain number of iterations. This is probably still a work +# in progress. Note that we could hook aggressively +# nfs_check_rpc_service() to try to implement this but we're better +# off testing nfs_check_rpc_service() using independent code... even +# if it is incomplete and hacky. So, if the 60.nfs eventscript +# changes and the tests start to fail then it may be due to this +# function being incomplete. +rpc_set_service_failure_response() +{ + _rpc_service="$1" + _numfails="${2:-1}" # default 1 + + # Default + ok_null + if [ "$_numfails" -eq 0 ]; then + return + fi + + nfs_load_config + + # A handy newline. :-) + _nl=" +" + + _dir="${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}" + + _file=$(ls "$_dir"/[0-9][0-9]."${_rpc_service}.check") + [ -r "$_file" ] || + die "RPC check file \"$_file\" does not exist or is not unique" + + _out="${CTDB_TEST_TMP_DIR}/rpc_failure_output" + : >"$_out" + _rc_file="${CTDB_TEST_TMP_DIR}/rpc_result" + + ( + # Subshell to restrict scope variables... + + # Defaults + # shellcheck disable=SC2034 + # Unused, but for completeness, possible future use + family="tcp" + version="" + unhealthy_after=1 + restart_every=0 + service_stop_cmd="" + service_start_cmd="" + # shellcheck disable=SC2034 + # Unused, but for completeness, possible future use + service_check_cmd="" + service_debug_cmd="" + + # Don't bother syntax checking, eventscript does that... + . "$_file" + + # Just use the first version, or use default. This is + # dumb but handles all the cases that we care about + # now... + if [ -n "$version" ]; then + _ver="${version%% *}" + else + case "$_rpc_service" in + portmapper) _ver="" ;; + *) _ver=1 ;; + esac + fi + _rpc_check_out="\ +$_rpc_service failed RPC check: +rpcinfo: RPC: Program not registered +program $_rpc_service${_ver:+ version }${_ver} is not available" + + if [ $unhealthy_after -gt 0 ] && + [ "$_numfails" -ge $unhealthy_after ]; then + _unhealthy=true + echo 1 >"$_rc_file" + echo "ERROR: ${_rpc_check_out}" >>"$_out" + else + _unhealthy=false + echo 0 >"$_rc_file" + fi + + if [ $restart_every -gt 0 ] && + [ $((_numfails % restart_every)) -eq 0 ]; then + if ! $_unhealthy; then + echo "WARNING: ${_rpc_check_out}" >>"$_out" + fi + + echo "Trying to restart service \"${_rpc_service}\"..." \ + >>"$_out" + + guess_output "$service_stop_cmd" >>"$_out" + + if [ -n "$service_debug_cmd" ]; then + $service_debug_cmd >>"$_out" 2>&1 + fi + + guess_output "$service_start_cmd" >>"$_out" + fi + ) + + read -r _rc <"$_rc_file" + required_result "$_rc" <"$_out" + + rm -f "$_out" "$_rc_file" +} + +program_stack_traces() +{ + _prog="$1" + _max="${2:-1}" + + _count=1 + if [ "$_prog" = "nfsd" ]; then + _pids="$FAKE_NFSD_THREAD_PIDS" + else + _pids="$FAKE_RPC_THREAD_PIDS" + fi + for _pid in $_pids; do + [ $_count -le "$_max" ] || break + + program_stack_trace "$_prog" "$_pid" + _count=$((_count + 1)) + done +} + +# Run an NFS eventscript iteratively. +# +# - 1st argument is the number of iterations. +# +# - 2nd argument is the NFS/RPC service being tested +# +# rpcinfo is used on each iteration to test the availability of the +# service +# +# If this is not set or null then no RPC service is checked and the +# required output is not reset on each iteration. This is useful in +# baseline tests to confirm that the eventscript and test +# infrastructure is working correctly. +# +# - Subsequent arguments come in pairs: an iteration number and +# something to eval before that iteration. Each time an iteration +# number is matched the associated argument is given to eval after +# the default setup is done. The iteration numbers need to be given +# in ascending order. +# +# These arguments can allow a service to be started or stopped +# before a particular iteration. +# +nfs_iterate_test() +{ + _repeats="$1" + _rpc_service="$2" + if [ -n "$2" ]; then + shift 2 + else + shift + fi + + # shellcheck disable=SC2154 + # Variables defined in define_test() + echo "Running $_repeats iterations of \"$script $event\" $args" + + _iterate_failcount=0 + for _iteration in $(seq 1 "$_repeats"); do + # This is not a numerical comparison because $1 will + # often not be set. + if [ "$_iteration" = "$1" ]; then + debug <<EOF +################################################## +EOF + eval "$2" + debug <<EOF +################################################## +EOF + shift 2 + fi + if [ -n "$_rpc_service" ]; then + if rpcinfo -T tcp localhost "$_rpc_service" \ + >/dev/null 2>&1 ; then + _iterate_failcount=0 + else + _iterate_failcount=$((_iterate_failcount + 1)) + fi + rpc_set_service_failure_response \ + "$_rpc_service" $_iterate_failcount + fi + _out=$(simple_test 2>&1) + _ret=$? + if "$CTDB_TEST_VERBOSE" || [ $_ret -ne 0 ]; then + cat <<EOF +################################################## +Iteration ${_iteration}: +$_out +EOF + fi + if [ $_ret -ne 0 ]; then + exit $_ret + fi + done +} |