Adding upstream version 2:4.20.0+dfsg.upstream/2%4.20.0+dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1172 insertions, 0 deletions

diff --git a/ctdb/config/functions b/ctdb/config/functions
new file mode 100755
index 0000000..a40b276
--- /dev/null
+++ b/ctdb/config/functions
@@ -0,0 +1,1172 @@
+# Hey Emacs, this is a -*- shell-script -*- !!!
+
+# utility functions for ctdb event scripts
+
+if [ -z "$CTDB_BASE" ]; then
+	echo 'CTDB_BASE unset in CTDB functions file'
+	exit 1
+fi
+export CTDB_BASE
+
+# CTDB_VARDIR is used elsewhere
+# shellcheck disable=SC2034
+CTDB_VARDIR="/usr/local/var/lib/ctdb"
+
+CTDB="${CTDB:-/usr/local/bin/ctdb}"
+
+# Only (and always) override these variables in test code
+
+if [ -z "$CTDB_SCRIPT_VARDIR" ]; then
+	CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/scripts"
+fi
+
+if [ -z "$CTDB_SYS_ETCDIR" ]; then
+	CTDB_SYS_ETCDIR="/etc"
+fi
+
+if [ -z "$CTDB_HELPER_BINDIR" ]; then
+	CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
+fi
+
+#######################################
+# pull in a system config file, if any
+
+load_system_config()
+{
+	for _i; do
+
+		if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/${_i}" ]; then
+			. "${CTDB_SYS_ETCDIR}/sysconfig/${_i}"
+			return
+		elif [ -f "${CTDB_SYS_ETCDIR}/default/${_i}" ]; then
+			. "${CTDB_SYS_ETCDIR}/default/${_i}"
+			return
+		fi
+	done
+}
+
+# load_script_options [ component script ]
+#   script is an event script name relative to a component
+#   component is currently ignored
+load_script_options()
+{
+	if [ $# -eq 2 ]; then
+		_script="$2"
+	elif [ $# -eq 0 ]; then
+		_script=""
+	else
+		die "usage: load_script_options [ component script ]"
+	fi
+
+	_options="${CTDB_BASE}/script.options"
+
+	if [ -r "$_options" ]; then
+		. "$_options"
+	fi
+
+	if [ -n "$_script" ]; then
+		_s="${CTDB_BASE}/events/legacy/${_script}"
+	else
+		_s="${0%.script}"
+	fi
+	_options="${_s}.options"
+
+	if [ -r "$_options" ]; then
+		. "$_options"
+	fi
+}
+
+##############################################################
+
+die()
+{
+	_msg="$1"
+	_rc="${2:-1}"
+
+	echo "$_msg" >&2
+	exit "$_rc"
+}
+
+# Log given message or stdin to either syslog or a CTDB log file
+# $1 is the tag passed to logger if syslog is in use.
+script_log()
+{
+	_tag="$1"
+	shift
+
+	case "$CTDB_LOGGING" in
+	file:)
+		if [ -n "$*" ] ; then
+			echo "$*"
+		else
+			cat
+		fi >&2
+		;;
+	file:* | "")
+		if [ -n "$CTDB_LOGGING" ]; then
+			_file="${CTDB_LOGGING#file:}"
+		else
+			_file="/usr/local/var/log/log.ctdb"
+		fi
+		{
+			if [ -n "$*" ]; then
+				echo "$*"
+			else
+				cat
+			fi
+		} >>"$_file"
+		;;
+	*)
+		# Handle all syslog:* variants here too.  There's no tool to do
+		# the lossy things, so just use logger.
+		logger -t "ctdbd: ${_tag}" "$@"
+		;;
+	esac
+}
+
+# When things are run in the background in an eventscript then logging
+# output might get lost.  This is the "solution".  :-)
+background_with_logging()
+{
+	(
+		"$@" 2>&1 </dev/null |
+			script_log "${script_name}&"
+	) &
+
+	return 0
+}
+
+##############################################################
+# check number of args for different events
+ctdb_check_args()
+{
+	case "$1" in
+	takeip | releaseip)
+		if [ $# != 4 ]; then
+			echo "ERROR: must supply interface, IP and maskbits"
+			exit 1
+		fi
+		;;
+	updateip)
+		if [ $# != 5 ]; then
+			echo "ERROR: must supply old interface, new interface, IP and maskbits"
+			exit 1
+		fi
+		;;
+	esac
+}
+
+##############################################################
+# determine on what type of system (init style) we are running
+detect_init_style()
+{
+	_init_style_file="${CTDB_SCRIPT_VARDIR}/init-style"
+
+	if [ ! -f "$_init_style_file" ]; then
+		if [ -n "$CTDB_INIT_STYLE" ]; then
+			echo "$CTDB_INIT_STYLE" >"$_init_style_file"
+			return
+		fi
+
+		# Subshell to contain variables in os-release file
+		(
+			_os_release="${CTDB_SYS_ETCDIR}/os-release"
+			if [ -f "$_os_release" ]; then
+				. "$_os_release"
+				case "$ID" in
+				centos | fedora | rhel)
+					echo "redhat"
+					;;
+				debian | ubuntu)
+					echo "debian"
+					;;
+				sles | suse)
+					echo "suse"
+					;;
+				*)
+					case "$ID_LIKE" in
+					*centos* | *rhel*)
+						echo "redhat"
+						;;
+					*)
+						echo "$ID"
+						;;
+					esac
+					;;
+				esac
+			else
+				echo "WARNING: unknown distribution ${ID}" >&2
+				echo "unknown"
+			fi
+		) >"$_init_style_file"
+	fi
+
+	read -r CTDB_INIT_STYLE <"$_init_style_file"
+}
+
+######################################################
+# simulate /sbin/service on platforms that don't have it
+# _service() makes it easier to hook the service() function for
+# testing.
+_service()
+{
+	_service_name="$1"
+	_op="$2"
+
+	# do nothing, when no service was specified
+	[ -z "$_service_name" ] && return
+
+	if [ -x /sbin/service ]; then
+		$_nice /sbin/service "$_service_name" "$_op"
+	elif [ -x /usr/sbin/service ]; then
+		$_nice /usr/sbin/service "$_service_name" "$_op"
+	elif [ -x /bin/systemctl ]; then
+		$_nice /bin/systemctl "$_op" "$_service_name"
+	elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
+		$_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
+	elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
+		$_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
+	fi
+}
+
+service()
+{
+	_nice=""
+	_service "$@"
+}
+
+######################################################
+# simulate /sbin/service (niced) on platforms that don't have it
+nice_service()
+{
+	_nice="nice"
+	_service "$@"
+}
+
+######################################################
+# Cached retrieval of PNN from local node.  This never changes so why
+# open a client connection to the server each time this is needed?
+ctdb_get_pnn()
+{
+	_pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
+	if [ ! -f "$_pnn_file" ]; then
+		$CTDB pnn >"$_pnn_file"
+	fi
+
+	cat "$_pnn_file"
+}
+
+# Cached retrieval of private IP address from local node.  This never
+# changes.
+ctdb_get_ip_address()
+{
+	_ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
+	if [ ! -f "$_ip_addr_file" ]; then
+		$CTDB -X nodestatus |
+			awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
+	fi
+
+	cat "$_ip_addr_file"
+}
+
+# Cached retrieval of database options for use by event scripts.
+#
+# If the variables are already set then they should not be overwritten
+# - this should only happen during event script testing.
+ctdb_get_db_options()
+{
+	_db_opts_file="${CTDB_SCRIPT_VARDIR}/db_options.cache"
+
+	if [ ! -f "$_db_opts_file" ]; then
+		{
+			ctdb_translate_option "database" \
+				"volatile database directory" \
+				"CTDB_DBDIR"
+			ctdb_translate_option "database" \
+				"persistent database directory" \
+				"CTDB_DBDIR_PERSISTENT"
+			ctdb_translate_option "database" \
+				"state database directory" \
+				"CTDB_DBDIR_STATE"
+		} >"$_db_opts_file"
+	fi
+
+	. "$_db_opts_file"
+}
+
+ctdb_translate_option()
+{
+	_section="$1"
+	_opt="$2"
+	_variable="$3"
+
+	# ctdb-config already prints an error if something goes wrong
+	_t=$("${CTDB_HELPER_BINDIR}/ctdb-config" get "$_section" "$_opt") ||
+		exit $?
+	echo "${_variable}=\"${_t}\""
+}
+
+######################################################
+# wrapper around /proc/ settings to allow them to be hooked
+# for testing
+# 1st arg is relative path under /proc/, 2nd arg is value to set
+set_proc()
+{
+	echo "$2" >"/proc/$1"
+}
+
+set_proc_maybe()
+{
+	if [ -w "/proc/$1" ]; then
+		set_proc "$1" "$2"
+	fi
+}
+
+######################################################
+# wrapper around getting file contents from /proc/ to allow
+# this to be hooked for testing
+# 1st arg is relative path under /proc/
+get_proc()
+{
+	cat "/proc/$1"
+}
+
+######################################################
+# Print up to $_max kernel stack traces for processes named $_program
+program_stack_traces()
+{
+	_prog="$1"
+	_max="${2:-1}"
+
+	_count=1
+	for _pid in $(pidof "$_prog"); do
+		[ "$_count" -le "$_max" ] || break
+
+		# Do this first to avoid racing with process exit
+		_stack=$(get_proc "${_pid}/stack" 2>/dev/null)
+		if [ -n "$_stack" ]; then
+			echo "Stack trace for ${_prog}[${_pid}]:"
+			echo "$_stack"
+			_count=$((_count + 1))
+		fi
+	done
+}
+
+######################################################
+# Ensure $service_name is set
+assert_service_name()
+{
+	# service_name is set by the event script
+	# shellcheck disable=SC2154
+	[ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
+}
+
+######################################################
+# check a set of directories is available
+# return 1 on a missing directory
+# directories are read from stdin
+######################################################
+ctdb_check_directories_probe()
+{
+	while IFS="" read -r d; do
+		case "$d" in
+		*%*)
+			continue
+			;;
+		*)
+			[ -d "${d}/." ] || return 1
+			;;
+		esac
+	done
+}
+
+######################################################
+# check a set of directories is available
+# directories are read from stdin
+######################################################
+ctdb_check_directories()
+{
+	ctdb_check_directories_probe || {
+		echo "ERROR: $service_name directory \"$d\" not available"
+		exit 1
+	}
+}
+
+######################################################
+# check a set of tcp ports
+# usage: ctdb_check_tcp_ports <ports...>
+######################################################
+
+# Check whether something is listening on all of the given TCP ports
+# using the "ctdb checktcpport" command.
+ctdb_check_tcp_ports()
+{
+	if [ -z "$1" ]; then
+		echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
+		exit 1
+	fi
+
+	for _p; do # process each function argument (port)
+		_cmd="$CTDB checktcpport $_p"
+		_out=$($_cmd 2>&1)
+		_ret=$?
+		case "$_ret" in
+		0)
+			echo "$service_name not listening on TCP port $_p"
+			return 1
+			;;
+		98)
+			# Couldn't bind, something already listening, next port
+			continue
+			;;
+		*)
+			echo "unexpected error (${_ret}) running \"${_cmd}\""
+			if [ -n "$_out" ]; then
+				echo "$_out"
+			fi
+			return $_ret
+			;;
+		esac
+	done
+
+	# All ports listening
+	return 0
+}
+
+######################################################
+# check a unix socket
+# usage: ctdb_check_unix_socket SOCKPATH
+######################################################
+ctdb_check_unix_socket()
+{
+	_sockpath="$1"
+
+	if [ -z "$_sockpath" ]; then
+		echo "ERROR: ctdb_check_unix_socket() requires socket path"
+		return 1
+	fi
+
+	_out=$(ss -l -x "src ${_sockpath}" | tail -n +2)
+	if [ -z "$_out" ]; then
+		echo "ERROR: ${service_name} not listening on ${_sockpath}"
+		return 1
+	fi
+}
+
+################################################
+# kill off any TCP connections with the given IP
+################################################
+kill_tcp_connections()
+{
+	_iface="$1"
+	_ip="$2"
+
+	_oneway=false
+	if [ "$3" = "oneway" ]; then
+		_oneway=true
+	fi
+
+	get_tcp_connections_for_ip "$_ip" | {
+		_killcount=0
+		_connections=""
+		_nl="
+"
+		while read -r _dst _src; do
+			_destport="${_dst##*:}"
+			__oneway=$_oneway
+			case $_destport in
+			# we only do one-way killtcp for CIFS
+			139 | 445) __oneway=true ;;
+			esac
+
+			_connections="${_connections}${_nl}${_src} ${_dst}"
+			if ! $__oneway; then
+				_connections="${_connections}${_nl}${_dst} ${_src}"
+			fi
+
+			_killcount=$((_killcount + 1))
+		done
+
+		if [ $_killcount -eq 0 ]; then
+			return
+		fi
+
+		if [ -n "$CTDB_KILLTCP_DEBUGLEVEL" ]; then
+			_debuglevel="$CTDB_KILLTCP_DEBUGLEVEL"
+		else
+			_debuglevel="$CTDB_DEBUGLEVEL"
+		fi
+		echo "$_connections" |
+			CTDB_DEBUGLEVEL="$_debuglevel" \
+				"${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
+			echo "Failed to kill TCP connections"
+			return
+		}
+
+		_connections=$(get_tcp_connections_for_ip "$_ip")
+		if [ -z "$_connections" ]; then
+			_remaining=0
+		else
+			_remaining=$(echo "$_connections" | wc -l)
+		fi
+
+		_actually_killed=$((_killcount - _remaining))
+
+		_t="${_actually_killed}/${_killcount}"
+		echo "Killed ${_t} TCP connections to released IP $_ip"
+
+		if [ -n "$_connections" ]; then
+			echo "Remaining connections:"
+			echo "$_connections" | sed -e 's|^|  |'
+		fi
+	}
+}
+
+##################################################################
+# kill off the local end for any TCP connections with the given IP
+##################################################################
+kill_tcp_connections_local_only()
+{
+	kill_tcp_connections "$@" "oneway"
+}
+
+##################################################################
+# tickle any TCP connections with the given IP
+##################################################################
+tickle_tcp_connections()
+{
+	_ip="$1"
+
+	# Get connections, both directions
+	_conns=$(get_tcp_connections_for_ip "$_ip" |
+		awk '{ print $1, $2 ; print $2, $1 }')
+
+	echo "$_conns" | awk '{ print "Tickle TCP connection", $1, $2 }'
+	echo "$_conns" | ctdb tickle
+}
+
+get_tcp_connections_for_ip()
+{
+	_ip="$1"
+
+	ss -tn state established "src [$_ip]" | awk 'NR > 1 {print $3, $4}'
+}
+
+########################################################
+
+add_ip_to_iface()
+{
+	_iface=$1
+	_ip=$2
+	_maskbits=$3
+
+	# Ensure interface is up
+	ip link set "$_iface" up ||
+		die "Failed to bringup interface $_iface"
+
+	# Only need to define broadcast for IPv4
+	case "$_ip" in
+	*:*) _bcast="" ;;
+	*) _bcast="brd +" ;;
+	esac
+
+	# Intentionally unquoted multi-word value here
+	# shellcheck disable=SC2086
+	ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
+		echo "Failed to add $_ip/$_maskbits on dev $_iface"
+		return 1
+	}
+
+	# Wait 5 seconds for IPv6 addresses to stop being tentative...
+	if [ -z "$_bcast" ]; then
+		for _x in $(seq 1 10); do
+			ip addr show to "${_ip}/128" | grep -q "tentative" || break
+			sleep 0.5
+		done
+
+		# If the address was a duplicate then it won't be on the
+		# interface so flag an error.
+		_t=$(ip addr show to "${_ip}/128")
+		case "$_t" in
+		"")
+			echo "Failed to add $_ip/$_maskbits on dev $_iface"
+			return 1
+			;;
+		*tentative* | *dadfailed*)
+			echo "Failed to add $_ip/$_maskbits on dev $_iface"
+			ip addr del "$_ip/$_maskbits" dev "$_iface"
+			return 1
+			;;
+		esac
+	fi
+}
+
+delete_ip_from_iface()
+{
+	_iface=$1
+	_ip=$2
+	_maskbits=$3
+
+	# This could be set globally for all interfaces but it is probably
+	# better to avoid surprises, so limit it the interfaces where CTDB
+	# has public IP addresses.  There isn't anywhere else convenient
+	# to do this so just set it each time.  This is much cheaper than
+	# remembering and re-adding secondaries.
+	set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
+
+	ip addr del "$_ip/$_maskbits" dev "$_iface" || {
+		echo "Failed to del $_ip on dev $_iface"
+		return 1
+	}
+}
+
+# If the given IP is hosted then print 2 items: maskbits and iface
+ip_maskbits_iface()
+{
+	_addr="$1"
+
+	case "$_addr" in
+	*:*) _bits=128 ;;
+	*) _bits=32 ;;
+	esac
+	ip addr show to "${_addr}/${_bits}" 2>/dev/null |
+		awk 'NR == 1 { iface = $2; sub(":$", "", iface) ;
+		       sub("@.*", "", iface) }
+             $1 ~ /inet/ { mask = $2; sub(".*/", "", mask);
+                           print mask, iface }'
+}
+
+drop_ip()
+{
+	_addr="${1%/*}" # Remove optional maskbits
+
+	# Intentional word splitting here
+	# shellcheck disable=SC2046
+	set -- $(ip_maskbits_iface "$_addr")
+	if [ -n "$1" ]; then
+		_maskbits="$1"
+		_iface="$2"
+		echo "Removing public address $_addr/$_maskbits from device $_iface"
+		delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
+	fi
+}
+
+drop_all_public_ips()
+{
+	# _x is intentionally ignored
+	# shellcheck disable=SC2034
+	while read -r _ip _x; do
+		case "$_ip" in
+		\#*) continue ;;
+		esac
+		drop_ip "$_ip"
+	done <"${CTDB_BASE}/public_addresses"
+}
+
+flush_route_cache()
+{
+	set_proc_maybe sys/net/ipv4/route/flush 1
+	set_proc_maybe sys/net/ipv6/route/flush 1
+}
+
+########################################################
+# Interface monitoring
+
+# If the interface is a virtual one (e.g. VLAN) then get the
+# underlying interface
+interface_get_real()
+{
+	_iface="$1"
+
+	# If $_iface is a VLAN (i.e. contains an '@') then strip every
+	# before the '@', otherwise print the whole interface
+	echo "${_iface##*@}"
+}
+
+# Check whether an interface is operational
+interface_monitor()
+{
+	_iface="$1"
+
+	_iface_info=$(ip -br link show "$_iface" 2>&1) || {
+		echo "ERROR: Monitored interface ${_iface} does not exist"
+		return 1
+	}
+
+	# If the interface is a virtual one (e.g. VLAN) then get the
+	# underlying interface.
+	_realiface=$(interface_get_real "${_iface_info%% *}")
+
+	if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null); then
+		# This is a bond: various monitoring strategies
+		echo "$_bi" | grep -q 'Currently Active Slave: None' && {
+			echo "ERROR: No active slaves for bond device ${_realiface}"
+			return 1
+		}
+		echo "$_bi" | grep -q '^MII Status: up' || {
+			echo "ERROR: public network interface ${_realiface} is down"
+			return 1
+		}
+		echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
+			# This works around a bug in the driver where the
+			# overall bond status can be up but none of the actual
+			# physical interfaces have a link.
+			echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
+				echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
+				return 1
+			}
+		}
+
+		return 0
+	else
+		# Not a bond
+		case "$_iface" in
+		lo*)
+			# loopback is always working
+			return 0
+			;;
+		ib*)
+			# we don't know how to test ib links
+			return 0
+			;;
+		*)
+			ethtool "$_iface" | grep -q 'Link detected: yes' || {
+				# On some systems, this is not successful when a
+				# cable is plugged but the interface has not been
+				# brought up previously. Bring the interface up
+				# and try again...
+				ip link set "$_iface" up
+				ethtool "$_iface" | grep -q 'Link detected: yes' || {
+					echo "ERROR: No link on the public network interface ${_iface}"
+					return 1
+				}
+			}
+			return 0
+			;;
+		esac
+	fi
+}
+
+########################################################
+# Simple counters
+_ctdb_counter_common()
+{
+	[ $# -le 1 ] || die "usage: _ctdb_counter_common [name]"
+
+	if [ $# -eq 1 ]; then
+		_counter_name="${1}.failcount"
+	else
+		_counter_name="failcount"
+	fi
+
+	if [ -z "$script_state_dir" ]; then
+		die "ctdb_counter_* functions need ctdb_setup_state_dir()"
+	fi
+
+	_counter_file="${script_state_dir}/${_counter_name}"
+}
+# Some code passes an argument
+# shellcheck disable=SC2120
+ctdb_counter_init()
+{
+	_ctdb_counter_common "$1"
+
+	: >"$_counter_file"
+}
+ctdb_counter_incr()
+{
+	_ctdb_counter_common "$1"
+
+	# unary counting using newlines!
+	echo >>"$_counter_file"
+}
+ctdb_counter_get()
+{
+	_ctdb_counter_common "$1"
+	# unary counting!
+	_val=$(wc -c 2>/dev/null <"$_counter_file" || echo 0)
+	# Strip leading spaces from output of wc (on freebsd)
+	# shellcheck disable=SC2086
+	echo $_val
+}
+
+#
+# Fail counter/threshold combination to control warnings and node unhealthy
+#
+
+_failcount_validate_threshold()
+{
+	case "$1" in
+	"") return 1 ;; # A failure that doesn't need a warning
+	*)
+		if echo "$1" | grep -qx '[0-9]*'; then
+			return 0
+		fi
+
+		echo "WARNING: ${1} is an invalid threshold in \"${2}\" check"
+		return 1
+		;;
+	esac
+}
+
+_failcount_common()
+{
+	_thing="$1"
+
+	_counter=$(echo "$_thing" | sed -e 's@/@_SLASH_@g' -e 's@ @_@g')
+}
+
+failcount_init()
+{
+	_thing="$1"
+
+	_failcount_common "$_thing"
+
+	ctdb_counter_init "$_counter"
+}
+
+failcount_reset()
+{
+	_thing="$1"
+
+	_failcount_common "$_thing"
+
+	_failcount=$(ctdb_counter_get "$_counter")
+	if [ "$_failcount" -eq 0 ]; then
+		return
+	fi
+
+	printf 'NOTICE: %s: no longer failing\n' "$_thing"
+	ctdb_counter_init "$_counter"
+}
+
+failcount_incr()
+{
+	_thing="$1"
+	_thresholds="$2"
+	_output="$3"
+
+	_failcount_common "$_thing"
+
+	ctdb_counter_incr "$_counter"
+	_failcount=$(ctdb_counter_get "$_counter")
+
+	case "$_thresholds" in
+	*:*)
+		_warn_threshold="${_thresholds%:*}"
+		_unhealthy_threshold="${_thresholds#*:}"
+		;;
+	"")
+		_warn_threshold=1
+		_unhealthy_threshold=""
+		;;
+	*)
+		_warn_threshold="$_thresholds"
+		_unhealthy_threshold=""
+		;;
+	esac
+
+	if _failcount_validate_threshold "$_unhealthy_threshold" "$_thing"; then
+		if [ "$_failcount" -ge "$_unhealthy_threshold" ]; then
+			printf 'ERROR: %s: fail count %d >= threshold %d\n' \
+			       "$_thing" \
+			       "$_failcount" \
+			       "$_unhealthy_threshold"
+			# Only print output when exceeding the
+			# unhealthy threshold
+			if [ "$_failcount" -eq "$_unhealthy_threshold" ] && \
+				   [ -n "$_output" ]; then
+				echo "$_output"
+			fi
+			exit 1
+		fi
+	fi
+
+	if _failcount_validate_threshold "$_warn_threshold" "$_thing"; then
+		if [ "$_failcount" -lt "$_warn_threshold" ]; then
+			return 0
+		fi
+	fi
+
+	printf 'WARNING: %s: fail count %d >= threshold %d\n' \
+	       "$_thing" \
+	       "$_failcount" \
+	       "$_warn_threshold"
+	if [ "$_failcount" -eq "$_warn_threshold" ] && [ -n "$_output" ]; then
+		# Only print output when exceeding the warning threshold
+		echo "$_output"
+	fi
+}
+
+########################################################
+
+# ctdb_setup_state_dir <type> <name>
+#   Sets/creates script_state_dir)
+ctdb_setup_state_dir()
+{
+	[ $# -eq 2 ] || die "usage: ctdb_setup_state_dir <type> <name>"
+
+	_type="$1"
+	_name="$2"
+
+	script_state_dir="${CTDB_SCRIPT_VARDIR}/${_type}/${_name}"
+
+	mkdir -p "$script_state_dir" ||
+		die "Error creating script state dir \"${script_state_dir}\""
+}
+
+##################################################################
+# Reconfigure a service on demand
+
+_ctdb_service_reconfigure_common()
+{
+	if [ -z "$script_state_dir" ]; then
+		die "ctdb_service_*_reconfigure() needs ctdb_setup_state_dir()"
+	fi
+
+	_ctdb_service_reconfigure_flag="${script_state_dir}/need_reconfigure"
+}
+
+ctdb_service_needs_reconfigure()
+{
+	_ctdb_service_reconfigure_common
+	[ -e "$_ctdb_service_reconfigure_flag" ]
+}
+
+ctdb_service_set_reconfigure()
+{
+	_ctdb_service_reconfigure_common
+	: >"$_ctdb_service_reconfigure_flag"
+}
+
+ctdb_service_unset_reconfigure()
+{
+	_ctdb_service_reconfigure_common
+	rm -f "$_ctdb_service_reconfigure_flag"
+}
+
+ctdb_service_reconfigure()
+{
+	echo "Reconfiguring service \"${service_name}\"..."
+	ctdb_service_unset_reconfigure
+	service_reconfigure || return $?
+	# Intentionally have this use $service_name as default
+	# shellcheck disable=SC2119
+	ctdb_counter_init
+}
+
+# Default service_reconfigure() function does nothing.
+service_reconfigure()
+{
+	:
+}
+
+# Default service_start() and service_stop() functions.
+
+# These may be overridden in an eventscript.
+service_start()
+{
+	service "$service_name" start
+}
+
+service_stop()
+{
+	service "$service_name" stop
+}
+
+##################################################################
+
+# This exists only for backward compatibility with 3rd party scripts
+# that call it
+ctdb_standard_event_handler()
+{
+	:
+}
+
+iptables_wrapper()
+{
+	_family="$1"
+	shift
+	if [ "$_family" = "inet6" ]; then
+		_iptables_cmd="ip6tables"
+	else
+		_iptables_cmd="iptables"
+	fi
+
+	# iptables doesn't like being re-entered, so flock-wrap it.
+	flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
+}
+
+# AIX (and perhaps others?) doesn't have mktemp
+# type is commonly supported and more portable than which(1)
+# shellcheck disable=SC2039
+if ! type mktemp >/dev/null 2>&1; then
+	mktemp()
+	{
+		_dir=false
+		if [ "$1" = "-d" ]; then
+			_dir=true
+			shift
+		fi
+		_d="${TMPDIR:-/tmp}"
+		_hex10=$(dd if=/dev/urandom count=20 2>/dev/null |
+			cksum |
+			awk '{print $1}')
+		_t="${_d}/tmp.${_hex10}"
+		(
+			umask 077
+			if $_dir; then
+				mkdir "$_t"
+			else
+				: >"$_t"
+			fi
+		)
+		echo "$_t"
+	}
+fi
+
+######################################################################
+# NFS callout handling
+
+nfs_callout_init()
+{
+	_state_dir="$1"
+
+	if [ -z "$CTDB_NFS_CALLOUT" ]; then
+		CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
+	fi
+	# Always export, for statd callout
+	export CTDB_NFS_CALLOUT
+
+	# If the callout wants to use this then it must create it
+	export CTDB_NFS_CALLOUT_STATE_DIR="${_state_dir}/callout-state"
+
+	# Export, if set, for use by clustered NFS callouts
+	if [ -n "$CTDB_NFS_STATE_FS_TYPE" ]; then
+		export CTDB_NFS_STATE_FS_TYPE
+	fi
+	if [ -n "$CTDB_NFS_STATE_MNT" ]; then
+		export CTDB_NFS_STATE_MNT
+	fi
+
+	nfs_callout_cache="${_state_dir}/nfs_callout_cache"
+	nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
+	nfs_callout_cache_ops="${nfs_callout_cache}/ops"
+}
+
+nfs_callout_register()
+{
+	mkdir -p "$nfs_callout_cache_ops"
+	rm -f "$nfs_callout_cache_ops"/*
+
+	echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"
+
+	_t=$("$CTDB_NFS_CALLOUT" "register")
+	if [ -n "$_t" ]; then
+		echo "$_t" |
+			while IFS="" read -r _op; do
+				touch "${nfs_callout_cache_ops}/${_op}"
+			done
+	else
+		touch "${nfs_callout_cache_ops}/ALL"
+	fi
+}
+
+nfs_callout()
+{
+	# Re-run registration if $CTDB_NFS_CALLOUT has changed
+	_prev=""
+	if [ -r "$nfs_callout_cache_callout" ]; then
+		read -r _prev <"$nfs_callout_cache_callout"
+	fi
+	if [ "$CTDB_NFS_CALLOUT" != "$_prev" ]; then
+		nfs_callout_register
+	fi
+
+	# Run the operation if it is registered...
+	if [ -e "${nfs_callout_cache_ops}/${1}" ] ||
+		[ -e "${nfs_callout_cache_ops}/ALL" ]; then
+		"$CTDB_NFS_CALLOUT" "$@"
+	fi
+}
+
+########################################################
+# tickle handling
+########################################################
+
+update_tickles()
+{
+	_port="$1"
+
+	tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
+	mkdir -p "$tickledir"
+
+	# What public IPs do I hold?
+	_pnn=$(ctdb_get_pnn)
+	_ips=$($CTDB -X ip | awk -F'|' -v pnn="$_pnn" '$3 == pnn {print $2}')
+
+	# IPs and port as ss filters
+	_ip_filter=""
+	for _ip in $_ips; do
+		_ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
+	done
+	_port_filter="sport == :${_port}"
+
+	# Record connections to our public IPs in a temporary file.
+	# This temporary file is in CTDB's private state directory and
+	# $$ is used to avoid a very rare race involving CTDB's script
+	# debugging.  No security issue, nothing to see here...
+	_my_connections="${tickledir}/${_port}.connections.$$"
+	# Parentheses are needed around the filters for precedence but
+	# the parentheses can't be empty!
+	#
+	# Recent versions of ss print square brackets around IPv6
+	# addresses.  While it is desirable to update CTDB's address
+	# parsing and printing code, something needs to be done here
+	# for backward compatibility, so just delete the brackets.
+	ss -tn state established \
+		"${_ip_filter:+( ${_ip_filter} )}" \
+		"${_port_filter:+( ${_port_filter} )}" |
+		awk 'NR > 1 {print $4, $3}' |
+		tr -d '][' |
+		sort >"$_my_connections"
+
+	# Record our current tickles in a temporary file
+	_my_tickles="${tickledir}/${_port}.tickles.$$"
+	for _i in $_ips; do
+		$CTDB -X gettickles "$_i" "$_port" |
+			awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
+	done |
+		sort >"$_my_tickles"
+
+	# Add tickles for connections that we haven't already got tickles for
+	comm -23 "$_my_connections" "$_my_tickles" |
+		$CTDB addtickle
+
+	# Remove tickles for connections that are no longer there
+	comm -13 "$_my_connections" "$_my_tickles" |
+		$CTDB deltickle
+
+	rm -f "$_my_connections" "$_my_tickles"
+
+	# Remove stale files from killed scripts
+	# Files can't have spaces in name, more portable than -print0/-0
+	# shellcheck disable=SC2038
+	(cd "$tickledir" && find . -type f -mmin +10 | xargs -r rm)
+}
+
+########################################################
+# load a site local config file
+########################################################
+
+[ -x "${CTDB_BASE}/rc.local" ] && {
+	. "${CTDB_BASE}/rc.local"
+}
+
+[ -d "${CTDB_BASE}/rc.local.d" ] && {
+	for i in "${CTDB_BASE}/rc.local.d"/*; do
+		[ -x "$i" ] && . "$i"
+	done
+}
+
+script_name="${0##*/}" # basename