summaryrefslogtreecommitdiffstats
path: root/ctdb/config
diff options
context:
space:
mode:
Diffstat (limited to 'ctdb/config')
-rw-r--r--ctdb/config/README31
-rwxr-xr-xctdb/config/ctdb-crash-cleanup.sh27
-rw-r--r--ctdb/config/ctdb.conf22
-rwxr-xr-xctdb/config/ctdb.init161
-rw-r--r--ctdb/config/ctdb.sudoers3
-rw-r--r--ctdb/config/ctdb.sysconfig11
-rw-r--r--ctdb/config/ctdb.tunables2
-rwxr-xr-xctdb/config/debug-hung-script.sh61
-rwxr-xr-xctdb/config/debug_locks.sh218
-rw-r--r--ctdb/config/events/README193
-rwxr-xr-xctdb/config/events/legacy/00.ctdb.script130
-rwxr-xr-xctdb/config/events/legacy/01.reclock.script34
-rwxr-xr-xctdb/config/events/legacy/05.system.script198
-rwxr-xr-xctdb/config/events/legacy/06.nfs.script39
-rwxr-xr-xctdb/config/events/legacy/10.interface.script262
-rwxr-xr-xctdb/config/events/legacy/11.natgw.script242
-rwxr-xr-xctdb/config/events/legacy/11.routing.script49
-rwxr-xr-xctdb/config/events/legacy/13.per_ip_routing.script438
-rwxr-xr-xctdb/config/events/legacy/20.multipathd.script83
-rwxr-xr-xctdb/config/events/legacy/31.clamd.script37
-rwxr-xr-xctdb/config/events/legacy/40.vsftpd.script57
-rwxr-xr-xctdb/config/events/legacy/41.httpd.script78
-rwxr-xr-xctdb/config/events/legacy/47.samba-dcerpcd.script66
-rwxr-xr-xctdb/config/events/legacy/48.netbios.script75
-rwxr-xr-xctdb/config/events/legacy/49.winbind.script55
-rwxr-xr-xctdb/config/events/legacy/50.samba.script166
-rwxr-xr-xctdb/config/events/legacy/60.nfs.script301
-rwxr-xr-xctdb/config/events/legacy/70.iscsi.script87
-rwxr-xr-xctdb/config/events/legacy/91.lvs.script124
-rwxr-xr-xctdb/config/functions1172
-rw-r--r--ctdb/config/nfs-checks.d/00.portmapper.check2
-rw-r--r--ctdb/config/nfs-checks.d/10.status.check7
-rw-r--r--ctdb/config/nfs-checks.d/20.nfs.check7
-rw-r--r--ctdb/config/nfs-checks.d/30.nlockmgr.check6
-rw-r--r--ctdb/config/nfs-checks.d/40.mountd.check7
-rw-r--r--ctdb/config/nfs-checks.d/50.rquotad.check7
-rw-r--r--ctdb/config/nfs-checks.d/README31
-rwxr-xr-xctdb/config/nfs-linux-kernel-callout441
-rwxr-xr-xctdb/config/notification.README36
-rwxr-xr-xctdb/config/notify.sh19
-rw-r--r--ctdb/config/script.options16
-rwxr-xr-xctdb/config/statd-callout254
42 files changed, 5255 insertions, 0 deletions
diff --git a/ctdb/config/README b/ctdb/config/README
new file mode 100644
index 0000000..d28f4f0
--- /dev/null
+++ b/ctdb/config/README
@@ -0,0 +1,31 @@
+This directory contains run-time support scripts for CTDB.
+
+Selected highlights:
+
+ ctdb.init
+
+ An initscript for starting ctdbd at boot time.
+
+ events/
+
+ Eventscripts. See events/README for more details.
+
+ functions
+
+ Support functions, sourced by eventscripts and other scripts.
+
+ statd-callout
+
+ rpc.statd high-availability callout to support lock migration on
+ failover.
+
+Notes:
+
+* All of these scripts are written in POSIX Bourne shell. Please
+ avoid bash-isms, including the use of "local" variables (which are
+ not available in POSIX shell).
+
+* Do not use absolute paths for commands. Unit tests attempt to
+ replace many commands with stubs and can not do this if commands are
+ specified with absolute paths. The functions file controls $PATH so
+ absolute paths should not be required.
diff --git a/ctdb/config/ctdb-crash-cleanup.sh b/ctdb/config/ctdb-crash-cleanup.sh
new file mode 100755
index 0000000..95cfd75
--- /dev/null
+++ b/ctdb/config/ctdb-crash-cleanup.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# This script can be called from a cronjob to automatically drop/release
+# all public ip addresses if CTDBD has crashed or stopped running.
+#
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && echo "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+# If ctdb is running, just exit
+if service ctdb status >/dev/null 2>&1 ; then
+ exit 0
+fi
+
+load_script_options "failover" "11.natgw"
+
+if [ ! -f "$CTDB_BASE/public_addresses" ] ; then
+ die "No public addresses file found. Can't clean up."
+fi
+
+drop_all_public_ips 2>&1 | script_log "ctdb-crash-cleanup.sh"
+
+if [ -n "$CTDB_NATGW_PUBLIC_IP" ] ; then
+ drop_ip "$CTDB_NATGW_PUBLIC_IP" "ctdb-crash-cleanup.sh"
+fi
diff --git a/ctdb/config/ctdb.conf b/ctdb/config/ctdb.conf
new file mode 100644
index 0000000..8e1b376
--- /dev/null
+++ b/ctdb/config/ctdb.conf
@@ -0,0 +1,22 @@
+# See ctdb.conf(5) for documentation
+#
+# See ctdb-script.options(5) for documentation about event script
+# options
+
+[logging]
+ # Enable logging to syslog
+ # location = syslog
+
+ # Default log level
+ # log level = NOTICE
+
+[cluster]
+ # Shared cluster lock file to avoid split brain. Daemon
+ # default is no cluster lock. Do NOT run CTDB without a
+ # cluster lock file unless you know exactly what you are
+ # doing.
+ #
+ # Please see the CLUSTER LOCK section in ctdb(7) for more
+ # details.
+ #
+ # cluster lock = !/bin/false CLUSTER LOCK NOT CONFIGURED
diff --git a/ctdb/config/ctdb.init b/ctdb/config/ctdb.init
new file mode 100755
index 0000000..6a7f781
--- /dev/null
+++ b/ctdb/config/ctdb.init
@@ -0,0 +1,161 @@
+#!/bin/sh
+
+# Start and stop CTDB (Clustered TDB daemon)
+#
+# chkconfig: - 90 01
+#
+# description: Starts and stops CTDB
+# pidfile: /var/run/ctdb/ctdbd.pid
+# config: /etc/sysconfig/ctdb
+
+### BEGIN INIT INFO
+# Provides: ctdb
+# Required-Start: $local_fs $syslog $network $remote_fs
+# Required-Stop: $local_fs $syslog $network $remote_fs
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: start and stop ctdb service
+# Description: Start and stop CTDB (Clustered TDB daemon)
+### END INIT INFO
+
+# Source function library.
+if [ -f /etc/init.d/functions ] ; then
+ # Red Hat
+ . /etc/init.d/functions
+elif [ -f /etc/rc.d/init.d/functions ] ; then
+ # Red Hat
+ . /etc/rc.d/init.d/functions
+elif [ -f /etc/rc.status ] ; then
+ # SUSE
+ . /etc/rc.status
+ rc_reset
+ LC_ALL=en_US.UTF-8
+elif [ -f /lib/lsb/init-functions ] ; then
+ # Debian
+ . /lib/lsb/init-functions
+fi
+
+# Avoid using root's TMPDIR
+unset TMPDIR
+
+[ -n "$CTDB_BASE" ] || export CTDB_BASE="/etc/ctdb"
+
+. "${CTDB_BASE}/functions"
+
+load_system_config "network"
+
+# check networking is up (for redhat)
+if [ "$NETWORKING" = "no" ] ; then
+ exit 0
+fi
+
+load_system_config "ctdb"
+
+detect_init_style
+export CTDB_INIT_STYLE
+
+ctdbd="${CTDBD:-/usr/sbin/ctdbd}"
+ctdb="${CTDB:-/usr/bin/ctdb}"
+pidfile="/var/run/ctdb/ctdbd.pid"
+
+############################################################
+
+start()
+{
+ printf "Starting ctdbd service: "
+
+ case "$CTDB_INIT_STYLE" in
+ suse)
+ startproc "$ctdbd"
+ rc_status -v
+ ;;
+ redhat)
+ daemon --pidfile "$pidfile" "$ctdbd"
+ RETVAL=$?
+ echo
+ [ $RETVAL -eq 0 ] && touch /var/lock/subsys/ctdb || RETVAL=1
+ return $RETVAL
+ ;;
+ debian)
+ eval start-stop-daemon --start --quiet --background --exec "$ctdbd"
+ ;;
+ esac
+}
+
+stop()
+{
+ printf "Shutting down ctdbd service: "
+
+ case "$CTDB_INIT_STYLE" in
+ suse)
+ "$ctdb" "shutdown"
+ rc_status -v
+ ;;
+ redhat)
+ "$ctdb" "shutdown"
+ RETVAL=$?
+ # Common idiom in Red Hat init scripts - success() always
+ # succeeds so this does behave like if-then-else
+ # shellcheck disable=SC2015
+ [ $RETVAL -eq 0 ] && success || failure
+ echo ""
+ [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/ctdb
+ return $RETVAL
+ ;;
+ debian)
+ "$ctdb" "shutdown"
+ log_end_msg $?
+ ;;
+ esac
+}
+
+restart()
+{
+ stop
+ start
+}
+
+check_status ()
+{
+ case "$CTDB_INIT_STYLE" in
+ suse)
+ checkproc -p "$pidfile" "$ctdbd"
+ rc_status -v
+ ;;
+ redhat)
+ status -p "$pidfile" -l "ctdb" "$ctdbd"
+ ;;
+ debian)
+ status_of_proc -p "$pidfile" "$ctdbd" "ctdb"
+ ;;
+ esac
+}
+
+############################################################
+
+case "$1" in
+ start)
+ start
+ ;;
+ stop)
+ stop
+ ;;
+ restart|reload|force-reload)
+ restart
+ ;;
+ status)
+ check_status
+ ;;
+ condrestart|try-restart)
+ if check_status >/dev/null ; then
+ restart
+ fi
+ ;;
+ cron)
+ # used from cron to auto-restart ctdb
+ check_status >/dev/null 2>&1 || restart
+ ;;
+ *)
+ echo "Usage: $0 {start|stop|restart|reload|force-reload|status|cron|condrestart|try-restart}"
+ exit 1
+esac
diff --git a/ctdb/config/ctdb.sudoers b/ctdb/config/ctdb.sudoers
new file mode 100644
index 0000000..1c23818
--- /dev/null
+++ b/ctdb/config/ctdb.sudoers
@@ -0,0 +1,3 @@
+Defaults!/usr/local/etc/ctdb/statd-callout !requiretty
+
+rpcuser ALL=(ALL) NOPASSWD: /usr/local/etc/ctdb/statd-callout
diff --git a/ctdb/config/ctdb.sysconfig b/ctdb/config/ctdb.sysconfig
new file mode 100644
index 0000000..fc57929
--- /dev/null
+++ b/ctdb/config/ctdb.sysconfig
@@ -0,0 +1,11 @@
+# If using SYSV init, install this as /etc/sysconfig/ctdb,
+# /etc/default/ctdb or similar
+
+# Allow 1M open files
+ulimit -n 1048576
+
+# Allow core files to be created
+ulimit -c unlimited
+
+# Useful if default detection doesn't work
+# CTDB_INIT_STYLE=debian
diff --git a/ctdb/config/ctdb.tunables b/ctdb/config/ctdb.tunables
new file mode 100644
index 0000000..b99e5cd
--- /dev/null
+++ b/ctdb/config/ctdb.tunables
@@ -0,0 +1,2 @@
+# Set some CTDB tunable variables during CTDB startup?
+# MutexEnabled=0
diff --git a/ctdb/config/debug-hung-script.sh b/ctdb/config/debug-hung-script.sh
new file mode 100755
index 0000000..c1ac0f1
--- /dev/null
+++ b/ctdb/config/debug-hung-script.sh
@@ -0,0 +1,61 @@
+#!/bin/sh
+
+# This script only works on Linux. Please modify (and submit patches)
+# for other operating systems.
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && echo "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+load_script_options
+
+# Testing hook
+if [ -n "$CTDB_DEBUG_HUNG_SCRIPT_LOGFILE" ] ; then
+ tmp="${CTDB_DEBUG_HUNG_SCRIPT_LOGFILE}.part"
+ exec >>"$tmp" 2>&1
+fi
+
+(
+ # No use running several of these in parallel if, say, "releaseip"
+ # event hangs for multiple IPs. In that case the output would be
+ # interleaved in the log and would just be confusing.
+ flock --wait 2 9 || exit 1
+
+ echo "===== Start of hung script debug for PID=\"$1\", event=\"$2\" ====="
+
+ echo "pstree -p -a ${1}:"
+ out=$(pstree -p -a "$1")
+ echo "$out"
+
+ # Check for processes matching a regular expression and print
+ # stack staces. This could help confirm that certain processes
+ # are stuck in certain places such as the cluster filesystem. The
+ # regexp must separate items with "|" and must not contain
+ # parentheses. The default pattern can be replaced for testing.
+ default_pat='exportfs|rpcinfo'
+ pat="${CTDB_DEBUG_HUNG_SCRIPT_STACKPAT:-${default_pat}}"
+ echo "$out" |
+ sed -r -n "s@.*-(.*(${pat}).*),([0-9]*).*@\\3 \\1@p" |
+ while read pid name ; do
+ trace=$(cat "/proc/${pid}/stack" 2>/dev/null)
+ # No! Checking the exit code afterwards is actually clearer...
+ # shellcheck disable=SC2181
+ if [ $? -eq 0 ] ; then
+ echo "---- Stack trace of interesting process ${pid}[${name}] ----"
+ echo "$trace"
+ fi
+ done
+
+ if [ "$2" != "init" ] ; then
+ echo "---- ctdb scriptstatus ${2}: ----"
+ $CTDB scriptstatus "$2"
+ fi
+
+ echo "===== End of hung script debug for PID=\"$1\", event=\"$2\" ====="
+
+ if [ -n "$CTDB_DEBUG_HUNG_SCRIPT_LOGFILE" ] ; then
+ mv "$tmp" "$CTDB_DEBUG_HUNG_SCRIPT_LOGFILE"
+ fi
+
+) 9>"${CTDB_SCRIPT_VARDIR}/debug-hung-script.lock"
diff --git a/ctdb/config/debug_locks.sh b/ctdb/config/debug_locks.sh
new file mode 100755
index 0000000..6c730ee
--- /dev/null
+++ b/ctdb/config/debug_locks.sh
@@ -0,0 +1,218 @@
+#!/bin/sh
+
+# This script attempts to find processes holding locks on a particular
+# CTDB database and dumps a stack trace for each such processe.
+#
+# There are 2 cases:
+#
+# * Samba is configured to use fcntl locks
+#
+# In this case /proc/locks is parsed to find potential lock holders
+#
+# * Samba is configured to use POSIX robust mutexes
+#
+# In this case the helper program tdb_mutex_check is used to find
+# potential lock holders.
+#
+# This helper program uses a private glibc struct field, so is
+# neither portable nor supported. If this field is not available
+# then the helper is not built. Unexpected changes in internal
+# glibc structures may cause unexpected results, including crashes.
+# Bug reports for this helper program are not accepted without an
+# accompanying patch.
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && echo "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+if [ $# -ne 4 ] ; then
+ die "usage: $0 <pid> { DB | RECORD } <tdb_path> { FCNTL | MUTEX }"
+fi
+
+lock_helper_pid="$1"
+# lock_scope is unused for now
+# shellcheck disable=SC2034
+lock_scope="$2"
+tdb_path="$3"
+lock_type="$4"
+
+# type is at least mentioned in POSIX and more is portable than which(1)
+# shellcheck disable=SC2039
+if ! type gstack >/dev/null 2>&1 ; then
+ gstack ()
+ {
+ _pid="$1"
+
+ gdb -batch --quiet -nx "/proc/${_pid}/exe" "$_pid" \
+ -ex "thread apply all bt" 2>/dev/null |
+ grep '^\(#\|Thread \)'
+ }
+fi
+
+# Load/cache database options from configuration file
+ctdb_get_db_options
+
+dump_stack ()
+{
+ _pid="$1"
+
+ echo "----- Stack trace for PID=${_pid} -----"
+ _state=$(ps -p "$_pid" -o state= | cut -c 1)
+ if [ "$_state" = "D" ] ; then
+ # Don't run gstack on a process in D state since
+ # gstack will hang until the process exits D state.
+ # Although it is possible for a process to transition
+ # to D state after this check, it is unlikely because
+ # if a process is stuck in D state then it is probably
+ # the reason why this script was called. Note that a
+ # kernel stack almost certainly won't help diagnose a
+ # deadlock... but it will probably give us someone to
+ # blame!
+ echo "----- Process in D state, printing kernel stack only"
+ get_proc "${_pid}/stack"
+ else
+ gstack "$_pid"
+ fi
+}
+
+dump_stacks ()
+{
+ _pids="$1"
+
+ # Use word splitting to squash whitespace
+ # shellcheck disable=SC2086
+ _pids=$(echo $_pids | tr ' ' '\n' | sort -u)
+
+ for _pid in $_pids; do
+ dump_stack "$_pid"
+ done
+}
+
+get_tdb_file_id ()
+{
+ if ! _device_inode=$(stat -c "%d:%i" "$tdb_path" 2>/dev/null) ; then
+ die "Unable to stat \"${tdb_path}\""
+ fi
+ _device="${_device_inode%%:*}"
+ _device_major=$((_device >> 8))
+ _device_minor=$((_device & 0xff))
+ _inode="${_device_inode#*:}"
+ printf '%02x:%02x:%u\n' "$_device_major" "$_device_minor" "$_inode"
+}
+
+debug_via_proc_locks ()
+{
+ # Get file ID to match relevant column in /proc/locks
+ _file_id=$(get_tdb_file_id)
+
+ # Log information from /proc/locks about the waiting process
+ _tdb=$(basename "$tdb_path")
+ _comm=$(ps -p "$lock_helper_pid" -o comm=)
+ _out=$(get_proc "locks" |
+ awk -v pid="$lock_helper_pid" \
+ -v file_id="$_file_id" \
+ -v file="$_tdb" \
+ -v comm="$_comm" \
+ '$2 == "->" &&
+ $3 == "POSIX" &&
+ $4 == "ADVISORY" &&
+ $5 == "WRITE" &&
+ $6 == pid &&
+ $7 == file_id { print $6, comm, file, $8, $9 }')
+ if [ -n "$_out" ] ; then
+ echo "Waiter:"
+ echo "$_out"
+ fi
+
+ # Parse /proc/locks and find process holding locks on $tdb_path
+ # extract following information
+ # pid process_name tdb_name offsets
+ _out=$(get_proc "locks" |
+ awk -v pid="$lock_helper_pid" \
+ -v file_id="$_file_id" \
+ -v file="$_tdb" \
+ '$2 == "POSIX" &&
+ $3 == "ADVISORY" &&
+ $4 == "WRITE" &&
+ $5 != pid &&
+ $6 == file_id { print $5, file, $7, $8 }' |
+ while read -r _pid _rest ; do
+ _pname=$(ps -p "$_pid" -o comm=)
+ echo "$_pid $_pname $_rest"
+ done)
+
+ if [ -z "$_out" ]; then
+ return
+ fi
+
+ # Log information about locks
+ echo "Lock holders:"
+ echo "$_out"
+
+ _pids=$(echo "$_out" | awk '{ print $1 }')
+
+ lock_holder_pids="${lock_holder_pids:+${lock_holder_pids} }${_pids}"
+}
+
+debug_via_tdb_mutex ()
+{
+ _helper="${CTDB_HELPER_BINDIR}/tdb_mutex_check"
+ if [ ! -x "$_helper" ] ; then
+ # Mutex helper not available - not supported?
+ # Avoid not found error...
+ return
+ fi
+
+ # Helper should always succeed
+ if ! _t=$("$_helper" "$tdb_path") ; then
+ return
+ fi
+
+ _out=$(echo "$_t" | sed -n -e 's#^\[\(.*\)\] pid=\(.*\)#\2 \1#p')
+
+ if [ -z "$_out" ]; then
+ if [ -n "$_t" ] ; then
+ echo "$_t" | grep -F 'trylock failed'
+ fi
+ return
+ fi
+
+ # Get process names, append $tdb_path
+ _out=$(echo "$_out" |
+ while read -r _pid _rest ; do
+ _pname=$(ps -p "$_pid" -o comm=)
+ _tdb=$(basename "$tdb_path")
+ echo "${_pid} ${_pname} ${_tdb} ${_rest}"
+ done)
+
+ # Log information about locks
+ echo "Lock holders:"
+ echo "$_out"
+
+ # Get PIDs of processes that are holding locks
+ _pids=$(echo "$_out" |
+ awk -v pid="$lock_helper_pid" '$1 != pid {print $1}')
+
+ lock_holder_pids="${lock_holder_pids:+${lock_holder_pids} }${_pids}"
+}
+
+(
+ flock -n 9 || exit 1
+
+ echo "===== Start of debug locks PID=$$ ====="
+
+ lock_holder_pids=""
+
+ debug_via_proc_locks
+
+ if [ "$lock_type" = "MUTEX" ] ; then
+ debug_via_tdb_mutex
+ fi
+
+ dump_stacks "$lock_holder_pids"
+
+ echo "===== End of debug locks PID=$$ ====="
+)9>"${CTDB_SCRIPT_VARDIR}/debug_locks.lock" | script_log "ctdbd-lock"
+
+exit 0
diff --git a/ctdb/config/events/README b/ctdb/config/events/README
new file mode 100644
index 0000000..6553830
--- /dev/null
+++ b/ctdb/config/events/README
@@ -0,0 +1,193 @@
+The events/ directory contains event scripts used by CTDB. Event
+scripts are triggered on certain events, such as startup, monitoring
+or public IP allocation. Scripts may be specific to services,
+networking or internal CTDB operations.
+
+Scripts are divided into subdirectories for different CTDB components.
+Right now the only component is "legacy".
+
+All event scripts start with the prefix 'NN.' where N is a digit. The
+event scripts are run in sequence based on NN. Thus 10.interface will
+be run before 60.nfs. It is recommended to keep each NN unique.
+However, scripts with the same NN prefix will be executed in
+alphanumeric sort order.
+
+As a special case, any eventscript that ends with a '~' character will be
+ignored since this is a common postfix that some editors will append to
+older versions of a file. Similarly, any eventscript with multiple '.'s
+will be ignored as package managers can create copies with additional
+suffix starting with '.' (e.g. .rpmnew, .dpkg-dist).
+
+Only executable event scripts are run by CTDB. Any event script that
+does not have execute permission is ignored.
+
+The eventscripts are called with varying number of arguments. The
+first argument is the event name and the rest of the arguments depend
+on the event name.
+
+Event scripts must return 0 for success and non-zero for failure.
+
+Output of event scripts is logged. On failure the output of the
+failing event script is included in the output of "ctdb scriptstatus".
+
+The following events are supported (with arguments shown):
+
+init
+
+ This event is triggered once when CTDB is starting up. This
+ event is used to do some basic cleanup and initialisation.
+
+ During the "init" event CTDB is not listening on its Unix
+ domain socket, so the "ctdb" CLI will not work.
+
+ Failure of this event will cause CTDB to terminate.
+
+ Example: 00.ctdb creates $CTDB_SCRIPT_VARDIR
+
+setup
+
+ This event is triggered once, after the "init" event has
+ completed.
+
+ For this and any subsequent events the CTDB Unix domain socket
+ is available, so the "ctdb" CLI will work.
+
+ Failure of this event will cause CTDB to terminate.
+
+ Example: 11.natgw checks that it has valid configuration
+
+startup
+
+ This event is triggered after the "setup" event has completed
+ and CTDB has finished its initial database recovery.
+
+ This event starts all services that are managed by CTDB. Each
+ service that is managed by CTDB should implement this event
+ and use it to (re)start the service.
+
+ If the "startup" event fails then CTDB will retry it until it
+ succeeds. There is no limit on the number of retries.
+
+ Example: 50.samba uses this event to start the Samba daemon.
+
+shutdown
+
+ This event is triggered when CTDB is shutting down.
+
+ This event shuts down all services that are managed by CTDB.
+ Each service that is managed by CTDB should implement this
+ event and use it to stop the service.
+
+ Example: 50.samba uses this event to shut down the Samba
+ daemon.
+
+monitor
+
+ This event is run periodically. The interval between
+ successive "monitor" events is configured using the
+ MonitorInterval tunable, which defaults to 15 seconds.
+
+ This event is triggered by CTDB to continuously monitor that
+ all managed services are healthy. If all event scripts
+ complete then the monitor event successfully then the node is
+ marked HEALTHY. If any event script fails then no subsequent
+ scripts will be run for that event and the node is marked
+ UNHEALTHY.
+
+ Each service that is managed by CTDB should implement this
+ event and use it to monitor the service.
+
+ Example: 10.interface checks that each configured interface
+ for public IP addresses has a physical link established.
+
+startrecovery
+
+ This event is triggered every time a database recovery process
+ is started.
+
+ This is rarely used.
+
+recovered
+
+ This event is triggered every time a database recovery process
+ is completed.
+
+ This is rarely used.
+
+takeip <interface> <ip-address> <netmask-bits>
+
+ This event is triggered for each public IP address taken by a
+ node during IP address (re)assignment. Multiple "takeip"
+ events can be run in parallel if multiple IP addresses are
+ being assigned.
+
+ Example: In 10.interface the "ip" command (from the Linux
+ iproute2 package) is used to add the specified public IP
+ address to the specified interface. The "ip" command can
+ safely be run concurrently. However, the "iptables" command
+ cannot be run concurrently so a wrapper is used to serialise
+ runs using exclusive locking.
+
+ If substantial work is required to reconfigure a service when
+ a public IP address is taken over it can be better to defer
+ service reconfiguration to the "ipreallocated" event, after
+ all IP addresses have been assigned.
+
+ Example: 60.nfs uses ctdb_service_set_reconfigure() to flag
+ that public IP addresses have changed so that service
+ reconfiguration will occur in the "ipreallocated" event.
+
+releaseip <interface> <ip-address> <netmask-bits>
+
+ This event is triggered for each public IP address released by
+ a node during IP address (re)assignment. Multiple "releaseip"
+ events can be run in parallel if multiple IP addresses are
+ being unassigned.
+
+ In all other regards, this event is analogous to the "takeip"
+ event above.
+
+updateip <old-interface> <new-interface> <ip-address> <netmask-bits>
+
+ This event is triggered for each public IP address moved
+ between interfaces on a node during IP address (re)assignment.
+ Multiple "updateip" events can be run in parallel if multiple
+ IP addresses are being moved.
+
+ This event is only used if multiple interfaces are capable of
+ hosting an IP address, as specified in the public addresses
+ configuration file.
+
+ This event is similar to the "takeip" event above.
+
+ipreallocated
+
+ This event is triggered on all nodes as the last step of
+ public IP address (re)assignment. It is unconditionally
+ triggered after any "releaseip", "takeip" and "updateip"
+ events, even though these events may not run on some nodes if
+ there are no relevant changes. That is, the "ipreallocated"
+ event is triggered unconditionally, even on nodes where public
+ IP addresses assignments have not changed.
+
+ This event is used to reconfigure services.
+
+ Since "ipreallocated" is always run, this allows
+ reconfiguration to depend on the states of other nodes rather
+ that just IP addresses.
+
+ Example: 11.natgw recalculates the NAT gateway master and
+ updates the relevant network configuration on each node if the
+ NAT gateway master has changed.
+
+Additional notes for "takeip", "releaseip", "updateip",
+"ipreallocated":
+
+* Failure of any of these events causes IP allocation to be retried.
+
+* An event script can use ctdb_service_set_reconfigure() in "takeip",
+ "releaseip" or "updateip" events to flag that its service needs to
+ be reconfigured. The "ipreallocated" event can then use
+ ctdb_service_needs_reconfigure() to test if there were public IPs
+ changes to determine what type of reconfiguration (if any) is
+ needed.
diff --git a/ctdb/config/events/legacy/00.ctdb.script b/ctdb/config/events/legacy/00.ctdb.script
new file mode 100755
index 0000000..81c16af
--- /dev/null
+++ b/ctdb/config/events/legacy/00.ctdb.script
@@ -0,0 +1,130 @@
+#!/bin/sh
+
+# Event script for ctdb-specific setup and other things that don't fit
+# elsewhere.
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+load_script_options
+
+############################################################
+
+# type is commonly supported and more portable than which(1)
+# shellcheck disable=SC2039
+select_tdb_checker ()
+{
+ # Find the best TDB consistency check available.
+ use_tdb_tool_check=false
+ type tdbtool >/dev/null 2>&1 && found_tdbtool=true
+ type tdbdump >/dev/null 2>&1 && found_tdbdump=true
+
+ if $found_tdbtool && echo "help" | tdbtool | grep -q check ; then
+ use_tdb_tool_check=true
+ elif $found_tdbtool && $found_tdbdump ; then
+ cat <<EOF
+WARNING: The installed 'tdbtool' does not offer the 'check' subcommand.
+ Using 'tdbdump' for database checks.
+ Consider updating 'tdbtool' for better checks!
+EOF
+ elif $found_tdbdump ; then
+ cat <<EOF
+WARNING: 'tdbtool' is not available.
+ Using 'tdbdump' to check the databases.
+ Consider installing a recent 'tdbtool' for better checks!
+EOF
+ else
+ cat <<EOF
+WARNING: Cannot check databases since neither
+ 'tdbdump' nor 'tdbtool check' is available.
+ Consider installing tdbtool or at least tdbdump!
+EOF
+ return 1
+ fi
+}
+
+check_tdb ()
+{
+ _db="$1"
+
+ if $use_tdb_tool_check ; then
+ # tdbtool always exits with 0 :-(
+ if timeout 10 tdbtool "$_db" check 2>/dev/null |
+ grep -q "Database integrity is OK" ; then
+ return 0
+ else
+ return 1
+ fi
+ else
+ timeout 10 tdbdump "$_db" >/dev/null 2>/dev/null
+ return $?
+ fi
+}
+
+check_persistent_databases ()
+{
+ _dir="${CTDB_DBDIR_PERSISTENT:-${CTDB_VARDIR}/persistent}"
+ [ -d "$_dir" ] || return 0
+
+ for _db in "$_dir/"*.tdb.*[0-9] ; do
+ [ -r "$_db" ] || continue
+ check_tdb "$_db" || \
+ die "Persistent database $_db is corrupted! CTDB will not start."
+ done
+}
+
+check_non_persistent_databases ()
+{
+ _dir="${CTDB_DBDIR:-${CTDB_VARDIR}}"
+ [ -d "$_dir" ] || return 0
+
+ for _db in "${_dir}/"*.tdb.*[0-9] ; do
+ [ -r "$_db" ] || continue
+ check_tdb "$_db" || {
+ _backup="${_db}.$(date +'%Y%m%d.%H%M%S').corrupt"
+ cat <<EOF
+WARNING: database ${_db} is corrupted.
+ Moving to backup ${_backup} for later analysis.
+EOF
+ mv "$_db" "$_backup"
+
+ # Now remove excess backups
+ _max="${CTDB_MAX_CORRUPT_DB_BACKUPS:-10}"
+ _bdb="${_db##*/}" # basename
+ find "$_dir" -name "${_bdb}.*.corrupt" |
+ sort -r |
+ tail -n +$((_max + 1)) |
+ xargs rm -f
+ }
+ done
+}
+
+############################################################
+
+ctdb_check_args "$@"
+
+case "$1" in
+init)
+ # make sure we have a blank state directory for the scripts to work with
+ rm -rf "$CTDB_SCRIPT_VARDIR"
+ mkdir -p "$CTDB_SCRIPT_VARDIR" || \
+ die "mkdir -p ${CTDB_SCRIPT_VARDIR} - failed - $?" $?
+
+ # Load/cache database options from configuration file
+ ctdb_get_db_options
+
+ if select_tdb_checker ; then
+ check_persistent_databases || exit $?
+ check_non_persistent_databases
+ fi
+ ;;
+
+startup)
+ $CTDB attach ctdb.tdb persistent
+ ;;
+esac
+
+# all OK
+exit 0
diff --git a/ctdb/config/events/legacy/01.reclock.script b/ctdb/config/events/legacy/01.reclock.script
new file mode 100755
index 0000000..0406875
--- /dev/null
+++ b/ctdb/config/events/legacy/01.reclock.script
@@ -0,0 +1,34 @@
+#!/bin/sh
+# script to check accessibility to the reclock file on a node
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+case "$1" in
+init)
+ recovery_lock=$("${CTDB_HELPER_BINDIR}/ctdb-config" \
+ get cluster "recovery lock")
+ # xshellcheck disable=SC2181
+ # Above is already complicated enough without embedding into "if"
+ case $? in
+ 0) : ;;
+ 2) exit 0 ;; # ENOENT: not configured
+ *) die "Unexpected error getting recovery lock configuration"
+ esac
+
+ if [ -z "$recovery_lock" ] ; then
+ exit 0
+ fi
+
+ # If a helper is specified then exit because this script can't
+ # do anything useful
+ case "$recovery_lock" in
+ !*) exit 0 ;;
+ esac
+
+ d=$(dirname "$recovery_lock")
+ mkdir -p "$d"
+ ;;
+esac
diff --git a/ctdb/config/events/legacy/05.system.script b/ctdb/config/events/legacy/05.system.script
new file mode 100755
index 0000000..bf36dd2
--- /dev/null
+++ b/ctdb/config/events/legacy/05.system.script
@@ -0,0 +1,198 @@
+#!/bin/sh
+# ctdb event script for checking local file system utilization
+
+[ -n "$CTDB_BASE" ] ||
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+load_script_options
+
+ctdb_setup_state_dir "service" "system-monitoring"
+
+validate_percentage()
+{
+ case "$1" in
+ "") return 1 ;; # A failure that doesn't need a warning
+ [0-9] | [0-9][0-9] | 100) return 0 ;;
+ *)
+ echo "WARNING: ${1} is an invalid percentage in \"${2}\" check"
+ return 1
+ ;;
+ esac
+}
+
+check_thresholds()
+{
+ _thing="$1"
+ _thresholds="$2"
+ _usage="$3"
+ _unhealthy_callout="$4"
+
+ case "$_thresholds" in
+ *:*)
+ _warn_threshold="${_thresholds%:*}"
+ _unhealthy_threshold="${_thresholds#*:}"
+ ;;
+ *)
+ _warn_threshold="$_thresholds"
+ _unhealthy_threshold=""
+ ;;
+ esac
+
+ _t=$(echo "$_thing" | sed -e 's@/@SLASH_@g' -e 's@ @_@g')
+ # script_state_dir set by ctdb_setup_state_dir()
+ # shellcheck disable=SC2154
+ _cache="${script_state_dir}/cache_${_t}"
+ if [ -r "$_cache" ]; then
+ read -r _prev <"$_cache"
+ else
+ _prev=0
+ fi
+ if validate_percentage "$_unhealthy_threshold" "$_thing"; then
+ if [ "$_usage" -ge "$_unhealthy_threshold" ]; then
+ printf 'ERROR: %s utilization %d%% >= threshold %d%%\n' \
+ "$_thing" \
+ "$_usage" \
+ "$_unhealthy_threshold"
+ # Only run unhealthy callout if passing the
+ # unhealthy threshold. That is, if the
+ # previous usage was below the threshold.
+ if [ "$_prev" -lt "$_unhealthy_threshold" ]; then
+ eval "$_unhealthy_callout"
+ fi
+ echo "$_usage" >"$_cache"
+ exit 1
+ fi
+ fi
+
+ if validate_percentage "$_warn_threshold" "$_thing"; then
+ if [ "$_usage" -ge "$_warn_threshold" ]; then
+ if [ "$_usage" = "$_prev" ]; then
+ return
+ fi
+ printf 'WARNING: %s utilization %d%% >= threshold %d%%\n' \
+ "$_thing" \
+ "$_usage" \
+ "$_warn_threshold"
+ echo "$_usage" >"$_cache"
+ else
+ if [ ! -r "$_cache" ]; then
+ return
+ fi
+ printf 'NOTICE: %s utilization %d%% < threshold %d%%\n' \
+ "$_thing" \
+ "$_usage" \
+ "$_warn_threshold"
+ rm -f "$_cache"
+ fi
+ fi
+}
+
+set_monitor_filsystem_usage_defaults()
+{
+ _fs_defaults_cache="${script_state_dir}/cache_filsystem_usage_defaults"
+
+ if [ ! -r "$_fs_defaults_cache" ]; then
+ # Determine filesystem for each database directory, generate
+ # an entry to warn at 90%, de-duplicate entries, put all items
+ # on 1 line (so the read below gets everything)
+ for _t in "${CTDB_DBDIR:-${CTDB_VARDIR}}" \
+ "${CTDB_DBDIR_PERSISTENT:-${CTDB_VARDIR}/persistent}" \
+ "${CTDB_DBDIR_STATE:-${CTDB_VARDIR}/state}"; do
+ df -kP "$_t" | awk 'NR == 2 { printf "%s:90\n", $6 }'
+ done | sort -u | xargs >"$_fs_defaults_cache"
+ fi
+
+ read -r CTDB_MONITOR_FILESYSTEM_USAGE <"$_fs_defaults_cache"
+}
+
+monitor_filesystem_usage()
+{
+ if [ -z "$CTDB_MONITOR_FILESYSTEM_USAGE" ]; then
+ set_monitor_filsystem_usage_defaults
+ fi
+
+ # Check each specified filesystem, specified in format
+ # <fs_mount>:<fs_warn_threshold>[:fs_unhealthy_threshold]
+ for _fs in $CTDB_MONITOR_FILESYSTEM_USAGE; do
+ _fs_mount="${_fs%%:*}"
+ _fs_thresholds="${_fs#*:}"
+
+ if [ ! -d "$_fs_mount" ]; then
+ echo "WARNING: Directory ${_fs_mount} does not exist"
+ continue
+ fi
+
+ # Get current utilization
+ _fs_usage=$(df -kP "$_fs_mount" |
+ sed -n -e 's@.*[[:space:]]\([[:digit:]]*\)%.*@\1@p')
+ if [ -z "$_fs_usage" ]; then
+ printf 'WARNING: Unable to get FS utilization for %s\n' \
+ "$_fs_mount"
+ continue
+ fi
+
+ check_thresholds "Filesystem ${_fs_mount}" \
+ "$_fs_thresholds" \
+ "$_fs_usage"
+ done
+}
+
+# shellcheck disable=SC2317
+# Called indirectly via check_thresholds()
+dump_memory_info()
+{
+ get_proc "meminfo"
+ ps auxfww
+ set_proc "sysrq-trigger" "m"
+}
+
+monitor_memory_usage()
+{
+ # Defaults
+ if [ -z "$CTDB_MONITOR_MEMORY_USAGE" ]; then
+ CTDB_MONITOR_MEMORY_USAGE=80
+ fi
+
+ _meminfo=$(get_proc "meminfo")
+ # Intentional word splitting here
+ # shellcheck disable=SC2046
+ set -- $(echo "$_meminfo" | awk '
+$1 == "MemAvailable:" { memavail += $2 }
+$1 == "MemFree:" { memfree += $2 }
+$1 == "Cached:" { memfree += $2 }
+$1 == "Buffers:" { memfree += $2 }
+$1 == "MemTotal:" { memtotal = $2 }
+$1 == "SwapFree:" { swapfree = $2 }
+$1 == "SwapTotal:" { swaptotal = $2 }
+END {
+ if (memavail != 0) { memfree = memavail ; }
+ if (memtotal + swaptotal != 0) {
+ usedtotal = memtotal - memfree + swaptotal - swapfree
+ print int(usedtotal / (memtotal + swaptotal) * 100)
+ } else {
+ print 0
+ }
+}')
+ _mem_usage="$1"
+
+ check_thresholds "System memory" \
+ "$CTDB_MONITOR_MEMORY_USAGE" \
+ "$_mem_usage" \
+ dump_memory_info
+}
+
+case "$1" in
+monitor)
+ # Load/cache database options from configuration file
+ ctdb_get_db_options
+
+ rc=0
+ monitor_filesystem_usage || rc=$?
+ monitor_memory_usage || rc=$?
+ exit $rc
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/06.nfs.script b/ctdb/config/events/legacy/06.nfs.script
new file mode 100755
index 0000000..b937d43
--- /dev/null
+++ b/ctdb/config/events/legacy/06.nfs.script
@@ -0,0 +1,39 @@
+#!/bin/sh
+# script to manage nfs in a clustered environment
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+service_name="nfs"
+
+load_script_options "service" "60.nfs"
+
+ctdb_setup_state_dir "service" "$service_name"
+
+######################################################################
+
+nfs_callout_pre ()
+{
+ _event="$1"
+ shift
+
+ nfs_callout "${_event}-pre" "$@"
+}
+
+######################################################################
+
+# script_state_dir set by ctdb_setup_state_dir()
+# shellcheck disable=SC2154
+nfs_callout_init "$script_state_dir"
+
+case "$1" in
+takeip)
+ nfs_callout_pre "$@"
+ ;;
+
+releaseip)
+ nfs_callout_pre "$@"
+ ;;
+esac
diff --git a/ctdb/config/events/legacy/10.interface.script b/ctdb/config/events/legacy/10.interface.script
new file mode 100755
index 0000000..fead88c
--- /dev/null
+++ b/ctdb/config/events/legacy/10.interface.script
@@ -0,0 +1,262 @@
+#!/bin/sh
+
+#################################
+# interface event script for ctdb
+# this adds/removes IPs from your
+# public interface
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+load_script_options
+
+ctdb_public_addresses="${CTDB_BASE}/public_addresses"
+
+if [ ! -f "$ctdb_public_addresses" ]; then
+ if [ "$1" = "init" ] ; then
+ echo "No public addresses file found"
+ fi
+ exit 0
+fi
+
+# This sets $all_interfaces as a side-effect.
+get_all_interfaces ()
+{
+ # Get all the interfaces listed in the public_addresses file
+ all_interfaces=$(sed -e '/^#.*/d' \
+ -e 's/^[^\t ]*[\t ]*//' \
+ -e 's/,/ /g' \
+ -e 's/[\t ]*$//' "$ctdb_public_addresses")
+
+ # Get the interfaces for which CTDB has public IPs configured.
+ # That is, for all but the 1st line, get the 1st field.
+ ctdb_ifaces=$($CTDB -X ifaces | sed -e '1d' -e 's@^|@@' -e 's@|.*@@')
+
+ # Add $ctdb_ifaces and make $all_interfaces unique
+ # Use word splitting to squash whitespace
+ # shellcheck disable=SC2086
+ all_interfaces=$(echo $all_interfaces $ctdb_ifaces | tr ' ' '\n' | sort -u)
+}
+
+monitor_interfaces()
+{
+ get_all_interfaces
+
+ down_interfaces_found=false
+ up_interfaces_found=false
+
+ # Note that this loop must not exit early. It must process
+ # all interfaces so that the correct state for each interface
+ # is set in CTDB using setifacelink.
+ for _iface in $all_interfaces ; do
+ if interface_monitor "$_iface" ; then
+ up_interfaces_found=true
+ $CTDB setifacelink "$_iface" up >/dev/null 2>&1
+ else
+ down_interfaces_found=true
+ $CTDB setifacelink "$_iface" down >/dev/null 2>&1
+ fi
+ done
+
+ if ! $down_interfaces_found ; then
+ return 0
+ fi
+
+ if ! $up_interfaces_found ; then
+ return 1
+ fi
+
+ if [ "$CTDB_PARTIALLY_ONLINE_INTERFACES" != "yes" ]; then
+ return 1
+ fi
+
+ return 0
+}
+
+# Sets: iface, ip, maskbits
+get_iface_ip_maskbits ()
+{
+ _iface_in="$1"
+ ip="$2"
+ _maskbits_in="$3"
+
+ # Intentional word splitting here
+ # shellcheck disable=SC2046
+ set -- $(ip_maskbits_iface "$ip")
+ if [ -n "$1" ] ; then
+ maskbits="$1"
+ iface="$2"
+
+ if [ "$iface" != "$_iface_in" ] ; then
+ printf \
+ 'WARNING: Public IP %s hosted on interface %s but VNN says %s\n' \
+ "$ip" "$iface" "$_iface_in"
+ fi
+ if [ "$maskbits" != "$_maskbits_in" ] ; then
+ printf \
+ 'WARNING: Public IP %s has %s bit netmask but VNN says %s\n' \
+ "$ip" "$maskbits" "$_maskbits_in"
+ fi
+ else
+ die "ERROR: Unable to determine interface for IP ${ip}"
+ fi
+}
+
+ip_block ()
+{
+ _ip="$1"
+ _iface="$2"
+
+ case "$_ip" in
+ *:*) _family="inet6" ;;
+ *) _family="inet" ;;
+ esac
+
+ # Extra delete copes with previously killed script
+ iptables_wrapper "$_family" \
+ -D INPUT -i "$_iface" -d "$_ip" -j DROP 2>/dev/null
+ iptables_wrapper "$_family" \
+ -I INPUT -i "$_iface" -d "$_ip" -j DROP
+}
+
+ip_unblock ()
+{
+ _ip="$1"
+ _iface="$2"
+
+ case "$_ip" in
+ *:*) _family="inet6" ;;
+ *) _family="inet" ;;
+ esac
+
+ iptables_wrapper "$_family" \
+ -D INPUT -i "$_iface" -d "$_ip" -j DROP 2>/dev/null
+}
+
+ctdb_check_args "$@"
+
+case "$1" in
+init)
+ # make sure that we only respond to ARP messages from the NIC where
+ # a particular ip address is associated.
+ get_proc sys/net/ipv4/conf/all/arp_filter >/dev/null 2>&1 && {
+ set_proc sys/net/ipv4/conf/all/arp_filter 1
+ }
+
+ _promote="sys/net/ipv4/conf/all/promote_secondaries"
+ get_proc "$_promote" >/dev/null 2>&1 || \
+ die "Public IPs only supported if promote_secondaries is available"
+
+ # make sure we drop any ips that might still be held if
+ # previous instance of ctdb got killed with -9 or similar
+ drop_all_public_ips
+ ;;
+
+startup)
+ monitor_interfaces
+ ;;
+
+shutdown)
+ drop_all_public_ips
+ ;;
+
+takeip)
+ iface=$2
+ ip=$3
+ maskbits=$4
+
+ add_ip_to_iface "$iface" "$ip" "$maskbits" || {
+ exit 1;
+ }
+
+ # In case a previous "releaseip" for this IP was killed...
+ ip_unblock "$ip" "$iface"
+
+ flush_route_cache
+ ;;
+
+releaseip)
+ # releasing an IP is a bit more complex than it seems. Once the IP
+ # is released, any open tcp connections to that IP on this host will end
+ # up being stuck. Some of them (such as NFS connections) will be unkillable
+ # so we need to use the killtcp ctdb function to kill them off. We also
+ # need to make sure that no new connections get established while we are
+ # doing this! So what we do is this:
+ # 1) firewall this IP, so no new external packets arrive for it
+ # 2) find existing connections, and kill them
+ # 3) remove the IP from the interface
+ # 4) remove the firewall rule
+ shift
+ get_iface_ip_maskbits "$@"
+
+ ip_block "$ip" "$iface"
+
+ kill_tcp_connections "$iface" "$ip"
+
+ delete_ip_from_iface "$iface" "$ip" "$maskbits" || {
+ ip_unblock "$ip" "$iface"
+ exit 1
+ }
+
+ ip_unblock "$ip" "$iface"
+
+ flush_route_cache
+ ;;
+
+updateip)
+ # moving an IP is a bit more complex than it seems.
+ # First we drop all traffic on the old interface.
+ # Then we try to add the ip to the new interface and before
+ # we finally remove it from the old interface.
+ #
+ # 1) firewall this IP, so no new external packets arrive for it
+ # 2) remove the IP from the old interface (and new interface, to be sure)
+ # 3) add the IP to the new interface
+ # 4) remove the firewall rule
+ # 5) use ctdb gratarp to propagate the new mac address
+ # 6) use netstat -tn to find existing connections, and tickle them
+ _oiface=$2
+ niface=$3
+ _ip=$4
+ _maskbits=$5
+
+ get_iface_ip_maskbits "$_oiface" "$_ip" "$_maskbits"
+ oiface="$iface"
+
+ # Could check maskbits too. However, that should never change
+ # so we want to notice if it does.
+ if [ "$oiface" = "$niface" ] ; then
+ echo "Redundant \"updateip\" - ${ip} already on ${niface}"
+ exit 0
+ fi
+
+ ip_block "$ip" "$oiface"
+
+ delete_ip_from_iface "$oiface" "$ip" "$maskbits" 2>/dev/null
+ delete_ip_from_iface "$niface" "$ip" "$maskbits" 2>/dev/null
+
+ add_ip_to_iface "$niface" "$ip" "$maskbits" || {
+ ip_unblock "$ip" "$oiface"
+ exit 1
+ }
+
+ ip_unblock "$ip" "$oiface"
+
+ flush_route_cache
+
+ # propagate the new mac address
+ $CTDB gratarp "$ip" "$niface"
+
+ # tickle all existing connections, so that dropped packets
+ # are retransmitted and the tcp streams work
+ tickle_tcp_connections "$ip"
+ ;;
+
+monitor)
+ monitor_interfaces || exit 1
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/11.natgw.script b/ctdb/config/events/legacy/11.natgw.script
new file mode 100755
index 0000000..fb93dea
--- /dev/null
+++ b/ctdb/config/events/legacy/11.natgw.script
@@ -0,0 +1,242 @@
+#!/bin/sh
+# Script to set up one of the nodes as a NAT gateway for all other nodes.
+# This is used to ensure that all nodes in the cluster can still originate
+# traffic to the external network even if there are no public addresses
+# available.
+#
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+service_name="natgw"
+
+load_script_options
+
+[ -n "$CTDB_NATGW_NODES" ] || exit 0
+export CTDB_NATGW_NODES
+
+ctdb_setup_state_dir "failover" "$service_name"
+
+# script_state_dir set by ctdb_setup_state_dir()
+# shellcheck disable=SC2154
+natgw_cfg_new="${script_state_dir}/cfg_new"
+natgw_cfg_old="${script_state_dir}/cfg_old"
+natgw_leader_old="${script_state_dir}/leader_old"
+
+ctdb_natgw_follower_only ()
+{
+ _ip_address=$(ctdb_get_ip_address)
+
+ awk -v my_ip="$_ip_address" \
+ '$1 == my_ip { if ($2 ~ "follower-only") { exit 0 } else { exit 1 } }' \
+ "$CTDB_NATGW_NODES"
+}
+
+natgw_check_config ()
+{
+ [ -r "$CTDB_NATGW_NODES" ] || \
+ die "error: CTDB_NATGW_NODES=${CTDB_NATGW_NODES} unreadable"
+ if ! ctdb_natgw_follower_only ; then
+ [ -n "$CTDB_NATGW_PUBLIC_IP" ] || \
+ die "Invalid configuration: CTDB_NATGW_PUBLIC_IP not set"
+ [ -n "$CTDB_NATGW_PUBLIC_IFACE" ] || \
+ die "Invalid configuration: CTDB_NATGW_PUBLIC_IFACE not set"
+ fi
+ [ -n "$CTDB_NATGW_PRIVATE_NETWORK" ] || \
+ die "Invalid configuration: CTDB_NATGW_PRIVATE_NETWORK not set"
+
+ # The default is to create a single default route
+ [ -n "$CTDB_NATGW_STATIC_ROUTES" ] || CTDB_NATGW_STATIC_ROUTES="0.0.0.0/0"
+}
+
+natgw_write_config ()
+{
+ _f="$1"
+
+ cat >"$_f" <<EOF
+CTDB_NATGW_NODES="$CTDB_NATGW_NODES"
+CTDB_NATGW_PUBLIC_IP="$CTDB_NATGW_PUBLIC_IP"
+CTDB_NATGW_PUBLIC_IFACE="$CTDB_NATGW_PUBLIC_IFACE"
+CTDB_NATGW_DEFAULT_GATEWAY="$CTDB_NATGW_DEFAULT_GATEWAY"
+CTDB_NATGW_PRIVATE_NETWORK="$CTDB_NATGW_PRIVATE_NETWORK"
+CTDB_NATGW_STATIC_ROUTES="$CTDB_NATGW_STATIC_ROUTES"
+EOF
+}
+
+natgw_config_has_changed ()
+{
+ natgw_write_config "$natgw_cfg_new"
+
+ # Non-existent old returns true, no log message
+ if [ ! -f "$natgw_cfg_old" ] ; then
+ return 0
+ fi
+
+ # Handle no change
+ if cmp "$natgw_cfg_old" "$natgw_cfg_new" >/dev/null 2>&1 ; then
+ return 1
+ fi
+
+ echo "NAT gateway configuration has changed"
+ return 0
+}
+
+_natgw_clear ()
+{
+ _ip="${CTDB_NATGW_PUBLIC_IP%/*}"
+ _maskbits="${CTDB_NATGW_PUBLIC_IP#*/}"
+
+ delete_ip_from_iface \
+ "$CTDB_NATGW_PUBLIC_IFACE" "$_ip" "$_maskbits" >/dev/null 2>&1
+ for _net_gw in $CTDB_NATGW_STATIC_ROUTES ; do
+ _net="${_net_gw%@*}"
+ ip route del "$_net" metric 10 >/dev/null 2>/dev/null
+ done
+
+ # Delete the masquerading setup from a previous iteration where we
+ # were the NAT-GW
+ iptables -D POSTROUTING -t nat \
+ -s "$CTDB_NATGW_PRIVATE_NETWORK" ! -d "$CTDB_NATGW_PRIVATE_NETWORK" \
+ -j MASQUERADE >/dev/null 2>/dev/null
+
+ iptables -D INPUT -p tcp --syn -d "${_ip}/32" -j REJECT 2>/dev/null
+}
+
+natgw_clear ()
+{
+ if [ -r "$natgw_cfg_old" ] ; then
+ (. "$natgw_cfg_old" ; _natgw_clear)
+ else
+ _natgw_clear
+ fi
+}
+
+natgw_set_leader ()
+{
+ set_proc sys/net/ipv4/ip_forward 1
+ iptables -A POSTROUTING -t nat \
+ -s "$CTDB_NATGW_PRIVATE_NETWORK" ! -d "$CTDB_NATGW_PRIVATE_NETWORK" \
+ -j MASQUERADE
+
+ # block all incoming connections to the NATGW IP address
+ ctdb_natgw_public_ip_host="${CTDB_NATGW_PUBLIC_IP%/*}/32"
+ iptables -D INPUT -p tcp --syn \
+ -d "$ctdb_natgw_public_ip_host" -j REJECT 2>/dev/null
+ iptables -I INPUT -p tcp --syn \
+ -d "$ctdb_natgw_public_ip_host" -j REJECT 2>/dev/null
+
+ ip addr add "$CTDB_NATGW_PUBLIC_IP" dev "$CTDB_NATGW_PUBLIC_IFACE"
+ for _net_gw in $CTDB_NATGW_STATIC_ROUTES ; do
+ _net="${_net_gw%@*}"
+ if [ "$_net" != "$_net_gw" ] ; then
+ _gw="${_net_gw#*@}"
+ else
+ _gw="$CTDB_NATGW_DEFAULT_GATEWAY"
+ fi
+
+ [ -n "$_gw" ] || continue
+ ip route add "$_net" metric 10 via "$_gw"
+ done
+}
+
+natgw_set_follower ()
+{
+ _natgwip="$1"
+
+ for _net_gw in $CTDB_NATGW_STATIC_ROUTES ; do
+ _net="${_net_gw%@*}"
+ ip route add "$_net" via "$_natgwip" metric 10
+ done
+}
+
+natgw_ensure_leader ()
+{
+ # Intentional word splitting here
+ # shellcheck disable=SC2046
+ set -- $("${CTDB_HELPER_BINDIR}/ctdb_natgw" leader)
+ natgwleader="${1:--1}" # Default is -1, for failure above
+ natgwip="$2"
+
+ if [ "$natgwleader" = "-1" ]; then
+ # Fail...
+ die "There is no NATGW leader node"
+ fi
+}
+
+natgw_leader_has_changed ()
+{
+ if [ -r "$natgw_leader_old" ] ; then
+ read _old_natgwleader <"$natgw_leader_old"
+ else
+ _old_natgwleader=""
+ fi
+ [ "$_old_natgwleader" != "$natgwleader" ]
+}
+
+natgw_save_state ()
+{
+ echo "$natgwleader" >"$natgw_leader_old"
+ # Created by natgw_config_has_changed()
+ mv "$natgw_cfg_new" "$natgw_cfg_old"
+}
+
+
+case "$1" in
+setup)
+ natgw_check_config
+ ;;
+
+startup)
+ natgw_check_config
+
+ # Error if CTDB_NATGW_PUBLIC_IP is listed in public addresses
+ ip_pat=$(echo "$CTDB_NATGW_PUBLIC_IP" | sed -e 's@\.@\\.@g')
+ ctdb_public_addresses="${CTDB_BASE}/public_addresses"
+ if grep -q "^${ip_pat}[[:space:]]" "$ctdb_public_addresses" ; then
+ die "ERROR: CTDB_NATGW_PUBLIC_IP same as a public address"
+ fi
+
+ # do not send out arp requests from loopback addresses
+ set_proc sys/net/ipv4/conf/all/arp_announce 2
+ ;;
+
+updatenatgw|ipreallocated)
+ natgw_check_config
+
+ natgw_ensure_leader
+
+ natgw_config_has_changed || natgw_leader_has_changed || exit 0
+
+ natgw_clear
+
+ pnn=$(ctdb_get_pnn)
+ if [ "$pnn" = "$natgwleader" ]; then
+ natgw_set_leader
+ else
+ natgw_set_follower "$natgwip"
+ fi
+
+ # flush our route cache
+ set_proc sys/net/ipv4/route/flush 1
+
+ # Only update saved state when NATGW successfully updated
+ natgw_save_state
+ ;;
+
+shutdown|removenatgw)
+ natgw_check_config
+ natgw_clear
+ ;;
+
+monitor)
+ natgw_check_config
+
+ if [ -n "$CTDB_NATGW_PUBLIC_IFACE" ] ; then
+ interface_monitor "$CTDB_NATGW_PUBLIC_IFACE" || exit 1
+ fi
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/11.routing.script b/ctdb/config/events/legacy/11.routing.script
new file mode 100755
index 0000000..7ba7f3b
--- /dev/null
+++ b/ctdb/config/events/legacy/11.routing.script
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Attempt to add a set of static routes.
+#
+# Do this in "ipreallocated" rather than just "startup" because some
+# of the routes might be missing because the corresponding interface
+# has not previously had any IPs assigned or IPs were previously
+# released and corresponding routes were dropped.
+#
+# Addition of some routes might fail, errors go to /dev/null.
+#
+# Routes to add are defined in $CTDB_BASE/static-routes. Syntax is:
+#
+# IFACE NET/MASK GATEWAY
+#
+# Example:
+#
+# bond1 10.3.3.0/24 10.0.0.1
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+load_script_options
+
+[ -f "${CTDB_BASE}/static-routes" ] || {
+ exit 0
+}
+
+case "$1" in
+ipreallocated)
+ while read iface dest gw; do
+ ip route add "$dest" via "$gw" dev "$iface" >/dev/null 2>&1
+ done <"${CTDB_BASE}/static-routes"
+ ;;
+
+updateip)
+ oiface=$2
+ niface=$3
+ while read iface dest gw; do
+ if [ "$niface" = "$iface" ] || [ "$oiface" = "$iface" ] ; then
+ ip route add "$dest" via "$gw" dev "$iface" >/dev/null 2>&1
+ fi
+ done <"${CTDB_BASE}/static-routes"
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/13.per_ip_routing.script b/ctdb/config/events/legacy/13.per_ip_routing.script
new file mode 100755
index 0000000..d7949c6
--- /dev/null
+++ b/ctdb/config/events/legacy/13.per_ip_routing.script
@@ -0,0 +1,438 @@
+#!/bin/sh
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+load_script_options
+
+service_name="per_ip_routing"
+
+# Do nothing if unconfigured
+[ -n "$CTDB_PER_IP_ROUTING_CONF" ] || exit 0
+
+table_id_prefix="ctdb."
+
+[ -n "$CTDB_PER_IP_ROUTING_RULE_PREF" ] || \
+ die "error: CTDB_PER_IP_ROUTING_RULE_PREF not configured"
+
+[ "$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" -lt "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] 2>/dev/null || \
+ die "error: CTDB_PER_IP_ROUTING_TABLE_ID_LOW[$CTDB_PER_IP_ROUTING_TABLE_ID_LOW] and/or CTDB_PER_IP_ROUTING_TABLE_ID_HIGH[$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH] improperly configured"
+
+if [ "$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" -le 253 ] && \
+ [ 255 -le "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] ; then
+ die "error: range CTDB_PER_IP_ROUTING_TABLE_ID_LOW[$CTDB_PER_IP_ROUTING_TABLE_ID_LOW]..CTDB_PER_IP_ROUTING_TABLE_ID_HIGH[$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH] must not include 253-255"
+fi
+
+have_link_local_config ()
+{
+ [ "$CTDB_PER_IP_ROUTING_CONF" = "__auto_link_local__" ]
+}
+
+if ! have_link_local_config && [ ! -r "$CTDB_PER_IP_ROUTING_CONF" ] ; then
+ die "error: CTDB_PER_IP_ROUTING_CONF=$CTDB_PER_IP_ROUTING_CONF file not found"
+fi
+
+ctdb_setup_state_dir "failover" "$service_name"
+
+######################################################################
+
+ipv4_is_valid_addr()
+{
+ _ip="$1"
+
+ _count=0
+ # Get the shell to break up the address into 1 word per octet
+ # Intentional word splitting here
+ # shellcheck disable=SC2086
+ for _o in $(export IFS="." ; echo $_ip) ; do
+ # The 2>/dev/null stops output from failures where an "octet"
+ # is not numeric. The test will still fail.
+ if ! [ 0 -le $_o ] && [ $_o -le 255 ] 2>/dev/null ; then
+ return 1
+ fi
+ _count=$((_count + 1))
+ done
+
+ # A valid IPv4 address has 4 octets
+ [ $_count -eq 4 ]
+}
+
+ensure_ipv4_is_valid_addr ()
+{
+ _event="$1"
+ _ip="$2"
+
+ ipv4_is_valid_addr "$_ip" || {
+ echo "$0: $_event not an ipv4 address skipping IP:$_ip"
+ exit 0
+ }
+}
+
+ipv4_host_addr_to_net ()
+{
+ _host="$1"
+ _maskbits="$2"
+
+ # Convert the host address to an unsigned long by splitting out
+ # the octets and doing the math.
+ _host_ul=0
+ # Intentional word splitting here
+ # shellcheck disable=SC2086
+ for _o in $(export IFS="." ; echo $_host) ; do
+ _host_ul=$(( (_host_ul << 8) + _o)) # work around Emacs color bug
+ done
+
+ # Calculate the mask and apply it.
+ _mask_ul=$(( 0xffffffff << (32 - _maskbits) ))
+ _net_ul=$(( _host_ul & _mask_ul ))
+
+ # Now convert to a network address one byte at a time.
+ _net=""
+ for _o in $(seq 1 4) ; do
+ _net="$((_net_ul & 255))${_net:+.}${_net}"
+ _net_ul=$((_net_ul >> 8))
+ done
+
+ echo "${_net}/${_maskbits}"
+}
+
+######################################################################
+
+ensure_rt_tables ()
+{
+ rt_tables="$CTDB_SYS_ETCDIR/iproute2/rt_tables"
+ # script_state_dir set by ctdb_setup_state_dir()
+ # shellcheck disable=SC2154
+ rt_tables_lock="${script_state_dir}/rt_tables_lock"
+
+ # This file should always exist. Even if this didn't exist on the
+ # system, adding a route will have created it. What if we startup
+ # and immediately shutdown? Let's be sure.
+ if [ ! -f "$rt_tables" ] ; then
+ mkdir -p "${rt_tables%/*}" # dirname
+ touch "$rt_tables"
+ fi
+}
+
+# Setup a table id to use for the given IP. We don't need to know it,
+# it just needs to exist in /etc/iproute2/rt_tables. Fail if no free
+# table id could be found in the configured range.
+ensure_table_id_for_ip ()
+{
+ _ip=$1
+
+ ensure_rt_tables
+
+ # Maintain a table id for each IP address we've ever seen in
+ # rt_tables. We use a "ctdb." prefix on the label.
+ _label="${table_id_prefix}${_ip}"
+
+ # This finds either the table id corresponding to the label or a
+ # new unused one (that is greater than all the used ones in the
+ # range).
+ (
+ # Note that die() just gets us out of the subshell...
+ flock --timeout 30 9 || \
+ die "ensure_table_id_for_ip: failed to lock file $rt_tables"
+
+ _new="$CTDB_PER_IP_ROUTING_TABLE_ID_LOW"
+ while read _t _l ; do
+ # Skip comments
+ case "$_t" in
+ \#*) continue ;;
+ esac
+ # Found existing: done
+ if [ "$_l" = "$_label" ] ; then
+ return 0
+ fi
+ # Potentially update the new table id to be used. The
+ # redirect stops error spam for a non-numeric value.
+ if [ "$_new" -le "$_t" ] && \
+ [ "$_t" -le "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] \
+ 2>/dev/null ; then
+ _new=$((_t + 1))
+ fi
+ done <"$rt_tables"
+
+ # If the new table id is legal then add it to the file and
+ # print it.
+ if [ "$_new" -le "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] ; then
+ printf '%d\t%s\n' "$_new" "$_label" >>"$rt_tables"
+ return 0
+ else
+ return 1
+ fi
+ ) 9>"$rt_tables_lock"
+}
+
+# Clean up all the table ids that we might own.
+clean_up_table_ids ()
+{
+ ensure_rt_tables
+
+ (
+ # Note that die() just gets us out of the subshell...
+ flock --timeout 30 9 || \
+ die "clean_up_table_ids: failed to lock file $rt_tables"
+
+ # Delete any items from the file that have a table id in our
+ # range or a label matching our label. Preserve comments.
+ _tmp="${rt_tables}.$$.ctdb"
+ awk -v min="$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" \
+ -v max="$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" \
+ -v pre="$table_id_prefix" \
+ '/^#/ ||
+ !(min <= $1 && $1 <= max) &&
+ !(index($2, pre) == 1) {
+ print $0 }' "$rt_tables" >"$_tmp"
+
+ mv "$_tmp" "$rt_tables"
+ ) 9>"$rt_tables_lock"
+}
+
+######################################################################
+
+# This prints the config for an IP, which is either relevant entries
+# from the config file or, if set to the magic link local value, some
+# link local routing config for the IP.
+get_config_for_ip ()
+{
+ _ip="$1"
+
+ if have_link_local_config ; then
+ # When parsing public_addresses also split on '/'. This means
+ # that we get the maskbits as item #2 without further parsing.
+ while IFS="/$IFS" read _i _maskbits _x ; do
+ if [ "$_ip" = "$_i" ] ; then
+ printf "%s" "$_ip "; ipv4_host_addr_to_net "$_ip" "$_maskbits"
+ fi
+ done <"${CTDB_BASE}/public_addresses"
+ else
+ while read _i _rest ; do
+ if [ "$_ip" = "$_i" ] ; then
+ printf '%s\t%s\n' "$_ip" "$_rest"
+ fi
+ done <"$CTDB_PER_IP_ROUTING_CONF"
+ fi
+}
+
+ip_has_configuration ()
+{
+ _ip="$1"
+
+ _conf=$(get_config_for_ip "$_ip")
+ [ -n "$_conf" ]
+}
+
+add_routing_for_ip ()
+{
+ _iface="$1"
+ _ip="$2"
+
+ # Do nothing if no config for this IP.
+ ip_has_configuration "$_ip" || return 0
+
+ ensure_table_id_for_ip "$_ip" || \
+ die "add_routing_for_ip: out of table ids in range $CTDB_PER_IP_ROUTING_TABLE_ID_LOW - $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH"
+
+ _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
+ _table_id="${table_id_prefix}${_ip}"
+
+ del_routing_for_ip "$_ip" >/dev/null 2>&1
+
+ ip rule add from "$_ip" pref "$_pref" table "$_table_id" || \
+ die "add_routing_for_ip: failed to add rule for $_ip"
+
+ # Add routes to table for any lines matching the IP.
+ get_config_for_ip "$_ip" |
+ while read _i _dest _gw ; do
+ _r="$_dest ${_gw:+via} $_gw dev $_iface table $_table_id"
+ # Intentionally unquoted multi-word value here
+ # shellcheck disable=SC2086
+ ip route add $_r || \
+ die "add_routing_for_ip: failed to add route: $_r"
+ done
+}
+
+del_routing_for_ip ()
+{
+ _ip="$1"
+
+ _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
+ _table_id="${table_id_prefix}${_ip}"
+
+ # Do this unconditionally since we own any matching table ids.
+ # However, print a meaningful message if something goes wrong.
+ _cmd="ip rule del from $_ip pref $_pref table $_table_id"
+ _out=$($_cmd 2>&1) || \
+ cat <<EOF
+WARNING: Failed to delete policy routing rule
+ Command "$_cmd" failed:
+ $_out
+EOF
+ # This should never usually fail, so don't redirect output.
+ # However, it can fail when deleting a rogue IP, since there will
+ # be no routes for that IP. In this case it should only fail when
+ # the rule deletion above has already failed because the table id
+ # is invalid. Therefore, go to a little bit of trouble to indent
+ # the failure message so that it is associated with the above
+ # warning message and doesn't look too nasty.
+ ip route flush table "$_table_id" 2>&1 | sed -e 's@^.@ &@'
+}
+
+######################################################################
+
+flush_rules_and_routes ()
+{
+ ip rule show |
+ while read _p _x _i _x _t ; do
+ # Remove trailing colon after priority/preference.
+ _p="${_p%:}"
+ # Only remove rules that match our priority/preference.
+ [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
+
+ echo "Removing ip rule for public address $_i for routing table $_t"
+ ip rule del from "$_i" table "$_t" pref "$_p"
+ ip route flush table "$_t" 2>/dev/null
+ done
+}
+
+# Add any missing routes. Some might have gone missing if, for
+# example, all IPs on the network were removed (possibly if the
+# primary was removed). If $1 is "force" then (re-)add all the
+# routes.
+add_missing_routes ()
+{
+ $CTDB ip -v -X | {
+ read _x # skip header line
+
+ # Read the rest of the lines. We're only interested in the
+ # "IP" and "ActiveInterface" columns. The latter is only set
+ # for addresses local to this node, making it easy to skip
+ # non-local addresses. For each IP local address we check if
+ # the relevant routing table is populated and populate it if
+ # not.
+ while IFS="|" read _x _ip _x _iface _x ; do
+ [ -n "$_iface" ] || continue
+
+ _table_id="${table_id_prefix}${_ip}"
+ if [ -z "$(ip route show table "$_table_id" 2>/dev/null)" ] || \
+ [ "$1" = "force" ] ; then
+ add_routing_for_ip "$_iface" "$_ip"
+ fi
+ done
+ } || exit $?
+}
+
+# Remove rules/routes for addresses that we're not hosting. If a
+# releaseip event failed in an earlier script then we might not have
+# had a chance to remove the corresponding rules/routes.
+remove_bogus_routes ()
+{
+ # Get a IPs current hosted by this node, each anchored with '@'.
+ _ips=$($CTDB ip -v -X | awk -F'|' 'NR > 1 && $4 != "" {printf "@%s@\n", $2}')
+
+ # x is intentionally ignored
+ # shellcheck disable=SC2034
+ ip rule show |
+ while read _p _x _i _x _t ; do
+ # Remove trailing colon after priority/preference.
+ _p="${_p%:}"
+ # Only remove rules that match our priority/preference.
+ [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
+ # Only remove rules for which we don't have an IP. This could
+ # be done with grep, but let's do it with shell prefix removal
+ # to avoid unnecessary processes. This falls through if
+ # "@${_i}@" isn't present in $_ips.
+ [ "$_ips" = "${_ips#*@"${_i}"@}" ] || continue
+
+ echo "Removing ip rule/routes for unhosted public address $_i"
+ del_routing_for_ip "$_i"
+ done
+}
+
+######################################################################
+
+ctdb_check_args "$@"
+
+case "$1" in
+startup)
+ flush_rules_and_routes
+
+ # make sure that we only respond to ARP messages from the NIC
+ # where a particular ip address is associated.
+ get_proc sys/net/ipv4/conf/all/arp_filter >/dev/null 2>&1 && {
+ set_proc sys/net/ipv4/conf/all/arp_filter 1
+ }
+ ;;
+
+shutdown)
+ flush_rules_and_routes
+ clean_up_table_ids
+ ;;
+
+takeip)
+ iface=$2
+ ip=$3
+ # maskbits included here so argument order is obvious
+ # shellcheck disable=SC2034
+ maskbits=$4
+
+ ensure_ipv4_is_valid_addr "$1" "$ip"
+ add_routing_for_ip "$iface" "$ip"
+
+ # flush our route cache
+ set_proc sys/net/ipv4/route/flush 1
+
+ $CTDB gratarp "$ip" "$iface"
+ ;;
+
+updateip)
+ # oiface, maskbits included here so argument order is obvious
+ # shellcheck disable=SC2034
+ oiface=$2
+ niface=$3
+ ip=$4
+ # shellcheck disable=SC2034
+ maskbits=$5
+
+ ensure_ipv4_is_valid_addr "$1" "$ip"
+ add_routing_for_ip "$niface" "$ip"
+
+ # flush our route cache
+ set_proc sys/net/ipv4/route/flush 1
+
+ $CTDB gratarp "$ip" "$niface"
+ tickle_tcp_connections "$ip"
+ ;;
+
+releaseip)
+ iface=$2
+ ip=$3
+ # maskbits included here so argument order is obvious
+ # shellcheck disable=SC2034
+ maskbits=$4
+
+ ensure_ipv4_is_valid_addr "$1" "$ip"
+ del_routing_for_ip "$ip"
+ ;;
+
+ipreallocated)
+ add_missing_routes
+ remove_bogus_routes
+ ;;
+
+reconfigure)
+ echo "Reconfiguring service \"${service_name}\"..."
+
+ add_missing_routes "force"
+ remove_bogus_routes
+
+ # flush our route cache
+ set_proc sys/net/ipv4/route/flush 1
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/20.multipathd.script b/ctdb/config/events/legacy/20.multipathd.script
new file mode 100755
index 0000000..a420251
--- /dev/null
+++ b/ctdb/config/events/legacy/20.multipathd.script
@@ -0,0 +1,83 @@
+#!/bin/sh
+# ctdb event script for monitoring the multipath daemon
+#
+# Configure monitporing of multipath devices by listing the device serials
+# in /etc/ctdb/multipathd :
+# CTDB_MONITOR_MPDEVICES="device1 device2 ..."
+#
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+service_name="multipathd"
+
+load_script_options
+
+[ -n "$CTDB_MONITOR_MPDEVICES" ] || exit 0
+
+ctdb_setup_state_dir "service" "$service_name"
+
+# script_state_dir set by ctdb_setup_state_dir()
+# shellcheck disable=SC2154
+multipath_fail="${script_state_dir}/fail"
+
+multipathd_check_background()
+{
+ for _device in $CTDB_MONITOR_MPDEVICES; do
+ # Check multipath knows about the device
+ _out=$(multipath -ll "$_device")
+ if [ -z "$_out" ] ; then
+ echo "ERROR: device \"${_device}\" not known to multipathd" \
+ >"$multipath_fail"
+ exit 1
+ fi
+
+ # Check for at least 1 active path
+ if ! echo "$_out" | grep 'prio=.* status=active' >/dev/null 2>&1 ; then
+ echo "ERROR: multipath device \"${_device}\" has no active paths" \
+ >"$multipath_fail"
+ exit 1
+ fi
+ done
+ exit 0
+}
+
+multipathd_check()
+{
+ # Run the actual check in the background since the call to
+ # multipath may block
+ multipathd_check_background </dev/null >/dev/null 2>&1 &
+ _pid="$!"
+ _timeleft=10
+
+ while [ $_timeleft -gt 0 ]; do
+ _timeleft=$((_timeleft - 1))
+
+ # see if the process still exists
+ kill -0 $_pid >/dev/null 2>&1 || {
+ if wait $_pid ; then
+ return 0
+ else
+ cat "$multipath_fail"
+ rm -f "$multipath_fail"
+ return 1
+ fi
+ }
+ sleep 1
+ done
+
+ echo "ERROR: callout to multipath checks hung"
+ # If hung then this probably won't work, but worth trying...
+ kill -9 $_pid >/dev/null 2>&1
+ return 1
+}
+
+case "$1" in
+monitor)
+ multipathd_check || die "multipath monitoring failed"
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/31.clamd.script b/ctdb/config/events/legacy/31.clamd.script
new file mode 100755
index 0000000..5d60fe3
--- /dev/null
+++ b/ctdb/config/events/legacy/31.clamd.script
@@ -0,0 +1,37 @@
+#!/bin/sh
+# event script to manage clamd in a cluster environment
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+detect_init_style
+
+case $CTDB_INIT_STYLE in
+redhat)
+ service_name="clamd"
+ ;;
+*)
+ service_name="clamav"
+ ;;
+esac
+
+load_script_options
+
+case "$1" in
+startup)
+ service "$service_name" stop > /dev/null 2>&1
+ service "$service_name" start || exit $?
+ ;;
+
+shutdown)
+ service "$service_name"_stop
+ ;;
+
+monitor)
+ ctdb_check_unix_socket "$CTDB_CLAMD_SOCKET" || exit $?
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/40.vsftpd.script b/ctdb/config/events/legacy/40.vsftpd.script
new file mode 100755
index 0000000..2d2aac4
--- /dev/null
+++ b/ctdb/config/events/legacy/40.vsftpd.script
@@ -0,0 +1,57 @@
+#!/bin/sh
+# event strict to manage vsftpd in a cluster environment
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+service_name="vsftpd"
+
+service_reconfigure ()
+{
+ # shellcheck disable=SC2317
+ # Called indirectly via ctdb_service_reconfigure()
+ service "$service_name" restart
+}
+
+load_script_options
+
+ctdb_setup_state_dir "service" "$service_name"
+
+port_21="vsftpd listening on TCP port 21"
+
+case "$1" in
+startup)
+ service "$service_name" stop > /dev/null 2>&1
+ service "$service_name" start
+ failcount_init "$port_21"
+ ;;
+
+shutdown)
+ service "$service_name" stop
+ ;;
+
+takeip|releaseip)
+ ctdb_service_set_reconfigure
+ ;;
+
+ipreallocated)
+ if ctdb_service_needs_reconfigure ; then
+ ctdb_service_reconfigure
+ fi
+ ;;
+
+monitor)
+ if ctdb_check_tcp_ports 21 ; then
+ failcount_reset "$port_21"
+ else
+ # Set defaults, if unset
+ : "${CTDB_VSFTPD_MONITOR_THRESHOLDS:=1:2}"
+
+ failcount_incr "$port_21" "$CTDB_VSFTPD_MONITOR_THRESHOLDS"
+ fi
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/41.httpd.script b/ctdb/config/events/legacy/41.httpd.script
new file mode 100755
index 0000000..dd90aed
--- /dev/null
+++ b/ctdb/config/events/legacy/41.httpd.script
@@ -0,0 +1,78 @@
+#!/bin/sh
+# event script to manage httpd in a cluster environment
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+detect_init_style
+
+case $CTDB_INIT_STYLE in
+redhat)
+ service_name="httpd"
+ ;;
+suse|debian|*)
+ service_name="apache2"
+ ;;
+esac
+
+load_script_options
+
+ctdb_setup_state_dir "service" "$service_name"
+
+# RHEL5 sometimes use a SIGKILL to terminate httpd, which then leaks
+# semaphores. This is a hack to clean them up.
+cleanup_httpd_semaphore_leak() {
+ killall -q -0 "$service_name" ||
+ for i in $(ipcs -s | awk '$3 == "apache" { print $2 }') ; do
+ ipcrm -s "$i"
+ done
+}
+
+##########
+
+service_start ()
+{
+ cleanup_httpd_semaphore_leak
+ service $service_name start
+}
+service_stop ()
+{
+ service $service_name stop
+ killall -q -9 $service_name || true
+}
+
+case "$1" in
+startup)
+ service_start
+ ctdb_counter_init
+ ;;
+
+shutdown)
+ service_stop
+ ;;
+
+monitor)
+ if ctdb_check_tcp_ports 80 >/dev/null 2>/dev/null ; then
+ ctdb_counter_init
+ else
+ ctdb_counter_incr
+ num_fails=$(ctdb_counter_get)
+ if [ "$num_fails" -eq 2 ] ; then
+ echo "HTTPD is not running. Trying to restart HTTPD."
+ service_stop
+ service_start
+ exit 0
+ elif [ "$num_fails" -ge 5 ] ; then
+ echo "HTTPD is not running. Trying to restart HTTPD."
+ service_stop
+ service_start
+ exit 1
+ fi
+ fi
+ ;;
+esac
+
+exit 0
+
diff --git a/ctdb/config/events/legacy/47.samba-dcerpcd.script b/ctdb/config/events/legacy/47.samba-dcerpcd.script
new file mode 100755
index 0000000..9492d55
--- /dev/null
+++ b/ctdb/config/events/legacy/47.samba-dcerpcd.script
@@ -0,0 +1,66 @@
+#!/bin/sh
+# ctdb event script for SAMBA DCERPCD Services
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+detect_init_style
+
+case $CTDB_INIT_STYLE in
+ *)
+ # distributions don't have this yet,
+ # but assume samba-dcerpcd as service name
+ CTDB_SERVICE_SAMBA_DCERPCD=${CTDB_SERVICE_SAMBA_DCERPCD:-samba-dcerpcd}
+ ;;
+esac
+
+load_script_options
+
+service_start ()
+{
+ # make sure samba-dcerpcd is not already started
+ service "$CTDB_SERVICE_SAMBA_DCERPCD" stop > /dev/null 2>&1
+ killall -0 -q samba-dcerpcd && {
+ sleep 1
+ # make absolutely sure samba-dcerpcd is dead
+ killall -q -9 samba-dcerpcd
+ }
+
+ # start Samba dcerpcd service. Start it reniced, as under very heavy load
+ # the number of smbd processes will mean that it leaves few cycles
+ # for anything else
+ nice_service "$CTDB_SERVICE_SAMBA_DCERPCD" start || die "Failed to start samba-dcerpcd"
+}
+
+service_stop ()
+{
+ service "$CTDB_SERVICE_SAMBA_DCERPCD" stop
+}
+
+service_status ()
+{
+ service "$CTDB_SERVICE_SAMBA_DCERPCD" status > /dev/null
+ test $? = 0 && return 0
+ service "$CTDB_SERVICE_SAMBA_DCERPCD" status
+}
+
+###########################
+
+case "$1" in
+startup)
+ service_start
+ ;;
+
+shutdown)
+ service_stop
+ ;;
+
+monitor)
+ service_status
+ ;;
+
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/48.netbios.script b/ctdb/config/events/legacy/48.netbios.script
new file mode 100755
index 0000000..1531e49
--- /dev/null
+++ b/ctdb/config/events/legacy/48.netbios.script
@@ -0,0 +1,75 @@
+#!/bin/sh
+# ctdb event script for Netbios Name Services
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+detect_init_style
+
+case $CTDB_INIT_STYLE in
+ suse)
+ CTDB_SERVICE_NMB=${CTDB_SERVICE_NMB:-nmb}
+ ;;
+ debian)
+ CTDB_SERVICE_NMB=${CTDB_SERVICE_NMB:-nmbd}
+ ;;
+ *)
+ # Use redhat style as default:
+ CTDB_SERVICE_NMB=${CTDB_SERVICE_NMB:-nmb}
+ ;;
+esac
+
+service_name="netbios"
+
+load_script_options
+
+ctdb_setup_state_dir "service" "$service_name"
+
+service_start ()
+{
+ # make sure nmbd is not already started
+ service "$CTDB_SERVICE_NMB" stop > /dev/null 2>&1
+ killall -0 -q nmbd && {
+ sleep 1
+ # make absolutely sure nmbd is dead
+ killall -q -9 nmbd
+ }
+
+ # start Samba nmbd service. Start it reniced, as under very heavy load
+ # the number of smbd processes will mean that it leaves few cycles
+ # for anything else
+ nice_service "$CTDB_SERVICE_NMB" start || die "Failed to start nmbd"
+}
+
+service_stop ()
+{
+ service "$CTDB_SERVICE_NMB" stop
+}
+
+service_status ()
+{
+ service "$CTDB_SERVICE_NMB" status > /dev/null
+ test $? = 0 && return 0
+ service "$CTDB_SERVICE_NMB" status
+}
+
+###########################
+
+case "$1" in
+startup)
+ service_start
+ ;;
+
+shutdown)
+ service_stop
+ ;;
+
+monitor)
+ service_status
+ ;;
+
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/49.winbind.script b/ctdb/config/events/legacy/49.winbind.script
new file mode 100755
index 0000000..852b541
--- /dev/null
+++ b/ctdb/config/events/legacy/49.winbind.script
@@ -0,0 +1,55 @@
+#!/bin/sh
+# ctdb event script for winbind
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+CTDB_SERVICE_WINBIND=${CTDB_SERVICE_WINBIND:-winbind}
+
+# service_name is used by various functions
+# shellcheck disable=SC2034
+service_name="winbind"
+
+load_script_options
+
+service_start ()
+{
+ service "$CTDB_SERVICE_WINBIND" stop >/dev/null 2>&1
+ killall -0 -q winbindd && {
+ sleep 1
+ # make absolutely sure winbindd is dead
+ killall -q -9 winbindd
+ }
+
+ service "$CTDB_SERVICE_WINBIND" start || \
+ die "Failed to start winbind"
+}
+
+service_stop ()
+{
+ service "$CTDB_SERVICE_WINBIND" stop
+}
+
+###########################
+
+case "$1" in
+startup)
+ service_start
+ ;;
+
+shutdown)
+ service_stop
+ ;;
+
+monitor)
+ if ! out=$(wbinfo -p 2>&1) ; then
+ echo "ERROR: wbinfo -p returned error"
+ echo "$out"
+ exit 1
+ fi
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/50.samba.script b/ctdb/config/events/legacy/50.samba.script
new file mode 100755
index 0000000..84600e2
--- /dev/null
+++ b/ctdb/config/events/legacy/50.samba.script
@@ -0,0 +1,166 @@
+#!/bin/sh
+# ctdb event script for Samba
+
+[ -n "$CTDB_BASE" ] ||
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+detect_init_style
+
+case $CTDB_INIT_STYLE in
+suse)
+ CTDB_SERVICE_SMB=${CTDB_SERVICE_SMB:-smb}
+ ;;
+debian)
+ CTDB_SERVICE_SMB=${CTDB_SERVICE_SMB:-smbd}
+ ;;
+*)
+ # Use redhat style as default:
+ CTDB_SERVICE_SMB=${CTDB_SERVICE_SMB:-smb}
+ ;;
+esac
+
+service_name="samba"
+
+load_script_options
+
+ctdb_setup_state_dir "service" "$service_name"
+
+service_start()
+{
+ # make sure samba is not already started
+ service "$CTDB_SERVICE_SMB" stop >/dev/null 2>&1
+ killall -0 -q smbd && {
+ sleep 1
+ # make absolutely sure samba is dead
+ killall -q -9 smbd
+ }
+ # start Samba service. Start it reniced, as under very heavy load
+ # the number of smbd processes will mean that it leaves few cycles
+ # for anything else
+ nice_service "$CTDB_SERVICE_SMB" start || die "Failed to start samba"
+}
+
+service_stop()
+{
+ service "$CTDB_SERVICE_SMB" stop
+ program_stack_traces "smbd" 5
+}
+
+######################################################################
+# Show the testparm output using a cached smb.conf to avoid delays due
+# to registry access.
+
+# script_state_dir set by ctdb_setup_state_dir()
+# shellcheck disable=SC2154
+smbconf_cache="$script_state_dir/smb.conf.cache"
+
+testparm_foreground_update()
+{
+ _timeout="$1"
+
+ # No need to remove these temporary files, since there are only 2
+ # of them.
+ _out="${smbconf_cache}.out"
+ _err="${smbconf_cache}.err"
+
+ timeout "$_timeout" testparm -v -s >"$_out" 2>"$_err"
+ case $? in
+ 0) : ;;
+ 124)
+ if [ -f "$smbconf_cache" ]; then
+ echo "WARNING: smb.conf cache update timed out - using old cache file"
+ return 1
+ else
+ echo "ERROR: smb.conf cache create failed - testparm command timed out"
+ exit 1
+ fi
+ ;;
+ *)
+ if [ -f "$smbconf_cache" ]; then
+ echo "WARNING: smb.conf cache update failed - using old cache file"
+ cat "$_err"
+ return 1
+ else
+ echo "ERROR: smb.conf cache create failed - testparm failed with:"
+ cat "$_err"
+ exit 1
+ fi
+ ;;
+ esac
+
+ # Only using $$ here to avoid a collision. This is written into
+ # CTDB's own state directory so there is no real need for a secure
+ # temporary file.
+ _tmpfile="${smbconf_cache}.$$"
+ # Patterns to exclude...
+ _pat='^[[:space:]]+(registry[[:space:]]+shares|include|copy|winbind[[:space:]]+separator)[[:space:]]+='
+ grep -Ev "$_pat" <"$_out" >"$_tmpfile"
+ mv "$_tmpfile" "$smbconf_cache" # atomic
+
+ return 0
+}
+
+testparm_background_update()
+{
+ _timeout="$1"
+
+ testparm_foreground_update "$_timeout" >/dev/null 2>&1 </dev/null &
+}
+
+testparm_get ()
+{
+ _param="$1"
+
+ sed -n \
+ -e "s|^[[:space:]]*${_param}[[:space:]]*=[[:space:]]\(..*\)|\1|p" \
+ "$smbconf_cache"
+
+}
+
+list_samba_shares()
+{
+ testparm_get "path" | sed -e 's/"//g'
+}
+
+list_samba_ports()
+{
+ testparm_get "smb ports"
+}
+
+###########################
+
+case "$1" in
+startup)
+ service_start
+ ;;
+
+shutdown)
+ service_stop
+ ;;
+
+monitor)
+ testparm_foreground_update 10
+ ret=$?
+
+ smb_ports="$CTDB_SAMBA_CHECK_PORTS"
+ if [ -z "$smb_ports" ]; then
+ smb_ports=$(list_samba_ports)
+ [ -n "$smb_ports" ] || die "Failed to set smb ports"
+ fi
+ # Intentionally unquoted multi-word value here
+ # shellcheck disable=SC2086
+ ctdb_check_tcp_ports $smb_ports || exit $?
+
+ if [ "$CTDB_SAMBA_SKIP_SHARE_CHECK" != "yes" ]; then
+ list_samba_shares | ctdb_check_directories || exit $?
+ fi
+
+ if [ $ret -ne 0 ]; then
+ testparm_background_update 10
+ fi
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/60.nfs.script b/ctdb/config/events/legacy/60.nfs.script
new file mode 100755
index 0000000..b7ae074
--- /dev/null
+++ b/ctdb/config/events/legacy/60.nfs.script
@@ -0,0 +1,301 @@
+#!/bin/sh
+# script to manage nfs in a clustered environment
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+service_name="nfs"
+
+load_system_config "nfs"
+
+load_script_options
+
+ctdb_setup_state_dir "service" "$service_name"
+
+######################################################################
+
+service_reconfigure ()
+{
+ # Restart lock manager, notify clients
+ # shellcheck disable=SC2317
+ # Called indirectly via check_thresholds()
+ if [ -x "${CTDB_BASE}/statd-callout" ] ; then
+ "${CTDB_BASE}/statd-callout" notify &
+ fi >/dev/null 2>&1
+}
+
+######################################################################
+
+######################################################
+# Check the health of NFS services
+#
+# Use .check files in $CTDB_NFS_CHECKS_DIR.
+# Default is "${CTDB_BASE}/nfs-checks.d/"
+######################################################
+nfs_check_services ()
+{
+ _dir="${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}"
+
+ # Files must end with .check - avoids editor backups, RPM fu, ...
+ for _f in "$_dir"/[0-9][0-9].*.check ; do
+ [ -r "$_f" ] || continue
+
+ _t="${_f%.check}"
+ _progname="${_t##*/[0-9][0-9].}"
+
+ nfs_check_service "$_progname" <"$_f"
+ done
+}
+
+######################################################
+# Check the health of an NFS service
+#
+# $1 - progname, passed to rpcinfo (looked up in /etc/rpc)
+#
+# Reads variables from stdin
+#
+# Variables are:
+#
+# * family - "tcp" or "udp" or space separated list
+# default: tcp, not used with "service_check_cmd"
+# * version - optional, RPC service version number
+# default is to omit to check for any version,
+# not used with "service_check_cmd"
+# * unhealthy_after - number of check fails before unhealthy
+# default: 1
+# * restart_every - number of check fails before restart
+# default: 0, meaning no restart
+# * service_stop_cmd - command to stop service
+# default: no default, must be provided if
+# restart_every > 0
+# * service_start_cmd - command to start service
+# default: no default, must be provided if
+# restart_every > 0
+# * service_check_cmd - command to check health of service
+# default is to check RPC service using rpcinfo
+# * service_debug_cmd - command to debug a service after trying to stop it;
+# for example, it can be useful to print stack
+# traces of threads that have not exited, since
+# they may be stuck doing I/O;
+# no default, see also function program_stack_traces()
+#
+# Quoting in values is not preserved
+#
+######################################################
+nfs_check_service ()
+{
+ _progname="$1"
+
+ # This sub-shell is created to intentionally limit the scope of
+ # variable values read from the .check files.
+ # shellcheck disable=SC2030
+ (
+ # Subshell to restrict scope variables...
+
+ # Defaults
+ family="tcp"
+ version=""
+ unhealthy_after=1
+ restart_every=0
+ service_stop_cmd=""
+ service_start_cmd=""
+ service_check_cmd=""
+ service_debug_cmd=""
+
+ # Eval line-by-line. Expands variable references in values.
+ # Also allows variable name checking, which seems useful.
+ while read _line ; do
+ case "$_line" in
+ \#*|"") : ;; # Ignore comments, blank lines
+
+ family=*|version=*|\
+ unhealthy_after=*|restart_every=*|\
+ service_stop_cmd=*|service_start_cmd=*|\
+ service_check_cmd=*|service_debug_cmd=*)
+
+ eval "$_line"
+ ;;
+ *)
+ echo "ERROR: Unknown variable for ${_progname}: ${_line}"
+ exit 1
+ esac
+ done
+
+ _ok=false
+ if [ -n "$service_check_cmd" ] ; then
+ # Using eval means variables can contain semicolon separated commands
+ if eval "$service_check_cmd" ; then
+ _ok=true
+ else
+ _err="monitoring service \"${_progname}\" failed"
+ fi
+ else
+ if nfs_check_rpcinfo \
+ "$_progname" "$version" "$family" >/dev/null ; then
+ _ok=true
+ else
+ _err="$ctdb_check_rpc_out"
+ fi
+ fi
+
+ if $_ok ; then
+ if [ $unhealthy_after -ne 1 ] || [ $restart_every -ne 0 ] ; then
+ ctdb_counter_init "$_progname"
+ fi
+ exit 0
+ fi
+
+ ctdb_counter_incr "$_progname"
+ _failcount=$(ctdb_counter_get "$_progname")
+
+ _unhealthy=false
+ if [ "$unhealthy_after" -gt 0 ] ; then
+ if [ "$_failcount" -ge "$unhealthy_after" ] ; then
+ _unhealthy=true
+ echo "ERROR: $_err"
+ fi
+ fi
+
+ if [ "$restart_every" -gt 0 ] ; then
+ if [ $((_failcount % restart_every)) -eq 0 ] ; then
+ if ! $_unhealthy ; then
+ echo "WARNING: $_err"
+ fi
+ nfs_restart_service
+ fi
+ fi
+
+ if $_unhealthy ; then
+ exit 1
+ fi
+
+ return 0
+ ) || exit 1
+}
+
+# Uses: service_stop_cmd, service_start_cmd, service_debug_cmd
+# This function is called within the sub-shell that shellcheck thinks
+# loses the above variable values.
+# shellcheck disable=SC2031
+nfs_restart_service ()
+{
+ if [ -z "$service_stop_cmd" ] || [ -z "$service_start_cmd" ] ; then
+ die "ERROR: Can not restart service \"${_progname}\" without corresponding service_start_cmd/service_stop_cmd settings"
+ fi
+
+ echo "Trying to restart service \"${_progname}\"..."
+ # Using eval means variables can contain semicolon separated commands
+ eval "$service_stop_cmd"
+ if [ -n "$service_debug_cmd" ] ; then
+ eval "$service_debug_cmd"
+ fi
+ background_with_logging eval "$service_start_cmd"
+}
+
+######################################################
+# Check an RPC service with rpcinfo
+######################################################
+ctdb_check_rpc ()
+{
+ _progname="$1" # passed to rpcinfo (looked up in /etc/rpc)
+ _version="$2" # optional, not passed if empty/unset
+ _family="${3:-tcp}" # optional, default is "tcp"
+
+ case "$_family" in
+ tcp6|udp6)
+ _localhost="${CTDB_RPCINFO_LOCALHOST6:-::1}"
+ ;;
+ *)
+ _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
+ esac
+
+ # $_version is not quoted because it is optional
+ # shellcheck disable=SC2086
+ if ! ctdb_check_rpc_out=$(rpcinfo -T "$_family" "$_localhost" \
+ "$_progname" $_version 2>&1) ; then
+ ctdb_check_rpc_out="$_progname failed RPC check:
+$ctdb_check_rpc_out"
+ echo "$ctdb_check_rpc_out"
+ return 1
+ fi
+}
+
+nfs_check_rpcinfo ()
+{
+ _progname="$1" # passed to rpcinfo (looked up in /etc/rpc)
+ _versions="$2" # optional, space separated, not passed if empty/unset
+ _families="${3:-tcp}" # optional, space separated, default is "tcp"
+
+ for _family in $_families ; do
+ if [ -n "$_versions" ] ; then
+ for _version in $_versions ; do
+ ctdb_check_rpc "$_progname" "$_version" "$_family" || return $?
+ done
+ else
+ ctdb_check_rpc "$_progname" "" "$_family" || return $?
+ fi
+ done
+}
+
+##################################################################
+# use statd-callout to update NFS lock info
+##################################################################
+nfs_update_lock_info ()
+{
+ if [ -x "$CTDB_BASE/statd-callout" ] ; then
+ "$CTDB_BASE/statd-callout" update
+ fi
+}
+
+######################################################################
+
+# script_state_dir set by ctdb_setup_state_dir()
+# shellcheck disable=SC2154
+nfs_callout_init "$script_state_dir"
+
+case "$1" in
+startup)
+ nfs_callout "$@" || exit $?
+ ;;
+
+shutdown)
+ nfs_callout "$@" || exit $?
+ ;;
+
+takeip)
+ nfs_callout "$@" || exit $?
+ ctdb_service_set_reconfigure
+ ;;
+
+releaseip)
+ nfs_callout "$@" || exit $?
+ ctdb_service_set_reconfigure
+ ;;
+
+ipreallocated)
+ if ctdb_service_needs_reconfigure ; then
+ ctdb_service_reconfigure
+ fi
+ ;;
+
+monitor)
+ nfs_callout "monitor-pre" || exit $?
+
+ # Check that directories for shares actually exist
+ if [ "$CTDB_NFS_SKIP_SHARE_CHECK" != "yes" ] ; then
+ nfs_callout "monitor-list-shares" | ctdb_check_directories || \
+ exit $?
+ fi
+
+ update_tickles 2049
+ nfs_update_lock_info
+
+ nfs_check_services
+
+ nfs_callout "monitor-post" || exit $?
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/70.iscsi.script b/ctdb/config/events/legacy/70.iscsi.script
new file mode 100755
index 0000000..e74651d
--- /dev/null
+++ b/ctdb/config/events/legacy/70.iscsi.script
@@ -0,0 +1,87 @@
+#!/bin/sh
+
+# CTDB event script for TGTD based iSCSI
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+# service_name is used by various functions
+# shellcheck disable=SC2034
+service_name="iscsi"
+
+load_script_options
+
+[ -z "$CTDB_START_ISCSI_SCRIPTS" ] && {
+ echo "No iscsi start script directory found"
+ exit 0
+}
+
+case "$1" in
+ipreallocated)
+ all_ips=$($CTDB -X ip | tail -n +2)
+
+ # Block the iSCSI port. Only block for the address families
+ # we have configured. This copes with, for example, ip6tables
+ # being unavailable on an IPv4-only system.
+ have_ipv4=false
+ have_ipv6=false
+ # x is intentionally ignored
+ # shellcheck disable=SC2034
+ while IFS='|' read x ip pnn x ; do
+ case "$ip" in
+ *:*) have_ipv6=true ;;
+ *) have_ipv4=true ;;
+ esac
+ done <<EOF
+$all_ips
+EOF
+ if $have_ipv4 ; then
+ iptables -I INPUT 1 -p tcp --dport 3260 -j DROP
+ fi
+ if $have_ipv6 ; then
+ ip6tables -I INPUT 1 -p tcp --dport 3260 -j DROP
+ fi
+
+ # Stop iSCSI daemon
+ killall -9 tgtd >/dev/null 2>/dev/null
+
+ pnn=$(ctdb_get_pnn)
+ [ -n "$pnn" ] || die "Failed to get node pnn"
+
+ # Start iSCSI daemon
+ tgtd >/dev/null 2>&1
+
+ # Run a script for each currently hosted public IP address
+ ips=$(echo "$all_ips" | awk -F'|' -v pnn="$pnn" '$3 == pnn {print $2}')
+ for ip in $ips ; do
+ script="${CTDB_START_ISCSI_SCRIPTS}/${ip}.sh"
+ if [ -x "$script" ] ; then
+ echo "Starting iSCSI service for public address ${ip}"
+ "$script"
+ fi
+ done
+
+ # Unblock iSCSI port. These can be unconditional (compared to
+ # blocking above), since errors are redirected.
+ while iptables -D INPUT -p tcp --dport 3260 -j DROP >/dev/null 2>&1 ; do
+ :
+ done
+ while ip6tables -D INPUT -p tcp --dport 3260 -j DROP >/dev/null 2>&1 ; do
+ :
+ done
+
+ ;;
+
+shutdown)
+ # Shutdown iSCSI daemon when ctdb goes down
+ killall -9 tgtd >/dev/null 2>&1
+ ;;
+
+monitor)
+ ctdb_check_tcp_ports 3260 || exit $?
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/events/legacy/91.lvs.script b/ctdb/config/events/legacy/91.lvs.script
new file mode 100755
index 0000000..8855068
--- /dev/null
+++ b/ctdb/config/events/legacy/91.lvs.script
@@ -0,0 +1,124 @@
+#!/bin/sh
+# script to manage the lvs ip multiplexer for a single public address cluster
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+load_script_options
+
+[ -n "$CTDB_LVS_NODES" ] || exit 0
+export CTDB_LVS_NODES
+
+# type is commonly supported and more portable than which(1)
+# shellcheck disable=SC2039
+if ! type ipvsadm >/dev/null 2>&1 ; then
+ echo "LVS configured but ipvsadm not found"
+ exit 0
+fi
+
+
+lvs_follower_only ()
+{
+ _ip_address=$(ctdb_get_ip_address)
+ awk -v my_ip="$_ip_address" \
+ '$1 == my_ip { if ($2 ~ "follower-only") { exit 0 } else { exit 1 } }' \
+ "$CTDB_LVS_NODES"
+}
+
+lvs_check_config ()
+{
+ [ -r "$CTDB_LVS_NODES" ] || \
+ die "error: CTDB_LVS_NODES=${CTDB_LVS_NODES} unreadable"
+ [ -n "$CTDB_LVS_PUBLIC_IP" ] || \
+ die "Invalid configuration: CTDB_LVS_PUBLIC_IP not set"
+ if ! lvs_follower_only ; then
+ [ -n "$CTDB_LVS_PUBLIC_IFACE" ] || \
+ die "Invalid configuration: CTDB_LVS_PUBLIC_IFACE not set"
+ fi
+}
+
+case "$1" in
+setup)
+ lvs_check_config
+ ;;
+startup)
+ lvs_check_config
+
+ ipvsadm -D -t "$CTDB_LVS_PUBLIC_IP" >/dev/null 2>&1
+ ipvsadm -D -u "$CTDB_LVS_PUBLIC_IP" >/dev/null 2>&1
+
+ ip addr add "${CTDB_LVS_PUBLIC_IP}/32" dev lo scope host
+
+ # do not respond to ARPs that are for ip addresses with scope 'host'
+ set_proc_maybe sys/net/ipv4/conf/all/arp_ignore 3
+ # do not send out arp requests from loopback addresses
+ set_proc_maybe sys/net/ipv4/conf/all/arp_announce 2
+ ;;
+
+shutdown)
+ lvs_check_config
+
+ ipvsadm -D -t "$CTDB_LVS_PUBLIC_IP"
+ ipvsadm -D -u "$CTDB_LVS_PUBLIC_IP"
+
+ ip addr del "${CTDB_LVS_PUBLIC_IP}/32" dev lo >/dev/null 2>&1
+
+ flush_route_cache
+ ;;
+
+ipreallocated)
+ lvs_check_config
+
+ # Kill connections
+ ipvsadm -D -t "$CTDB_LVS_PUBLIC_IP" >/dev/null 2>&1
+ ipvsadm -D -u "$CTDB_LVS_PUBLIC_IP" >/dev/null 2>&1
+ kill_tcp_connections_local_only \
+ "$CTDB_LVS_PUBLIC_IFACE" "$CTDB_LVS_PUBLIC_IP"
+
+ pnn=$(ctdb_get_pnn)
+ lvsleader=$("${CTDB_HELPER_BINDIR}/ctdb_lvs" leader)
+ if [ "$pnn" != "$lvsleader" ] ; then
+ # This node is not the LVS leader so change the IP address
+ # to have scope "host" so this node won't respond to ARPs
+ ip addr del "${CTDB_LVS_PUBLIC_IP}/32" dev lo >/dev/null 2>&1
+ ip addr add "${CTDB_LVS_PUBLIC_IP}/32" dev lo scope host
+ exit 0
+ fi
+
+ # Change the scope so this node starts responding to ARPs
+ ip addr del "${CTDB_LVS_PUBLIC_IP}/32" dev lo >/dev/null 2>&1
+ ip addr add "${CTDB_LVS_PUBLIC_IP}/32" dev lo >/dev/null 2>&1
+
+ ipvsadm -A -t "$CTDB_LVS_PUBLIC_IP" -p 1999999 -s lc
+ ipvsadm -A -u "$CTDB_LVS_PUBLIC_IP" -p 1999999 -s lc
+
+ # Add all nodes (except this node) as LVS servers
+ "${CTDB_HELPER_BINDIR}/ctdb_lvs" list |
+ awk -v pnn="$pnn" '$1 != pnn { print $2 }' |
+ while read ip ; do
+ ipvsadm -a -t "$CTDB_LVS_PUBLIC_IP" -r "$ip" -g
+ ipvsadm -a -u "$CTDB_LVS_PUBLIC_IP" -r "$ip" -g
+ done
+
+ # Add localhost too...
+ ipvsadm -a -t "$CTDB_LVS_PUBLIC_IP" -r 127.0.0.1
+ ipvsadm -a -u "$CTDB_LVS_PUBLIC_IP" -r 127.0.0.1
+
+ $CTDB gratarp \
+ "$CTDB_LVS_PUBLIC_IP" "$CTDB_LVS_PUBLIC_IFACE" >/dev/null 2>&1
+
+ flush_route_cache
+ ;;
+
+monitor)
+ lvs_check_config
+
+ if [ -n "$CTDB_LVS_PUBLIC_IFACE" ] ; then
+ interface_monitor "$CTDB_LVS_PUBLIC_IFACE" || exit 1
+ fi
+ ;;
+esac
+
+exit 0
diff --git a/ctdb/config/functions b/ctdb/config/functions
new file mode 100755
index 0000000..a40b276
--- /dev/null
+++ b/ctdb/config/functions
@@ -0,0 +1,1172 @@
+# Hey Emacs, this is a -*- shell-script -*- !!!
+
+# utility functions for ctdb event scripts
+
+if [ -z "$CTDB_BASE" ]; then
+ echo 'CTDB_BASE unset in CTDB functions file'
+ exit 1
+fi
+export CTDB_BASE
+
+# CTDB_VARDIR is used elsewhere
+# shellcheck disable=SC2034
+CTDB_VARDIR="/usr/local/var/lib/ctdb"
+
+CTDB="${CTDB:-/usr/local/bin/ctdb}"
+
+# Only (and always) override these variables in test code
+
+if [ -z "$CTDB_SCRIPT_VARDIR" ]; then
+ CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/scripts"
+fi
+
+if [ -z "$CTDB_SYS_ETCDIR" ]; then
+ CTDB_SYS_ETCDIR="/etc"
+fi
+
+if [ -z "$CTDB_HELPER_BINDIR" ]; then
+ CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
+fi
+
+#######################################
+# pull in a system config file, if any
+
+load_system_config()
+{
+ for _i; do
+
+ if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/${_i}" ]; then
+ . "${CTDB_SYS_ETCDIR}/sysconfig/${_i}"
+ return
+ elif [ -f "${CTDB_SYS_ETCDIR}/default/${_i}" ]; then
+ . "${CTDB_SYS_ETCDIR}/default/${_i}"
+ return
+ fi
+ done
+}
+
+# load_script_options [ component script ]
+# script is an event script name relative to a component
+# component is currently ignored
+load_script_options()
+{
+ if [ $# -eq 2 ]; then
+ _script="$2"
+ elif [ $# -eq 0 ]; then
+ _script=""
+ else
+ die "usage: load_script_options [ component script ]"
+ fi
+
+ _options="${CTDB_BASE}/script.options"
+
+ if [ -r "$_options" ]; then
+ . "$_options"
+ fi
+
+ if [ -n "$_script" ]; then
+ _s="${CTDB_BASE}/events/legacy/${_script}"
+ else
+ _s="${0%.script}"
+ fi
+ _options="${_s}.options"
+
+ if [ -r "$_options" ]; then
+ . "$_options"
+ fi
+}
+
+##############################################################
+
+die()
+{
+ _msg="$1"
+ _rc="${2:-1}"
+
+ echo "$_msg" >&2
+ exit "$_rc"
+}
+
+# Log given message or stdin to either syslog or a CTDB log file
+# $1 is the tag passed to logger if syslog is in use.
+script_log()
+{
+ _tag="$1"
+ shift
+
+ case "$CTDB_LOGGING" in
+ file:)
+ if [ -n "$*" ] ; then
+ echo "$*"
+ else
+ cat
+ fi >&2
+ ;;
+ file:* | "")
+ if [ -n "$CTDB_LOGGING" ]; then
+ _file="${CTDB_LOGGING#file:}"
+ else
+ _file="/usr/local/var/log/log.ctdb"
+ fi
+ {
+ if [ -n "$*" ]; then
+ echo "$*"
+ else
+ cat
+ fi
+ } >>"$_file"
+ ;;
+ *)
+ # Handle all syslog:* variants here too. There's no tool to do
+ # the lossy things, so just use logger.
+ logger -t "ctdbd: ${_tag}" "$@"
+ ;;
+ esac
+}
+
+# When things are run in the background in an eventscript then logging
+# output might get lost. This is the "solution". :-)
+background_with_logging()
+{
+ (
+ "$@" 2>&1 </dev/null |
+ script_log "${script_name}&"
+ ) &
+
+ return 0
+}
+
+##############################################################
+# check number of args for different events
+ctdb_check_args()
+{
+ case "$1" in
+ takeip | releaseip)
+ if [ $# != 4 ]; then
+ echo "ERROR: must supply interface, IP and maskbits"
+ exit 1
+ fi
+ ;;
+ updateip)
+ if [ $# != 5 ]; then
+ echo "ERROR: must supply old interface, new interface, IP and maskbits"
+ exit 1
+ fi
+ ;;
+ esac
+}
+
+##############################################################
+# determine on what type of system (init style) we are running
+detect_init_style()
+{
+ _init_style_file="${CTDB_SCRIPT_VARDIR}/init-style"
+
+ if [ ! -f "$_init_style_file" ]; then
+ if [ -n "$CTDB_INIT_STYLE" ]; then
+ echo "$CTDB_INIT_STYLE" >"$_init_style_file"
+ return
+ fi
+
+ # Subshell to contain variables in os-release file
+ (
+ _os_release="${CTDB_SYS_ETCDIR}/os-release"
+ if [ -f "$_os_release" ]; then
+ . "$_os_release"
+ case "$ID" in
+ centos | fedora | rhel)
+ echo "redhat"
+ ;;
+ debian | ubuntu)
+ echo "debian"
+ ;;
+ sles | suse)
+ echo "suse"
+ ;;
+ *)
+ case "$ID_LIKE" in
+ *centos* | *rhel*)
+ echo "redhat"
+ ;;
+ *)
+ echo "$ID"
+ ;;
+ esac
+ ;;
+ esac
+ else
+ echo "WARNING: unknown distribution ${ID}" >&2
+ echo "unknown"
+ fi
+ ) >"$_init_style_file"
+ fi
+
+ read -r CTDB_INIT_STYLE <"$_init_style_file"
+}
+
+######################################################
+# simulate /sbin/service on platforms that don't have it
+# _service() makes it easier to hook the service() function for
+# testing.
+_service()
+{
+ _service_name="$1"
+ _op="$2"
+
+ # do nothing, when no service was specified
+ [ -z "$_service_name" ] && return
+
+ if [ -x /sbin/service ]; then
+ $_nice /sbin/service "$_service_name" "$_op"
+ elif [ -x /usr/sbin/service ]; then
+ $_nice /usr/sbin/service "$_service_name" "$_op"
+ elif [ -x /bin/systemctl ]; then
+ $_nice /bin/systemctl "$_op" "$_service_name"
+ elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
+ $_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
+ elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
+ $_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
+ fi
+}
+
+service()
+{
+ _nice=""
+ _service "$@"
+}
+
+######################################################
+# simulate /sbin/service (niced) on platforms that don't have it
+nice_service()
+{
+ _nice="nice"
+ _service "$@"
+}
+
+######################################################
+# Cached retrieval of PNN from local node. This never changes so why
+# open a client connection to the server each time this is needed?
+ctdb_get_pnn()
+{
+ _pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
+ if [ ! -f "$_pnn_file" ]; then
+ $CTDB pnn >"$_pnn_file"
+ fi
+
+ cat "$_pnn_file"
+}
+
+# Cached retrieval of private IP address from local node. This never
+# changes.
+ctdb_get_ip_address()
+{
+ _ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
+ if [ ! -f "$_ip_addr_file" ]; then
+ $CTDB -X nodestatus |
+ awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
+ fi
+
+ cat "$_ip_addr_file"
+}
+
+# Cached retrieval of database options for use by event scripts.
+#
+# If the variables are already set then they should not be overwritten
+# - this should only happen during event script testing.
+ctdb_get_db_options()
+{
+ _db_opts_file="${CTDB_SCRIPT_VARDIR}/db_options.cache"
+
+ if [ ! -f "$_db_opts_file" ]; then
+ {
+ ctdb_translate_option "database" \
+ "volatile database directory" \
+ "CTDB_DBDIR"
+ ctdb_translate_option "database" \
+ "persistent database directory" \
+ "CTDB_DBDIR_PERSISTENT"
+ ctdb_translate_option "database" \
+ "state database directory" \
+ "CTDB_DBDIR_STATE"
+ } >"$_db_opts_file"
+ fi
+
+ . "$_db_opts_file"
+}
+
+ctdb_translate_option()
+{
+ _section="$1"
+ _opt="$2"
+ _variable="$3"
+
+ # ctdb-config already prints an error if something goes wrong
+ _t=$("${CTDB_HELPER_BINDIR}/ctdb-config" get "$_section" "$_opt") ||
+ exit $?
+ echo "${_variable}=\"${_t}\""
+}
+
+######################################################
+# wrapper around /proc/ settings to allow them to be hooked
+# for testing
+# 1st arg is relative path under /proc/, 2nd arg is value to set
+set_proc()
+{
+ echo "$2" >"/proc/$1"
+}
+
+set_proc_maybe()
+{
+ if [ -w "/proc/$1" ]; then
+ set_proc "$1" "$2"
+ fi
+}
+
+######################################################
+# wrapper around getting file contents from /proc/ to allow
+# this to be hooked for testing
+# 1st arg is relative path under /proc/
+get_proc()
+{
+ cat "/proc/$1"
+}
+
+######################################################
+# Print up to $_max kernel stack traces for processes named $_program
+program_stack_traces()
+{
+ _prog="$1"
+ _max="${2:-1}"
+
+ _count=1
+ for _pid in $(pidof "$_prog"); do
+ [ "$_count" -le "$_max" ] || break
+
+ # Do this first to avoid racing with process exit
+ _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
+ if [ -n "$_stack" ]; then
+ echo "Stack trace for ${_prog}[${_pid}]:"
+ echo "$_stack"
+ _count=$((_count + 1))
+ fi
+ done
+}
+
+######################################################
+# Ensure $service_name is set
+assert_service_name()
+{
+ # service_name is set by the event script
+ # shellcheck disable=SC2154
+ [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
+}
+
+######################################################
+# check a set of directories is available
+# return 1 on a missing directory
+# directories are read from stdin
+######################################################
+ctdb_check_directories_probe()
+{
+ while IFS="" read -r d; do
+ case "$d" in
+ *%*)
+ continue
+ ;;
+ *)
+ [ -d "${d}/." ] || return 1
+ ;;
+ esac
+ done
+}
+
+######################################################
+# check a set of directories is available
+# directories are read from stdin
+######################################################
+ctdb_check_directories()
+{
+ ctdb_check_directories_probe || {
+ echo "ERROR: $service_name directory \"$d\" not available"
+ exit 1
+ }
+}
+
+######################################################
+# check a set of tcp ports
+# usage: ctdb_check_tcp_ports <ports...>
+######################################################
+
+# Check whether something is listening on all of the given TCP ports
+# using the "ctdb checktcpport" command.
+ctdb_check_tcp_ports()
+{
+ if [ -z "$1" ]; then
+ echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
+ exit 1
+ fi
+
+ for _p; do # process each function argument (port)
+ _cmd="$CTDB checktcpport $_p"
+ _out=$($_cmd 2>&1)
+ _ret=$?
+ case "$_ret" in
+ 0)
+ echo "$service_name not listening on TCP port $_p"
+ return 1
+ ;;
+ 98)
+ # Couldn't bind, something already listening, next port
+ continue
+ ;;
+ *)
+ echo "unexpected error (${_ret}) running \"${_cmd}\""
+ if [ -n "$_out" ]; then
+ echo "$_out"
+ fi
+ return $_ret
+ ;;
+ esac
+ done
+
+ # All ports listening
+ return 0
+}
+
+######################################################
+# check a unix socket
+# usage: ctdb_check_unix_socket SOCKPATH
+######################################################
+ctdb_check_unix_socket()
+{
+ _sockpath="$1"
+
+ if [ -z "$_sockpath" ]; then
+ echo "ERROR: ctdb_check_unix_socket() requires socket path"
+ return 1
+ fi
+
+ _out=$(ss -l -x "src ${_sockpath}" | tail -n +2)
+ if [ -z "$_out" ]; then
+ echo "ERROR: ${service_name} not listening on ${_sockpath}"
+ return 1
+ fi
+}
+
+################################################
+# kill off any TCP connections with the given IP
+################################################
+kill_tcp_connections()
+{
+ _iface="$1"
+ _ip="$2"
+
+ _oneway=false
+ if [ "$3" = "oneway" ]; then
+ _oneway=true
+ fi
+
+ get_tcp_connections_for_ip "$_ip" | {
+ _killcount=0
+ _connections=""
+ _nl="
+"
+ while read -r _dst _src; do
+ _destport="${_dst##*:}"
+ __oneway=$_oneway
+ case $_destport in
+ # we only do one-way killtcp for CIFS
+ 139 | 445) __oneway=true ;;
+ esac
+
+ _connections="${_connections}${_nl}${_src} ${_dst}"
+ if ! $__oneway; then
+ _connections="${_connections}${_nl}${_dst} ${_src}"
+ fi
+
+ _killcount=$((_killcount + 1))
+ done
+
+ if [ $_killcount -eq 0 ]; then
+ return
+ fi
+
+ if [ -n "$CTDB_KILLTCP_DEBUGLEVEL" ]; then
+ _debuglevel="$CTDB_KILLTCP_DEBUGLEVEL"
+ else
+ _debuglevel="$CTDB_DEBUGLEVEL"
+ fi
+ echo "$_connections" |
+ CTDB_DEBUGLEVEL="$_debuglevel" \
+ "${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
+ echo "Failed to kill TCP connections"
+ return
+ }
+
+ _connections=$(get_tcp_connections_for_ip "$_ip")
+ if [ -z "$_connections" ]; then
+ _remaining=0
+ else
+ _remaining=$(echo "$_connections" | wc -l)
+ fi
+
+ _actually_killed=$((_killcount - _remaining))
+
+ _t="${_actually_killed}/${_killcount}"
+ echo "Killed ${_t} TCP connections to released IP $_ip"
+
+ if [ -n "$_connections" ]; then
+ echo "Remaining connections:"
+ echo "$_connections" | sed -e 's|^| |'
+ fi
+ }
+}
+
+##################################################################
+# kill off the local end for any TCP connections with the given IP
+##################################################################
+kill_tcp_connections_local_only()
+{
+ kill_tcp_connections "$@" "oneway"
+}
+
+##################################################################
+# tickle any TCP connections with the given IP
+##################################################################
+tickle_tcp_connections()
+{
+ _ip="$1"
+
+ # Get connections, both directions
+ _conns=$(get_tcp_connections_for_ip "$_ip" |
+ awk '{ print $1, $2 ; print $2, $1 }')
+
+ echo "$_conns" | awk '{ print "Tickle TCP connection", $1, $2 }'
+ echo "$_conns" | ctdb tickle
+}
+
+get_tcp_connections_for_ip()
+{
+ _ip="$1"
+
+ ss -tn state established "src [$_ip]" | awk 'NR > 1 {print $3, $4}'
+}
+
+########################################################
+
+add_ip_to_iface()
+{
+ _iface=$1
+ _ip=$2
+ _maskbits=$3
+
+ # Ensure interface is up
+ ip link set "$_iface" up ||
+ die "Failed to bringup interface $_iface"
+
+ # Only need to define broadcast for IPv4
+ case "$_ip" in
+ *:*) _bcast="" ;;
+ *) _bcast="brd +" ;;
+ esac
+
+ # Intentionally unquoted multi-word value here
+ # shellcheck disable=SC2086
+ ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
+ echo "Failed to add $_ip/$_maskbits on dev $_iface"
+ return 1
+ }
+
+ # Wait 5 seconds for IPv6 addresses to stop being tentative...
+ if [ -z "$_bcast" ]; then
+ for _x in $(seq 1 10); do
+ ip addr show to "${_ip}/128" | grep -q "tentative" || break
+ sleep 0.5
+ done
+
+ # If the address was a duplicate then it won't be on the
+ # interface so flag an error.
+ _t=$(ip addr show to "${_ip}/128")
+ case "$_t" in
+ "")
+ echo "Failed to add $_ip/$_maskbits on dev $_iface"
+ return 1
+ ;;
+ *tentative* | *dadfailed*)
+ echo "Failed to add $_ip/$_maskbits on dev $_iface"
+ ip addr del "$_ip/$_maskbits" dev "$_iface"
+ return 1
+ ;;
+ esac
+ fi
+}
+
+delete_ip_from_iface()
+{
+ _iface=$1
+ _ip=$2
+ _maskbits=$3
+
+ # This could be set globally for all interfaces but it is probably
+ # better to avoid surprises, so limit it the interfaces where CTDB
+ # has public IP addresses. There isn't anywhere else convenient
+ # to do this so just set it each time. This is much cheaper than
+ # remembering and re-adding secondaries.
+ set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
+
+ ip addr del "$_ip/$_maskbits" dev "$_iface" || {
+ echo "Failed to del $_ip on dev $_iface"
+ return 1
+ }
+}
+
+# If the given IP is hosted then print 2 items: maskbits and iface
+ip_maskbits_iface()
+{
+ _addr="$1"
+
+ case "$_addr" in
+ *:*) _bits=128 ;;
+ *) _bits=32 ;;
+ esac
+ ip addr show to "${_addr}/${_bits}" 2>/dev/null |
+ awk 'NR == 1 { iface = $2; sub(":$", "", iface) ;
+ sub("@.*", "", iface) }
+ $1 ~ /inet/ { mask = $2; sub(".*/", "", mask);
+ print mask, iface }'
+}
+
+drop_ip()
+{
+ _addr="${1%/*}" # Remove optional maskbits
+
+ # Intentional word splitting here
+ # shellcheck disable=SC2046
+ set -- $(ip_maskbits_iface "$_addr")
+ if [ -n "$1" ]; then
+ _maskbits="$1"
+ _iface="$2"
+ echo "Removing public address $_addr/$_maskbits from device $_iface"
+ delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
+ fi
+}
+
+drop_all_public_ips()
+{
+ # _x is intentionally ignored
+ # shellcheck disable=SC2034
+ while read -r _ip _x; do
+ case "$_ip" in
+ \#*) continue ;;
+ esac
+ drop_ip "$_ip"
+ done <"${CTDB_BASE}/public_addresses"
+}
+
+flush_route_cache()
+{
+ set_proc_maybe sys/net/ipv4/route/flush 1
+ set_proc_maybe sys/net/ipv6/route/flush 1
+}
+
+########################################################
+# Interface monitoring
+
+# If the interface is a virtual one (e.g. VLAN) then get the
+# underlying interface
+interface_get_real()
+{
+ _iface="$1"
+
+ # If $_iface is a VLAN (i.e. contains an '@') then strip every
+ # before the '@', otherwise print the whole interface
+ echo "${_iface##*@}"
+}
+
+# Check whether an interface is operational
+interface_monitor()
+{
+ _iface="$1"
+
+ _iface_info=$(ip -br link show "$_iface" 2>&1) || {
+ echo "ERROR: Monitored interface ${_iface} does not exist"
+ return 1
+ }
+
+ # If the interface is a virtual one (e.g. VLAN) then get the
+ # underlying interface.
+ _realiface=$(interface_get_real "${_iface_info%% *}")
+
+ if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null); then
+ # This is a bond: various monitoring strategies
+ echo "$_bi" | grep -q 'Currently Active Slave: None' && {
+ echo "ERROR: No active slaves for bond device ${_realiface}"
+ return 1
+ }
+ echo "$_bi" | grep -q '^MII Status: up' || {
+ echo "ERROR: public network interface ${_realiface} is down"
+ return 1
+ }
+ echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
+ # This works around a bug in the driver where the
+ # overall bond status can be up but none of the actual
+ # physical interfaces have a link.
+ echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
+ echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
+ return 1
+ }
+ }
+
+ return 0
+ else
+ # Not a bond
+ case "$_iface" in
+ lo*)
+ # loopback is always working
+ return 0
+ ;;
+ ib*)
+ # we don't know how to test ib links
+ return 0
+ ;;
+ *)
+ ethtool "$_iface" | grep -q 'Link detected: yes' || {
+ # On some systems, this is not successful when a
+ # cable is plugged but the interface has not been
+ # brought up previously. Bring the interface up
+ # and try again...
+ ip link set "$_iface" up
+ ethtool "$_iface" | grep -q 'Link detected: yes' || {
+ echo "ERROR: No link on the public network interface ${_iface}"
+ return 1
+ }
+ }
+ return 0
+ ;;
+ esac
+ fi
+}
+
+########################################################
+# Simple counters
+_ctdb_counter_common()
+{
+ [ $# -le 1 ] || die "usage: _ctdb_counter_common [name]"
+
+ if [ $# -eq 1 ]; then
+ _counter_name="${1}.failcount"
+ else
+ _counter_name="failcount"
+ fi
+
+ if [ -z "$script_state_dir" ]; then
+ die "ctdb_counter_* functions need ctdb_setup_state_dir()"
+ fi
+
+ _counter_file="${script_state_dir}/${_counter_name}"
+}
+# Some code passes an argument
+# shellcheck disable=SC2120
+ctdb_counter_init()
+{
+ _ctdb_counter_common "$1"
+
+ : >"$_counter_file"
+}
+ctdb_counter_incr()
+{
+ _ctdb_counter_common "$1"
+
+ # unary counting using newlines!
+ echo >>"$_counter_file"
+}
+ctdb_counter_get()
+{
+ _ctdb_counter_common "$1"
+ # unary counting!
+ _val=$(wc -c 2>/dev/null <"$_counter_file" || echo 0)
+ # Strip leading spaces from output of wc (on freebsd)
+ # shellcheck disable=SC2086
+ echo $_val
+}
+
+#
+# Fail counter/threshold combination to control warnings and node unhealthy
+#
+
+_failcount_validate_threshold()
+{
+ case "$1" in
+ "") return 1 ;; # A failure that doesn't need a warning
+ *)
+ if echo "$1" | grep -qx '[0-9]*'; then
+ return 0
+ fi
+
+ echo "WARNING: ${1} is an invalid threshold in \"${2}\" check"
+ return 1
+ ;;
+ esac
+}
+
+_failcount_common()
+{
+ _thing="$1"
+
+ _counter=$(echo "$_thing" | sed -e 's@/@_SLASH_@g' -e 's@ @_@g')
+}
+
+failcount_init()
+{
+ _thing="$1"
+
+ _failcount_common "$_thing"
+
+ ctdb_counter_init "$_counter"
+}
+
+failcount_reset()
+{
+ _thing="$1"
+
+ _failcount_common "$_thing"
+
+ _failcount=$(ctdb_counter_get "$_counter")
+ if [ "$_failcount" -eq 0 ]; then
+ return
+ fi
+
+ printf 'NOTICE: %s: no longer failing\n' "$_thing"
+ ctdb_counter_init "$_counter"
+}
+
+failcount_incr()
+{
+ _thing="$1"
+ _thresholds="$2"
+ _output="$3"
+
+ _failcount_common "$_thing"
+
+ ctdb_counter_incr "$_counter"
+ _failcount=$(ctdb_counter_get "$_counter")
+
+ case "$_thresholds" in
+ *:*)
+ _warn_threshold="${_thresholds%:*}"
+ _unhealthy_threshold="${_thresholds#*:}"
+ ;;
+ "")
+ _warn_threshold=1
+ _unhealthy_threshold=""
+ ;;
+ *)
+ _warn_threshold="$_thresholds"
+ _unhealthy_threshold=""
+ ;;
+ esac
+
+ if _failcount_validate_threshold "$_unhealthy_threshold" "$_thing"; then
+ if [ "$_failcount" -ge "$_unhealthy_threshold" ]; then
+ printf 'ERROR: %s: fail count %d >= threshold %d\n' \
+ "$_thing" \
+ "$_failcount" \
+ "$_unhealthy_threshold"
+ # Only print output when exceeding the
+ # unhealthy threshold
+ if [ "$_failcount" -eq "$_unhealthy_threshold" ] && \
+ [ -n "$_output" ]; then
+ echo "$_output"
+ fi
+ exit 1
+ fi
+ fi
+
+ if _failcount_validate_threshold "$_warn_threshold" "$_thing"; then
+ if [ "$_failcount" -lt "$_warn_threshold" ]; then
+ return 0
+ fi
+ fi
+
+ printf 'WARNING: %s: fail count %d >= threshold %d\n' \
+ "$_thing" \
+ "$_failcount" \
+ "$_warn_threshold"
+ if [ "$_failcount" -eq "$_warn_threshold" ] && [ -n "$_output" ]; then
+ # Only print output when exceeding the warning threshold
+ echo "$_output"
+ fi
+}
+
+########################################################
+
+# ctdb_setup_state_dir <type> <name>
+# Sets/creates script_state_dir)
+ctdb_setup_state_dir()
+{
+ [ $# -eq 2 ] || die "usage: ctdb_setup_state_dir <type> <name>"
+
+ _type="$1"
+ _name="$2"
+
+ script_state_dir="${CTDB_SCRIPT_VARDIR}/${_type}/${_name}"
+
+ mkdir -p "$script_state_dir" ||
+ die "Error creating script state dir \"${script_state_dir}\""
+}
+
+##################################################################
+# Reconfigure a service on demand
+
+_ctdb_service_reconfigure_common()
+{
+ if [ -z "$script_state_dir" ]; then
+ die "ctdb_service_*_reconfigure() needs ctdb_setup_state_dir()"
+ fi
+
+ _ctdb_service_reconfigure_flag="${script_state_dir}/need_reconfigure"
+}
+
+ctdb_service_needs_reconfigure()
+{
+ _ctdb_service_reconfigure_common
+ [ -e "$_ctdb_service_reconfigure_flag" ]
+}
+
+ctdb_service_set_reconfigure()
+{
+ _ctdb_service_reconfigure_common
+ : >"$_ctdb_service_reconfigure_flag"
+}
+
+ctdb_service_unset_reconfigure()
+{
+ _ctdb_service_reconfigure_common
+ rm -f "$_ctdb_service_reconfigure_flag"
+}
+
+ctdb_service_reconfigure()
+{
+ echo "Reconfiguring service \"${service_name}\"..."
+ ctdb_service_unset_reconfigure
+ service_reconfigure || return $?
+ # Intentionally have this use $service_name as default
+ # shellcheck disable=SC2119
+ ctdb_counter_init
+}
+
+# Default service_reconfigure() function does nothing.
+service_reconfigure()
+{
+ :
+}
+
+# Default service_start() and service_stop() functions.
+
+# These may be overridden in an eventscript.
+service_start()
+{
+ service "$service_name" start
+}
+
+service_stop()
+{
+ service "$service_name" stop
+}
+
+##################################################################
+
+# This exists only for backward compatibility with 3rd party scripts
+# that call it
+ctdb_standard_event_handler()
+{
+ :
+}
+
+iptables_wrapper()
+{
+ _family="$1"
+ shift
+ if [ "$_family" = "inet6" ]; then
+ _iptables_cmd="ip6tables"
+ else
+ _iptables_cmd="iptables"
+ fi
+
+ # iptables doesn't like being re-entered, so flock-wrap it.
+ flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
+}
+
+# AIX (and perhaps others?) doesn't have mktemp
+# type is commonly supported and more portable than which(1)
+# shellcheck disable=SC2039
+if ! type mktemp >/dev/null 2>&1; then
+ mktemp()
+ {
+ _dir=false
+ if [ "$1" = "-d" ]; then
+ _dir=true
+ shift
+ fi
+ _d="${TMPDIR:-/tmp}"
+ _hex10=$(dd if=/dev/urandom count=20 2>/dev/null |
+ cksum |
+ awk '{print $1}')
+ _t="${_d}/tmp.${_hex10}"
+ (
+ umask 077
+ if $_dir; then
+ mkdir "$_t"
+ else
+ : >"$_t"
+ fi
+ )
+ echo "$_t"
+ }
+fi
+
+######################################################################
+# NFS callout handling
+
+nfs_callout_init()
+{
+ _state_dir="$1"
+
+ if [ -z "$CTDB_NFS_CALLOUT" ]; then
+ CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
+ fi
+ # Always export, for statd callout
+ export CTDB_NFS_CALLOUT
+
+ # If the callout wants to use this then it must create it
+ export CTDB_NFS_CALLOUT_STATE_DIR="${_state_dir}/callout-state"
+
+ # Export, if set, for use by clustered NFS callouts
+ if [ -n "$CTDB_NFS_STATE_FS_TYPE" ]; then
+ export CTDB_NFS_STATE_FS_TYPE
+ fi
+ if [ -n "$CTDB_NFS_STATE_MNT" ]; then
+ export CTDB_NFS_STATE_MNT
+ fi
+
+ nfs_callout_cache="${_state_dir}/nfs_callout_cache"
+ nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
+ nfs_callout_cache_ops="${nfs_callout_cache}/ops"
+}
+
+nfs_callout_register()
+{
+ mkdir -p "$nfs_callout_cache_ops"
+ rm -f "$nfs_callout_cache_ops"/*
+
+ echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"
+
+ _t=$("$CTDB_NFS_CALLOUT" "register")
+ if [ -n "$_t" ]; then
+ echo "$_t" |
+ while IFS="" read -r _op; do
+ touch "${nfs_callout_cache_ops}/${_op}"
+ done
+ else
+ touch "${nfs_callout_cache_ops}/ALL"
+ fi
+}
+
+nfs_callout()
+{
+ # Re-run registration if $CTDB_NFS_CALLOUT has changed
+ _prev=""
+ if [ -r "$nfs_callout_cache_callout" ]; then
+ read -r _prev <"$nfs_callout_cache_callout"
+ fi
+ if [ "$CTDB_NFS_CALLOUT" != "$_prev" ]; then
+ nfs_callout_register
+ fi
+
+ # Run the operation if it is registered...
+ if [ -e "${nfs_callout_cache_ops}/${1}" ] ||
+ [ -e "${nfs_callout_cache_ops}/ALL" ]; then
+ "$CTDB_NFS_CALLOUT" "$@"
+ fi
+}
+
+########################################################
+# tickle handling
+########################################################
+
+update_tickles()
+{
+ _port="$1"
+
+ tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
+ mkdir -p "$tickledir"
+
+ # What public IPs do I hold?
+ _pnn=$(ctdb_get_pnn)
+ _ips=$($CTDB -X ip | awk -F'|' -v pnn="$_pnn" '$3 == pnn {print $2}')
+
+ # IPs and port as ss filters
+ _ip_filter=""
+ for _ip in $_ips; do
+ _ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
+ done
+ _port_filter="sport == :${_port}"
+
+ # Record connections to our public IPs in a temporary file.
+ # This temporary file is in CTDB's private state directory and
+ # $$ is used to avoid a very rare race involving CTDB's script
+ # debugging. No security issue, nothing to see here...
+ _my_connections="${tickledir}/${_port}.connections.$$"
+ # Parentheses are needed around the filters for precedence but
+ # the parentheses can't be empty!
+ #
+ # Recent versions of ss print square brackets around IPv6
+ # addresses. While it is desirable to update CTDB's address
+ # parsing and printing code, something needs to be done here
+ # for backward compatibility, so just delete the brackets.
+ ss -tn state established \
+ "${_ip_filter:+( ${_ip_filter} )}" \
+ "${_port_filter:+( ${_port_filter} )}" |
+ awk 'NR > 1 {print $4, $3}' |
+ tr -d '][' |
+ sort >"$_my_connections"
+
+ # Record our current tickles in a temporary file
+ _my_tickles="${tickledir}/${_port}.tickles.$$"
+ for _i in $_ips; do
+ $CTDB -X gettickles "$_i" "$_port" |
+ awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
+ done |
+ sort >"$_my_tickles"
+
+ # Add tickles for connections that we haven't already got tickles for
+ comm -23 "$_my_connections" "$_my_tickles" |
+ $CTDB addtickle
+
+ # Remove tickles for connections that are no longer there
+ comm -13 "$_my_connections" "$_my_tickles" |
+ $CTDB deltickle
+
+ rm -f "$_my_connections" "$_my_tickles"
+
+ # Remove stale files from killed scripts
+ # Files can't have spaces in name, more portable than -print0/-0
+ # shellcheck disable=SC2038
+ (cd "$tickledir" && find . -type f -mmin +10 | xargs -r rm)
+}
+
+########################################################
+# load a site local config file
+########################################################
+
+[ -x "${CTDB_BASE}/rc.local" ] && {
+ . "${CTDB_BASE}/rc.local"
+}
+
+[ -d "${CTDB_BASE}/rc.local.d" ] && {
+ for i in "${CTDB_BASE}/rc.local.d"/*; do
+ [ -x "$i" ] && . "$i"
+ done
+}
+
+script_name="${0##*/}" # basename
diff --git a/ctdb/config/nfs-checks.d/00.portmapper.check b/ctdb/config/nfs-checks.d/00.portmapper.check
new file mode 100644
index 0000000..24def35
--- /dev/null
+++ b/ctdb/config/nfs-checks.d/00.portmapper.check
@@ -0,0 +1,2 @@
+# portmapper
+unhealthy_after=1
diff --git a/ctdb/config/nfs-checks.d/10.status.check b/ctdb/config/nfs-checks.d/10.status.check
new file mode 100644
index 0000000..b8ce1e0
--- /dev/null
+++ b/ctdb/config/nfs-checks.d/10.status.check
@@ -0,0 +1,7 @@
+# status
+version="1"
+restart_every=2
+unhealthy_after=6
+service_stop_cmd="$CTDB_NFS_CALLOUT stop status"
+service_start_cmd="$CTDB_NFS_CALLOUT start status"
+service_debug_cmd="program_stack_traces rpc.statd 5"
diff --git a/ctdb/config/nfs-checks.d/20.nfs.check b/ctdb/config/nfs-checks.d/20.nfs.check
new file mode 100644
index 0000000..dad1cdc
--- /dev/null
+++ b/ctdb/config/nfs-checks.d/20.nfs.check
@@ -0,0 +1,7 @@
+# nfs
+version="3"
+restart_every=10
+unhealthy_after=2
+service_stop_cmd="$CTDB_NFS_CALLOUT stop nfs"
+service_start_cmd="$CTDB_NFS_CALLOUT start nfs"
+service_debug_cmd="program_stack_traces nfsd 5"
diff --git a/ctdb/config/nfs-checks.d/30.nlockmgr.check b/ctdb/config/nfs-checks.d/30.nlockmgr.check
new file mode 100644
index 0000000..6660ca0
--- /dev/null
+++ b/ctdb/config/nfs-checks.d/30.nlockmgr.check
@@ -0,0 +1,6 @@
+# nlockmgr
+version="4"
+restart_every=2
+unhealthy_after=6
+service_stop_cmd="$CTDB_NFS_CALLOUT stop nlockmgr"
+service_start_cmd="$CTDB_NFS_CALLOUT start nlockmgr"
diff --git a/ctdb/config/nfs-checks.d/40.mountd.check b/ctdb/config/nfs-checks.d/40.mountd.check
new file mode 100644
index 0000000..bfe4c27
--- /dev/null
+++ b/ctdb/config/nfs-checks.d/40.mountd.check
@@ -0,0 +1,7 @@
+# mountd
+version="1"
+restart_every=2
+unhealthy_after=6
+service_stop_cmd="$CTDB_NFS_CALLOUT stop mountd"
+service_start_cmd="$CTDB_NFS_CALLOUT start mountd"
+service_debug_cmd="program_stack_traces rpc.mountd 5"
diff --git a/ctdb/config/nfs-checks.d/50.rquotad.check b/ctdb/config/nfs-checks.d/50.rquotad.check
new file mode 100644
index 0000000..98bd8d9
--- /dev/null
+++ b/ctdb/config/nfs-checks.d/50.rquotad.check
@@ -0,0 +1,7 @@
+# rquotad
+version="1"
+restart_every=2
+unhealthy_after=6
+service_stop_cmd="$CTDB_NFS_CALLOUT stop rquotad"
+service_start_cmd="$CTDB_NFS_CALLOUT start rquotad"
+service_debug_cmd="program_stack_traces rpc.rquotad 5"
diff --git a/ctdb/config/nfs-checks.d/README b/ctdb/config/nfs-checks.d/README
new file mode 100644
index 0000000..044067a
--- /dev/null
+++ b/ctdb/config/nfs-checks.d/README
@@ -0,0 +1,31 @@
+NFS check configuration files.
+
+Files are named NN.RPCSERVICE.check. Files without a .check suffix
+are ignored.
+
+Supported variables are:
+
+* family - "tcp" or "udp" or space separated list
+ default: tcp, not used with "service_check_cmd"
+* version - optional, RPC service version number
+ default is to omit to check for any version,
+ not used with "service_check_cmd"
+* unhealthy_after - number of check fails before unhealthy
+ default: 1
+* restart_every - number of check fails before restart
+ default: 0, meaning no restart
+* service_stop_cmd - command to stop service
+ default: no default, must be provided if
+ restart_every > 0
+* service_start_cmd - command to start service
+ default: no default, must be provided if
+ restart_every > 0
+* service_check_cmd - command to check health of service
+ default is to check RPC service using rpcinfo
+* service_debug_cmd - command to debug a service after trying to stop it;
+ for example, it can be useful to print stack
+ traces of threads that have not exited, since
+ they may be stuck doing I/O;
+ no default, see also function program_stack_traces()
+
+Quoting inside values is not preserved.
diff --git a/ctdb/config/nfs-linux-kernel-callout b/ctdb/config/nfs-linux-kernel-callout
new file mode 100755
index 0000000..f2f3e38
--- /dev/null
+++ b/ctdb/config/nfs-linux-kernel-callout
@@ -0,0 +1,441 @@
+#!/bin/sh
+
+# Exit on 1st error
+set -e
+
+# NFS exports file. Some code below keeps a cache of output derived
+# from exportfs(8). When this file is updated the cache is invalid
+# and needs to be regenerated.
+#
+# To change the file, edit the default value below. Do not set
+# CTDB_NFS_EXPORTS_FILE - it isn't a configuration variable, just a
+# hook for testing.
+nfs_exports_file="${CTDB_NFS_EXPORTS_FILE:-/var/lib/nfs/etab}"
+
+# As above, edit the default value below. CTDB_NFS_DISTRO_STYLE is a
+# test variable only.
+nfs_distro_style="${CTDB_NFS_DISTRO_STYLE:-systemd-redhat}"
+
+# As above, edit the default value below. CTDB_SYS_ETCDIR is a
+# test variable only.
+etc_dir="${CTDB_SYS_ETCDIR:-/etc}"
+
+# A value of "AUTO" for any service means that service is usually
+# automatically started and stopped by one of the other services.
+# Such services will still be restarted by hand on failure, if
+# configured to do so. This allows services that should not be
+# running to be set to "".
+
+case "$nfs_distro_style" in
+systemd-*)
+ # Defaults
+ nfs_service="nfs-server"
+ nfs_lock_service="rpc-statd"
+ nfs_mountd_service="nfs-mountd"
+ nfs_status_service="rpc-statd"
+ nfs_rquotad_service="rpc-rquotad"
+ nfs_config="${etc_dir}/sysconfig/nfs"
+ nfs_rquotad_config="" # Not use with systemd, restart via service
+
+ case "$nfs_distro_style" in
+ *-redhat | *-suse)
+ : # Defaults only
+ ;;
+ *-debian)
+ nfs_rquotad_service="quotarpc"
+ ;;
+ *)
+ echo "Internal error"
+ exit 1
+ ;;
+ esac
+ ;;
+
+sysvinit-*)
+ # Defaults
+ nfs_service="nfs"
+ nfs_lock_service="AUTO"
+ nfs_mountd_service="AUTO"
+ nfs_status_service="AUTO"
+ nfs_rquotad_service="AUTO"
+ nfs_config="${etc_dir}/sysconfig/nfs"
+ nfs_rquotad_config="$nfs_config"
+
+ case "$nfs_distro_style" in
+ *-redhat)
+ nfs_lock_service="nfslock"
+ ;;
+ *-suse)
+ nfs_service="nfsserver"
+ ;;
+ *-debian)
+ nfs_service="nfs-kernel-server"
+ nfs_config="${etc_dir}/default/nfs-kernel-server"
+ nfs_rquotad_config="${etc_dir}/default/quota"
+ ;;
+ *)
+ echo "Internal error"
+ exit 1
+ ;;
+ esac
+ ;;
+
+*)
+ echo "Internal error"
+ exit 1
+ ;;
+esac
+
+# Override for unit testing
+if [ -z "$PROCFS_PATH" ]; then
+ PROCFS_PATH="/proc"
+fi
+
+##################################################
+
+usage()
+{
+ _c=$(basename "$0")
+ cat <<EOF
+usage: $_c { shutdown | startup }
+ $_c { stop | start } { nfs | nlockmgr }
+ $_c { monitor-list-shares | monitor-post }
+ $_c { register }
+EOF
+ exit 1
+}
+
+##################################################
+
+nfs_load_config()
+{
+ _config="${1:-${nfs_config}}"
+
+ if [ -r "$_config" ]; then
+ . "$_config"
+ fi
+}
+
+##################################################
+
+service_is_auto_started()
+{
+ [ "$1" = "AUTO" ]
+}
+
+service_is_defined()
+{
+ _service="$1"
+
+ [ -n "$_service" ] && ! service_is_auto_started "$_service"
+}
+
+service_if_defined()
+{
+ _service="$1"
+ _action="$2"
+
+ if service_is_defined "$_service"; then
+ service "$_service" "$_action"
+ fi
+}
+
+##################################################
+# Overall NFS service stop and start
+
+nfs_service_stop()
+{
+ service_if_defined "$nfs_rquotad_service" stop
+
+ service "$nfs_service" stop
+
+ service_if_defined "$nfs_lock_service" stop
+}
+
+nfs_service_start()
+{
+ service_if_defined "$nfs_lock_service" start
+
+ service "$nfs_service" start
+
+ service_if_defined "$nfs_rquotad_service" start
+}
+
+##################################################
+# service "stop" and "start" options for restarting
+
+manual_stop()
+{
+ case "$1" in
+ mountd)
+ killall -q -9 rpc.mountd
+ ;;
+ rquotad)
+ killall -q -9 rpc.rquotad
+ ;;
+ status)
+ killall -q -9 rpc.statd
+ ;;
+ *)
+ echo "$0: Internal error - invalid call to manual_stop()"
+ exit 1
+ ;;
+ esac
+}
+
+service_or_manual_stop()
+{
+ _rpc_service="$1"
+ _system_service="$2"
+
+ if service_is_defined "$_system_service"; then
+ service "$_system_service" stop
+ elif service_is_auto_started "$_system_service"; then
+ manual_stop "$_rpc_service"
+ fi
+}
+
+service_stop()
+{
+ _rpc_service="$1"
+
+ case "$_rpc_service" in
+ nfs)
+ echo 0 >"${PROCFS_PATH}/fs/nfsd/threads"
+ nfs_service_stop >/dev/null 2>&1 || true
+ pkill -9 nfsd
+ ;;
+ nlockmgr)
+ if service_is_defined "$nfs_lock_service" ; then
+ service "$nfs_lock_service" stop >/dev/null 2>&1 || true
+ else
+ service "$nfs_service" stop >/dev/null 2>&1 || true
+ fi
+ ;;
+ mountd)
+ service_or_manual_stop "$_rpc_service" "$nfs_mountd_service"
+ ;;
+ rquotad)
+ service_or_manual_stop "$_rpc_service" "$nfs_rquotad_service"
+ ;;
+ status)
+ service_or_manual_stop "$_rpc_service" "$nfs_status_service"
+ ;;
+ *)
+ usage
+ ;;
+ esac
+}
+
+manual_start()
+{
+ case "$1" in
+ mountd)
+ nfs_load_config
+ if [ -z "$RPCMOUNTDOPTS" ]; then
+ RPCMOUNTDOPTS="${MOUNTD_PORT:+-p }$MOUNTD_PORT"
+ fi
+ # shellcheck disable=SC2086
+ rpc.mountd $RPCMOUNTDOPTS
+ ;;
+ rquotad)
+ nfs_load_config "$nfs_rquotad_config"
+ if [ -z "$RPCRQUOTADOPTS" ]; then
+ RPCRQUOTADOPTS="${RQUOTAD_PORT:+-p }$RQUOTAD_PORT"
+ fi
+ # shellcheck disable=SC2086
+ rpc.rquotad $RPCRQUOTADOPTS
+ ;;
+ status)
+ nfs_load_config
+ # Red Hat uses STATDARG, Debian uses STATDOPTS
+ opts="${STATDARG:-${STATDOPTS:-''}}"
+ if [ -z "$opts" ]; then
+ # shellcheck disable=SC2086
+ set -- \
+ ${STATD_HA_CALLOUT:+-H} $STATD_HA_CALLOUT \
+ ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
+ ${STATD_PORT:+-p} $STATD_PORT \
+ ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
+ opts="$*"
+ fi
+ # shellcheck disable=SC2086
+ rpc.statd $opts
+ ;;
+ *)
+ echo "$0: Internal error - invalid call to manual_start()"
+ exit 1
+ ;;
+ esac
+}
+
+service_or_manual_start()
+{
+ _rpc_service="$1"
+ _system_service="$2"
+
+ if service_is_defined "$_system_service"; then
+ service "$_system_service" start
+ elif service_is_auto_started "$_system_service"; then
+ manual_start "$_rpc_service"
+ fi
+}
+
+service_start()
+{
+ _rpc_service="$1"
+
+ case "$_rpc_service" in
+ nfs)
+ nfs_service_start
+ ;;
+ nlockmgr)
+ if service_is_defined "$nfs_lock_service" ; then
+ service "$nfs_lock_service" start
+ else
+ service "$nfs_service" start
+ fi
+ ;;
+ mountd)
+ service_or_manual_start "$_rpc_service" "$nfs_mountd_service"
+ ;;
+ rquotad)
+ service_or_manual_start "$_rpc_service" "$nfs_rquotad_service"
+ ;;
+ status)
+ service_or_manual_start "$_rpc_service" "$nfs_status_service"
+ ;;
+ *)
+ usage
+ ;;
+ esac
+}
+
+##################################################
+# service init startup and final shutdown
+
+nfs_shutdown()
+{
+ nfs_service_stop
+}
+
+nfs_startup()
+{
+ nfs_service_stop || true
+ nfs_service_start
+ _f="${PROCFS_PATH}/sys/net/ipv4/tcp_tw_recycle"
+ if [ -f "$_f" ]; then
+ echo 1 >"$_f"
+ fi
+}
+
+##################################################
+# monitor-post support
+
+nfs_check_thread_count()
+{
+ # Load NFS configuration to get desired number of threads.
+ nfs_load_config
+
+ # If $RPCNFSDCOUNT/$USE_KERNEL_NFSD_NUMBER isn't set then we could
+ # guess the default from the initscript. However, let's just
+ # assume that those using the default don't care about the number
+ # of threads and that they have switched on this feature in error.
+ _configured_threads="${RPCNFSDCOUNT:-${USE_KERNEL_NFSD_NUMBER}}"
+ if [ -z "$_configured_threads" ] && type nfsconf >/dev/null 2>&1; then
+ _configured_threads=$(nfsconf --get nfsd threads) || true
+ fi
+ [ -n "$_configured_threads" ] || return 0
+
+ _threads_file="${PROCFS_PATH}/fs/nfsd/threads"
+
+ # nfsd should be running the configured number of threads. If
+ # there are a different number of threads then tell nfsd the
+ # correct number.
+ read -r _running_threads <"$_threads_file" || {
+ echo "WARNING: Reading \"${_threads_file}\" unexpectedly failed"
+ exit 0
+ }
+
+ # Intentionally not arithmetic comparison - avoids extra errors
+ # when above read fails in an unexpected way...
+ if [ "$_running_threads" != "$_configured_threads" ]; then
+ echo "Attempting to correct number of nfsd threads from ${_running_threads} to ${_configured_threads}"
+ echo "$_configured_threads" >"$_threads_file"
+ fi
+}
+
+##################################################
+# list share directories
+
+nfs_monitor_list_shares()
+{
+ _cache_file="${CTDB_NFS_CALLOUT_STATE_DIR}/list_shares_cache"
+ # -nt operator is well supported in Linux: dash, bash, ksh, ...
+ # shellcheck disable=SC2039,SC3013
+ if [ ! -r "$nfs_exports_file" ] || [ ! -r "$_cache_file" ] ||
+ [ "$nfs_exports_file" -nt "$_cache_file" ]; then
+ mkdir -p "$CTDB_NFS_CALLOUT_STATE_DIR"
+ # We could just use the contents of $nfs_exports_file.
+ # However, let's regard that file as internal to NFS and use
+ # exportfs, which is the public API.
+ if ! _exports=$(exportfs -v); then
+ echo "WARNING: failed to run exportfs to list NFS shares" >&2
+ return
+ fi
+
+ echo "$_exports" |
+ grep '^/' |
+ sed -e 's@[[:space:]][[:space:]]*[^[:space:]()][^[:space:]()]*([^[:space:]()][^[:space:]()]*)$@@' |
+ sort -u >"$_cache_file"
+ fi
+
+ cat "$_cache_file"
+}
+
+##################################################
+
+nfs_register()
+{
+ cat <<EOF
+shutdown
+startup
+stop
+start
+monitor-list-shares
+monitor-post
+EOF
+}
+
+##################################################
+
+case "$1" in
+shutdown)
+ nfs_shutdown
+ ;;
+startup)
+ nfs_startup
+ ;;
+stop)
+ service_stop "$2"
+ ;;
+start)
+ service_start "$2"
+ ;;
+monitor-list-shares)
+ nfs_monitor_list_shares
+ ;;
+monitor-post)
+ nfs_check_thread_count
+ ;;
+register)
+ nfs_register
+ ;;
+monitor-pre | releaseip | takeip | releaseip-pre | takeip-pre)
+ # Not required/implemented
+ :
+ ;;
+*)
+ usage
+ ;;
+esac
diff --git a/ctdb/config/notification.README b/ctdb/config/notification.README
new file mode 100755
index 0000000..16b632f
--- /dev/null
+++ b/ctdb/config/notification.README
@@ -0,0 +1,36 @@
+This directory should contain executable programs ending in ".script"
+to handle CTDB event notifications. The first and only argument
+passed to each program is the event, which is one of:
+
+ init, setup, startup, unhealthy, healthy
+
+An example script that sends SNMP traps for unhealthy/healthy might
+look like this:
+
+ #!/bin/sh
+
+ case "$1" in
+ unhealthy)
+ # Send an SNMP trap saying that the node is unhealthy:
+ snmptrap -m ALL -v 1 -c public 10.1.1.105 ctdb \
+ $(hostname) 0 0 $(date +"%s") ctdb.nodeHealth.0 i 1
+ ;;
+ healthy)
+ # Send an SNMP trap saying that the node is healthy again:
+ snmptrap -m ALL -v 1 -c public 10.1.1.105 ctdb \
+ $(hostname) 0 0 $(date +"%s") ctdb.nodeHealth.0 i 0
+ ;;
+ esac
+
+Alternatively, email could be sent:
+
+ #!/bin/sh
+
+ case "$1" in
+ unhealthy)
+ mail -s "$(hostname) is UNHEALTHY" foo@example.com </dev/null >/dev/null 2>&1
+ ;;
+ healthy)
+ mail -s "$(hostname) is HEALTHY" foo@example.com </dev/null >/dev/null 2>&1
+ ;;
+ esac
diff --git a/ctdb/config/notify.sh b/ctdb/config/notify.sh
new file mode 100755
index 0000000..db69afc
--- /dev/null
+++ b/ctdb/config/notify.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# This is script is invoked from ctdb when certain events happen. See
+# /etc/ctdb/events/notification/README for more details.
+
+d=$(dirname "$0")
+nd="${d}/events/notification"
+
+ok=true
+
+for i in "${nd}/"*.script ; do
+ # Files must be executable
+ [ -x "$i" ] || continue
+
+ # Flag failures
+ "$i" "$1" || ok=false
+done
+
+$ok
diff --git a/ctdb/config/script.options b/ctdb/config/script.options
new file mode 100644
index 0000000..79e82af
--- /dev/null
+++ b/ctdb/config/script.options
@@ -0,0 +1,16 @@
+# For now, use script.options to demonstrate these options. See
+# *.options examples for more specific examples.
+
+#
+# Samba configuration
+#
+
+# 50.samba.options
+# CTDB_SAMBA_SKIP_SHARE_CHECK=yes
+
+#
+# NFS configuration
+#
+
+# 60.nfs.options
+CTDB_RPCINFO_LOCALHOST="127.0.0.1"
diff --git a/ctdb/config/statd-callout b/ctdb/config/statd-callout
new file mode 100755
index 0000000..38c155e
--- /dev/null
+++ b/ctdb/config/statd-callout
@@ -0,0 +1,254 @@
+#!/bin/sh
+
+# This must run as root as CTDB tool commands need to access CTDB socket
+[ "$(id -u)" -eq 0 ] || exec sudo "$0" "$@"
+
+# statd must be configured to use this script as its high availability call-out.
+#
+# In most Linux versions this can be done using something like the following...
+#
+# /etc/sysconfig/nfs (Red Hat) or /etc/default/nfs-common (Debian):
+# NFS_HOSTNAME=myhostname
+# STATD_HOSTNAME="${NFS_HOSTNAME} -H /etc/ctdb/statd-callout"
+#
+# Newer Red Hat Linux variants instead use /etc/nfs.conf:
+# [statd]
+# name = myhostname
+# ha-callout = /etc/ctdb/statd-callout
+
+[ -n "$CTDB_BASE" ] || \
+ CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && echo "$PWD")
+
+. "${CTDB_BASE}/functions"
+
+# Overwrite this so we get some logging
+die ()
+{
+ script_log "statd-callout" "$@"
+ exit 1
+}
+
+# Try different variables to find config file for NFS_HOSTNAME
+load_system_config "nfs" "nfs-common"
+
+# If NFS_HOSTNAME not set then try to pull it out of /etc/nfs.conf
+if [ -z "$NFS_HOSTNAME" ]; then
+ if type nfsconf >/dev/null 2>&1; then
+ NFS_HOSTNAME=$(nfsconf --get statd name)
+ elif type git >/dev/null 2>&1; then
+ # git to the rescue!
+ NFS_HOSTNAME=$(git config --file=/etc/nfs.conf statd.name)
+ fi
+fi
+
+[ -n "$NFS_HOSTNAME" ] || \
+ die "NFS_HOSTNAME is not configured. statd-callout failed"
+
+############################################################
+
+ctdb_setup_state_dir "service" "nfs"
+
+# script_state_dir set by ctdb_setup_state_dir()
+# shellcheck disable=SC2154
+d="${script_state_dir}/statd-callout"
+
+mkdir -p "$d" || die "Failed to create directory \"${d}\""
+cd "$d" || die "Failed to change directory to \"${d}\""
+
+pnn=$(ctdb_get_pnn)
+
+############################################################
+
+send_notifies ()
+{
+ _smnotify="${CTDB_HELPER_BINDIR}/smnotify"
+
+ # State must monotonically increase, across the entire
+ # cluster. Use seconds since epoch and hope the time is in
+ # sync across nodes. Even numbers mean service is shut down,
+ # odd numbers mean service is started.
+
+ # Intentionally round to an even number
+ # shellcheck disable=SC2017
+ _state_even=$(( $(date '+%s') / 2 * 2))
+
+ _prev=""
+ while read _sip _cip ; do
+ # NOTE: Consider optimising smnotify to read all the
+ # data from stdin and then run it in the background.
+
+ # Reset stateval for each serverip
+ if [ "$_sip" != "$_prev" ] ; then
+ _stateval="$_state_even"
+ fi
+
+ # Send notifies for server shutdown
+ "$_smnotify" --client="$_cip" --ip="$_sip" \
+ --server="$_sip" --stateval="$_stateval"
+ "$_smnotify" --client="$_cip" --ip="$_sip" \
+ --server="$NFS_HOSTNAME" --stateval="$_stateval"
+
+ # Send notifies for server startup
+ _stateval=$((_stateval + 1))
+ "$_smnotify" --client="$_cip" --ip="$_sip" \
+ --server="$_sip" --stateval="$_stateval"
+ "$_smnotify" --client="$_cip" --ip="$_sip" \
+ --server="$NFS_HOSTNAME" --stateval="$_stateval"
+ done
+}
+
+delete_records ()
+{
+ while read _sip _cip ; do
+ _key="statd-state@${_sip}@${_cip}"
+ echo "\"${_key}\" \"\""
+ done | $CTDB ptrans "ctdb.tdb"
+}
+
+############################################################
+
+case "$1" in
+ # Keep a single file to keep track of the last "add-client" or
+ # "del-client'. These get pushed to ctdb.tdb during "update",
+ # which will generally be run once each "monitor" cycle. In this
+ # way we avoid scalability problems with flood of persistent
+ # transactions after a "notify" when all the clients re-take their
+ # locks.
+
+ add-client)
+ # statd does not tell us to which IP the client connected so
+ # we must add it to all the IPs that we serve
+ cip="$2"
+ date=$(date '+%s')
+ # x is intentionally ignored
+ # shellcheck disable=SC2034
+ $CTDB ip -X |
+ tail -n +2 |
+ while IFS="|" read x sip node x ; do
+ [ "$node" = "$pnn" ] || continue # not us
+ key="statd-state@${sip}@${cip}"
+ echo "\"${key}\" \"${date}\"" >"$key"
+ done
+ ;;
+
+ del-client)
+ # statd does not tell us from which IP the client disconnected
+ # so we must add it to all the IPs that we serve
+ cip="$2"
+ # x is intentionally ignored
+ # shellcheck disable=SC2034
+ $CTDB ip -X |
+ tail -n +2 |
+ while IFS="|" read x sip node x ; do
+ [ "$node" = "$pnn" ] || continue # not us
+ key="statd-state@${sip}@${cip}"
+ echo "\"${key}\" \"\"" >"$key"
+ done
+ ;;
+
+ update)
+ files=$(echo statd-state@*)
+ if [ "$files" = "statd-state@*" ] ; then
+ # No files!
+ exit 0
+ fi
+ # Filter out lines for any IP addresses that are not currently
+ # hosted public IP addresses.
+ ctdb_ips=$($CTDB ip | tail -n +2)
+ sed_expr=$(echo "$ctdb_ips" |
+ awk -v pnn="$pnn" 'pnn == $2 {
+ ip = $1; gsub(/\./, "\\.", ip);
+ printf "/statd-state@%s@/p\n", ip }')
+ # Intentional multi-word expansion for multiple files
+ # shellcheck disable=SC2086
+ items=$(sed -n "$sed_expr" $files)
+ if [ -n "$items" ] ; then
+ if echo "$items" | $CTDB ptrans "ctdb.tdb" ; then
+ # shellcheck disable=SC2086
+ rm $files
+ fi
+ fi
+ ;;
+
+ notify)
+ # we must restart the lockmanager (on all nodes) so that we get
+ # a clusterwide grace period (so other clients don't take out
+ # conflicting locks through other nodes before all locks have been
+ # reclaimed)
+
+ # we need these settings to make sure that no tcp connections survive
+ # across a very fast failover/failback
+ #echo 10 > /proc/sys/net/ipv4/tcp_fin_timeout
+ #echo 0 > /proc/sys/net/ipv4/tcp_max_tw_buckets
+ #echo 0 > /proc/sys/net/ipv4/tcp_max_orphans
+
+ # Delete the notification list for statd, we don't want it to
+ # ping any clients
+ rm -f /var/lib/nfs/statd/sm/*
+ rm -f /var/lib/nfs/statd/sm.bak/*
+
+ # We must also let some time pass between stopping and
+ # restarting the lock manager. Otherwise there is a window
+ # where the lock manager will respond "strangely" immediately
+ # after restarting it, which causes clients to fail to reclaim
+ # their locks.
+ nfs_callout_init
+ "$CTDB_NFS_CALLOUT" "stop" "nlockmgr" >/dev/null 2>&1
+ sleep 2
+ "$CTDB_NFS_CALLOUT" "start" "nlockmgr" >/dev/null 2>&1
+
+ # we now need to send out additional statd notifications to ensure
+ # that clients understand that the lockmanager has restarted.
+ # we have three cases:
+ # 1, clients that ignore the ip address the stat notification came from
+ # and ONLY care about the 'name' in the notify packet.
+ # these clients ONLY work with lock failover IFF that name
+ # can be resolved into an ipaddress that matches the one used
+ # to mount the share. (==linux clients)
+ # This is handled when starting lockmanager above, but those
+ # packets are sent from the "wrong" ip address, something linux
+ # clients are ok with, buth other clients will barf at.
+ # 2, Some clients only accept statd packets IFF they come from the
+ # 'correct' ip address.
+ # 2a,Send out the notification using the 'correct' ip address and also
+ # specify the 'correct' hostname in the statd packet.
+ # Some clients require both the correct source address and also the
+ # correct name. (these clients also ONLY work if the ip addresses
+ # used to map the share can be resolved into the name returned in
+ # the notify packet.)
+ # 2b,Other clients require that the source ip address of the notify
+ # packet matches the ip address used to take out the lock.
+ # I.e. that the correct source address is used.
+ # These clients also require that the statd notify packet contains
+ # the name as the ip address used when the lock was taken out.
+ #
+ # Both 2a and 2b are commonly used in lockmanagers since they maximize
+ # probability that the client will accept the statd notify packet and
+ # not just ignore it.
+ # For all IPs we serve, collect info and push to the config database
+
+ # Construct a sed expression to take catdb output and produce pairs of:
+ # server-IP client-IP
+ # but only for the server-IPs that are hosted on this node.
+ ctdb_all_ips=$($CTDB ip all | tail -n +2)
+ sed_expr=$(echo "$ctdb_all_ips" |
+ awk -v pnn="$pnn" 'pnn == $2 {
+ ip = $1; gsub(/\./, "\\.", ip);
+ printf "s/^key.*=.*statd-state@\\(%s\\)@\\([^\"]*\\).*/\\1 \\2/p\n", ip }')
+
+ statd_state=$($CTDB catdb ctdb.tdb | sed -n "$sed_expr" | sort)
+ [ -n "$statd_state" ] || exit 0
+
+ echo "$statd_state" | send_notifies
+ echo "$statd_state" | delete_records
+
+ # Remove any stale touch files (i.e. for IPs not currently
+ # hosted on this node and created since the last "update").
+ # There's nothing else we can do with them at this stage.
+ echo "$ctdb_all_ips" |
+ awk -v pnn="$pnn" 'pnn != $2 { print $1 }' |
+ while read sip ; do
+ rm -f "statd-state@${sip}@"*
+ done
+ ;;
+esac