diff options
Diffstat (limited to '')
| -rwxr-xr-x | heartbeat/galera.in | 1097 |
1 files changed, 1097 insertions, 0 deletions
diff --git a/heartbeat/galera.in b/heartbeat/galera.in new file mode 100755 index 0000000..6aed3e4 --- /dev/null +++ b/heartbeat/galera.in @@ -0,0 +1,1097 @@ +#!@BASH_SHELL@ +# +# Copyright (c) 2014 David Vossel <davidvossel@gmail.com> +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# + +## +# README. +# +# This agent only supports being configured as a multistate Promoted +# resource. +# +# Unpromoted vs Promoted role: +# +# During the 'Unpromoted' role, galera instances are in read-only mode and +# will not attempt to connect to the cluster. This role exists only as +# a means to determine which galera instance is the most up-to-date. The +# most up-to-date node will be used to bootstrap a galera cluster that +# has no current members. +# +# The galera instances will only begin to be promoted to the Promoted role +# once all the nodes in the 'wsrep_cluster_address' connection address +# have entered read-only mode. At that point the node containing the +# database that is most current will be promoted to Promoted. Once the first +# Promoted instance bootstraps the galera cluster, the other nodes will be +# promoted to Promoted as well. +# +# Example: Create a galera cluster using nodes rhel7-node1 rhel7-node2 rhel7-node3 +# +# pcs resource create db galera enable_creation=true \ +# wsrep_cluster_address="gcomm://rhel7-auto1,rhel7-auto2,rhel7-auto3" meta promoted-max=3 --promoted +# +# By setting the 'enable_creation' option, the database will be automatically +# generated at startup. The meta attribute 'promoted-max=3' means that all 3 +# nodes listed in the wsrep_cluster_address list will be allowed to connect +# to the galera cluster and perform replication. +# +# NOTE: If you have more nodes in the pacemaker cluster then you wish +# to have in the galera cluster, make sure to use location contraints to prevent +# pacemaker from attempting to place a galera instance on a node that is +# not in the 'wsrep_cluster_address" list. +# +## + +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ "$__OCF_ACTION" != "meta-data" ]; then + . ${OCF_FUNCTIONS_DIR}/mysql-common.sh + NODENAME=$(ocf_attribute_target) +fi + +# It is common for some galera instances to store +# check user that can be used to query status +# in this file +if [ -f "/etc/sysconfig/clustercheck" ]; then + . /etc/sysconfig/clustercheck +elif [ -f "/etc/default/clustercheck" ]; then + . /etc/default/clustercheck +fi + +# Parameter defaults + +OCF_RESKEY_wsrep_cluster_address_default="" +OCF_RESKEY_cluster_host_map_default="" +OCF_RESKEY_check_user_default="" +OCF_RESKEY_check_passwd_default="" +OCF_RESKEY_two_node_mode_default="false" + +: ${OCF_RESKEY_wsrep_cluster_address=${OCF_RESKEY_wsrep_cluster_address_default}} +: ${OCF_RESKEY_cluster_host_map=${OCF_RESKEY_cluster_host_map_default}} +: ${OCF_RESKEY_check_user=${OCF_RESKEY_check_user_default}} +: ${OCF_RESKEY_check_passwd=${OCF_RESKEY_check_passwd_default}} +: ${OCF_RESKEY_two_node_mode=${OCF_RESKEY_two_node_mode_default}} + +####################################################################### +# Defaults: + +OCF_RESKEY_check_passwd_use_empty_default=0 + +: ${OCF_RESKEY_check_passwd_use_empty=${OCF_RESKEY_check_passwd_use_empty_default}} + +####################################################################### + +usage() { + cat <<UEND +usage: $0 (start|stop|validate-all|meta-data|monitor|promote|demote) + +$0 manages a galera Database as an HA resource. + +The 'start' operation starts the database. +The 'stop' operation stops the database. +The 'status' operation reports whether the database is running +The 'monitor' operation reports whether the database seems to be working +The 'promote' operation makes this mysql server run as promoted +The 'demote' operation makes this mysql server run as unpromoted +The 'validate-all' operation reports whether the parameters are valid + +UEND +} + +meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="galera" version="1.0"> +<version>1.0</version> + +<longdesc lang="en"> +Resource script for managing galera database. +</longdesc> +<shortdesc lang="en">Manages a galera instance</shortdesc> +<parameters> + +<parameter name="binary" unique="0" required="0"> +<longdesc lang="en"> +Location of the MySQL server binary +</longdesc> +<shortdesc lang="en">MySQL server binary</shortdesc> +<content type="string" default="${OCF_RESKEY_binary_default}" /> +</parameter> + +<parameter name="client_binary" unique="0" required="0"> +<longdesc lang="en"> +Location of the MySQL client binary +</longdesc> +<shortdesc lang="en">MySQL client binary</shortdesc> +<content type="string" default="${OCF_RESKEY_client_binary_default}" /> +</parameter> + +<parameter name="config" unique="0" required="0"> +<longdesc lang="en"> +Configuration file +</longdesc> +<shortdesc lang="en">MySQL config</shortdesc> +<content type="string" default="${OCF_RESKEY_config_default}" /> +</parameter> + +<parameter name="datadir" unique="0" required="0"> +<longdesc lang="en"> +Directory containing databases +</longdesc> +<shortdesc lang="en">MySQL datadir</shortdesc> +<content type="string" default="${OCF_RESKEY_datadir_default}" /> +</parameter> + +<parameter name="user" unique="0" required="0"> +<longdesc lang="en"> +User running MySQL daemon +</longdesc> +<shortdesc lang="en">MySQL user</shortdesc> +<content type="string" default="${OCF_RESKEY_user_default}" /> +</parameter> + +<parameter name="group" unique="0" required="0"> +<longdesc lang="en"> +Group running MySQL daemon (for logfile and directory permissions) +</longdesc> +<shortdesc lang="en">MySQL group</shortdesc> +<content type="string" default="${OCF_RESKEY_group_default}"/> +</parameter> + +<parameter name="log" unique="0" required="0"> +<longdesc lang="en"> +The logfile to be used for mysqld. +</longdesc> +<shortdesc lang="en">MySQL log file</shortdesc> +<content type="string" default="${OCF_RESKEY_log_default}"/> +</parameter> + +<parameter name="pid" unique="0" required="0"> +<longdesc lang="en"> +The pidfile to be used for mysqld. +</longdesc> +<shortdesc lang="en">MySQL pid file</shortdesc> +<content type="string" default="${OCF_RESKEY_pid_default}"/> +</parameter> + +<parameter name="socket" unique="0" required="0"> +<longdesc lang="en"> +The socket to be used for mysqld. +</longdesc> +<shortdesc lang="en">MySQL socket</shortdesc> +<content type="string" default="${OCF_RESKEY_socket_default}"/> +</parameter> + +<parameter name="enable_creation" unique="0" required="0"> +<longdesc lang="en"> +If the MySQL database does not exist, it will be created +</longdesc> +<shortdesc lang="en">Create the database if it does not exist</shortdesc> +<content type="boolean" default="${OCF_RESKEY_enable_creation_default}"/> +</parameter> + +<parameter name="additional_parameters" unique="0" required="0"> +<longdesc lang="en"> +Additional parameters which are passed to the mysqld on startup. +(e.g. --skip-external-locking or --skip-grant-tables) +</longdesc> +<shortdesc lang="en">Additional parameters to pass to mysqld</shortdesc> +<content type="string" default="${OCF_RESKEY_additional_parameters_default}"/> +</parameter> + + +<parameter name="wsrep_cluster_address" unique="0" required="1"> +<longdesc lang="en"> +The galera cluster address. This takes the form of: +gcomm://node,node,node + +Only nodes present in this node list will be allowed to start a galera instance. +The galera node names listed in this address are expected to match valid +pacemaker node names. If both names need to differ, you must provide a +mapping in option cluster_host_map. +</longdesc> +<shortdesc lang="en">Galera cluster address</shortdesc> +<content type="string" default="${OCF_RESKEY_wsrep_cluster_address_default}"/> +</parameter> + +<parameter name="cluster_host_map" unique="0" required="0"> +<longdesc lang="en"> +A mapping of pacemaker node names to galera node names. + +To be used when both pacemaker and galera names need to differ, +(e.g. when galera names map to IP from a specific network interface) +This takes the form of: +pcmk1:node.1.galera;pcmk2:node.2.galera;pcmk3:node.3.galera + +where the galera resource started on node pcmk1 would be named +node.1.galera in the wsrep_cluster_address +</longdesc> +<shortdesc lang="en">Pacemaker to Galera name mapping</shortdesc> +<content type="string" default="${OCF_RESKEY_cluster_host_map_default}"/> +</parameter> + +<parameter name="check_user" unique="0" required="0"> +<longdesc lang="en"> +Cluster check user. +</longdesc> +<shortdesc lang="en">MySQL test user</shortdesc> +<content type="string" default="${OCF_RESKEY_check_user_default}" /> +</parameter> + +<parameter name="check_passwd" unique="0" required="0"> +<longdesc lang="en"> +Cluster check user password. Empty passwords are ignored unless +the parameter "check_passwd_use_empty" is set to 1. +</longdesc> +<shortdesc lang="en">check password</shortdesc> +<content type="string" default="${OCF_RESKEY_check_passwd_default}" /> +</parameter> + +<parameter name="check_passwd_use_empty" unique="0" required="0"> +<longdesc lang="en"> +Use an empty "check_passwd" password. If this parameter is set to 1, +"check_passwd" will be ignored and an empty password is used +when calling the "mysql" client binary. +</longdesc> +<shortdesc lang="en">check password use empty</shortdesc> +<content type="boolean" default="${OCF_RESKEY_check_passwd_use_empty_default}"/> +</parameter> + +<parameter name="two_node_mode" unique="0" required="0"> +<longdesc lang="en"> +If running in a 2-node pacemaker cluster, rely on pacemaker quorum +to allow automatic recovery even when the other node is unreachable. +Use it with caution! (and fencing) +</longdesc> +<shortdesc lang="en">Special recovery when running on a 2-node cluster</shortdesc> +<content type="boolean" default="${OCF_RESKEY_two_node_mode_default}"/> +</parameter> + +</parameters> + +<actions> +<action name="start" timeout="120s" /> +<action name="stop" timeout="120s" /> +<action name="status" timeout="60s" /> +<action name="monitor" depth="0" timeout="30s" interval="20s" /> +<action name="monitor" role="Promoted" depth="0" timeout="30s" interval="10s" /> +<action name="monitor" role="Unpromoted" depth="0" timeout="30s" interval="30s" /> +<action name="promote" timeout="300s" /> +<action name="demote" timeout="120s" /> +<action name="validate-all" timeout="5s" /> +<action name="meta-data" timeout="5s" /> +</actions> +</resource-agent> +END +} + +get_option_variable() +{ + local key=$1 + + $MYSQL $MYSQL_OPTIONS_CHECK -e "SHOW VARIABLES like '$key';" | tail -1 +} + +get_status_variable() +{ + local key=$1 + + $MYSQL $MYSQL_OPTIONS_CHECK -e "show status like '$key';" | tail -1 +} + +set_bootstrap_node() +{ + local node=$(ocf_attribute_target $1) + + ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -v "true" +} + +clear_bootstrap_node() +{ + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -D +} + +is_bootstrap() +{ + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" --quiet 2>/dev/null + +} + +set_no_grastate() +{ + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" -v "true" +} + +clear_no_grastate() +{ + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" -D +} + +is_no_grastate() +{ + local node=$(ocf_attribute_target $1) + ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" --quiet 2>/dev/null +} + +clear_last_commit() +{ + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -D +} + +set_last_commit() +{ + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -v $1 +} + +get_last_commit() +{ + local node=$(ocf_attribute_target $1) + + if [ -z "$node" ]; then + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null + else + ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null + fi +} + +clear_safe_to_bootstrap() +{ + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" -D +} + +set_safe_to_bootstrap() +{ + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" -v $1 +} + +get_safe_to_bootstrap() +{ + local node=$(ocf_attribute_target $1) + + if [ -z "$node" ]; then + ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" --quiet 2>/dev/null + else + ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" --quiet 2>/dev/null + fi +} + +wait_for_sync() +{ + local state=$(get_status_variable "wsrep_local_state") + + ocf_log info "Waiting for database to sync with the cluster. " + while [ "$state" != "4" ]; do + sleep 1 + state=$(get_status_variable "wsrep_local_state") + done + ocf_log info "Database synced." +} + +is_primary() +{ + cluster_status=$(get_status_variable "wsrep_cluster_status") + if [ "$cluster_status" = "Primary" ]; then + return 0 + fi + + if [ -z "$cluster_status" ]; then + ocf_exit_reason "Unable to retrieve wsrep_cluster_status, verify check_user '$OCF_RESKEY_check_user' has permissions to view status" + else + ocf_log info "Galera instance wsrep_cluster_status=${cluster_status}" + fi + return 1 +} + +is_readonly() +{ + local res=$(get_option_variable "read_only") + + if ! ocf_is_true "$res"; then + return 1 + fi + + cluster_status=$(get_status_variable "wsrep_cluster_status") + if ! [ "$cluster_status" = "Disconnected" ]; then + return 1 + fi + + return 0 +} + +is_two_node_mode_active() +{ + # crm_node or corosync-quorumtool cannot access various corosync + # flags when running inside a bundle, so only count the cluster + # members + ocf_is_true "$OCF_RESKEY_two_node_mode" && crm_mon_no_validation -1X | xmllint --xpath "count(//nodes/node[@type='member'])" - | grep -q -w 2 +} + +is_last_node_in_quorate_partition() +{ + # when a network split occurs in a 2-node cluster, pacemaker + # fences the other node and try to retain quorum. So until + # the fencing is resolved (and the status of the peer node + # is clean), we shouldn't consider ourself quorate. + local partition_members=$(${HA_SBIN_DIR}/crm_node -p | wc -w) + local quorate=$(${HA_SBIN_DIR}/crm_node -q) + local clean_members=$(crm_mon_no_validation -1X | xmllint --xpath 'count(//nodes/node[@type="member" and @unclean="false"])' -) + + [ "$partition_members" = 1 ] && [ "$quorate" = 1 ] && [ "$clean_members" = 2 ] +} + +master_exists() +{ + if [ "$__OCF_ACTION" = "demote" ]; then + # We don't want to detect master instances during demote. + # 1. we could be detecting ourselves as being master, which is no longer the case. + # 2. we could be detecting other master instances that are in the process of shutting down. + # by not detecting other master instances in "demote" we are deferring this check + # to the next recurring monitor operation which will be much more accurate + return 1 + fi + # determine if a master instance is already up and is healthy + ocf_version_cmp "$OCF_RESKEY_crm_feature_set" "3.1.0" + res=$? + if [ -z "$OCF_RESKEY_crm_feature_set" ] || [ $res -eq 2 ]; then + XMLOPT="--output-as=xml" + ocf_version_cmp "$OCF_RESKEY_crm_feature_set" "3.2.0" + if [ $? -eq 1 ]; then + crm_mon_no_validation -1 $XMLOPT >/dev/null 2>&1 + if [ $? -ne 0 ]; then + XMLOPT="--as-xml" + fi + fi + else + XMLOPT="--as-xml" + fi + crm_mon_no_validation -1 $XMLOPT | grep -q -i -E "resource.*id=\"${INSTANCE_ATTR_NAME}\".*role=\"(Promoted|Master)\".*active=\"true\".*orphaned=\"false\".*failed=\"false\"" + return $? +} + +clear_master_score() +{ + local node=$(ocf_attribute_target $1) + if [ -z "$node" ]; then + ocf_promotion_score -D + else + ocf_promotion_score -D -N $node + fi +} + +set_master_score() +{ + local node=$(ocf_attribute_target $1) + + if [ -z "$node" ]; then + ocf_promotion_score -v 100 + else + ocf_promotion_score -N $node -v 100 + fi +} + +promote_everyone() +{ + + for node in $(echo "$OCF_RESKEY_wsrep_cluster_address" | sed 's/gcomm:\/\///g' | tr -d ' ' | tr -s ',' ' '); do + local pcmk_node=$(galera_to_pcmk_name $node) + if [ -z "$pcmk_node" ]; then + ocf_log err "Could not determine pacemaker node from galera name <${node}>." + return + else + node=$pcmk_node + fi + + set_master_score $node + done +} + +greater_than_equal_long() +{ + # there are values we need to compare in this script + # that are too large for shell -gt to process + echo | awk -v n1="$1" -v n2="$2" '{if (n1>=n2) printf ("true"); else printf ("false");}' | grep -q "true" +} + +galera_to_pcmk_name() +{ + local galera=$1 + if [ -z "$OCF_RESKEY_cluster_host_map" ]; then + echo $galera + else + echo "$OCF_RESKEY_cluster_host_map" | tr ';' '\n' | tr -d ' ' | sed 's/:/ /' | awk -F' ' '$2=="'"$galera"'" {print $1;exit}' + fi +} + +pcmk_to_galera_name() +{ + local pcmk=$1 + if [ -z "$OCF_RESKEY_cluster_host_map" ]; then + echo $pcmk + else + echo "$OCF_RESKEY_cluster_host_map" | tr ';' '\n' | tr -d ' ' | sed 's/:/ /' | awk -F' ' '$1=="'"$pcmk"'" {print $2;exit}' + fi +} + + +detect_first_master() +{ + local best_commit=0 + local last_commit=0 + local missing_nodes=0 + local nodes="" + local nodes_recovered="" + local all_nodes + local best_node_gcomm + local best_node + local safe_to_bootstrap + + all_nodes=$(echo "$OCF_RESKEY_wsrep_cluster_address" | sed 's/gcomm:\/\///g' | tr -d ' ' | tr -s ',' ' ') + best_node_gcomm=$(echo "$all_nodes" | sed 's/^.* \(.*\)$/\1/') + best_node=$(galera_to_pcmk_name $best_node_gcomm) + if [ -z "$best_node" ]; then + ocf_log err "Could not determine initial best node from galera name <${best_node_gcomm}>." + return + fi + + # avoid selecting a recovered node as bootstrap if possible + for node in $all_nodes; do + local pcmk_node=$(galera_to_pcmk_name $node) + if [ -z "$pcmk_node" ]; then + ocf_log err "Could not determine pacemaker node from galera name <${node}>." + return + else + node=$pcmk_node + fi + + if is_no_grastate $node; then + nodes_recovered="$nodes_recovered $node" + else + nodes="$nodes $node" + fi + done + + for node in $nodes_recovered $nodes; do + # On clean shutdown, galera sets the last stopped node as 'safe to bootstrap', + # so use this hint when we can + safe_to_bootstrap=$(get_safe_to_bootstrap $node) + + # Special case for 2-node clusters: during a network split, rely on + # pacemaker's quorum to check whether we can restart galera + if [ "$safe_to_bootstrap" != "1" ] && [ "$node" = "$NODENAME" ] && is_two_node_mode_active; then + is_last_node_in_quorate_partition + if [ $? -eq 0 ]; then + ocf_log warn "Survived a split in a 2-node cluster, considering ourselves safe to bootstrap" + safe_to_bootstrap=1 + fi + fi + + if [ "$safe_to_bootstrap" = "1" ]; then + # Galera marked the node as safe to boostrap during shutdown. Let's just + # pick it as our bootstrap node. + ocf_log info "Node <${node}> is marked as safe to bootstrap." + best_node=$node + + # We don't need to wait for the other nodes to report state in this case + missing_nodes=0 + break + fi + + last_commit=$(get_last_commit $node) + + if [ -z "$last_commit" ]; then + ocf_log info "Waiting on node <${node}> to report database status before Master instances can start." + missing_nodes=1 + continue + fi + + # this means -1, or that no commit has occured yet. + if [ "$last_commit" = "18446744073709551615" ]; then + last_commit="0" + fi + + greater_than_equal_long "$last_commit" "$best_commit" + if [ $? -eq 0 ]; then + best_node=$(ocf_attribute_target $node) + best_commit=$last_commit + fi + + done + + if [ $missing_nodes -eq 1 ]; then + return + fi + + ocf_log info "Promoting $best_node to be our bootstrap node" + set_bootstrap_node $best_node + set_master_score $best_node +} + +detect_safe_to_bootstrap() +{ + local safe_to_bootstrap="" + local uuid="" + local seqno="" + + if [ -f ${OCF_RESKEY_datadir}/grastate.dat ]; then + ocf_log info "attempting to read safe_to_bootstrap flag from ${OCF_RESKEY_datadir}/grastate.dat" + safe_to_bootstrap=$(sed -n 's/^safe_to_bootstrap:\s*\(.*\)$/\1/p' < ${OCF_RESKEY_datadir}/grastate.dat) + uuid=$(sed -n 's/^uuid:\s*\(.*\)$/\1/p' < ${OCF_RESKEY_datadir}/grastate.dat) + seqno=$(sed -n 's/^seqno:\s*\(.*\)$/\1/p' < ${OCF_RESKEY_datadir}/grastate.dat) + fi + + if [ -z "$uuid" ] || \ + [ "$uuid" = "00000000-0000-0000-0000-000000000000" ]; then + clear_safe_to_bootstrap + return + fi + if [ "$safe_to_bootstrap" = "1" ]; then + if [ -z "$seqno" ] || [ "$seqno" = "-1" ]; then + clear_safe_to_bootstrap + return + fi + fi + + if [ "$safe_to_bootstrap" = "1" ] || [ "$safe_to_bootstrap" = "0" ]; then + set_safe_to_bootstrap $safe_to_bootstrap + else + clear_safe_to_bootstrap + fi +} + +detect_last_commit() +{ + local last_commit + local recover_args="--defaults-file=$OCF_RESKEY_config \ + --pid-file=$OCF_RESKEY_pid \ + --socket=$OCF_RESKEY_socket \ + --datadir=$OCF_RESKEY_datadir" + local recovery_file_regex='s/.*WSREP\:.*position\s*recovery.*--log_error='\''\([^'\'']*\)'\''.*/\1/p' + local recovered_position_regex='s/.*WSREP\:\s*[R|r]ecovered\s*position.*\:\(.*\)\s*$/\1/p' + + # codership/galera#354 + # Some ungraceful shutdowns can leave an empty gvwstate.dat on + # disk. This will prevent galera to join the cluster if it is + # configured to attempt PC recovery. Removing that file makes the + # node fall back to the normal, unoptimized joining process. + if [ -f ${OCF_RESKEY_datadir}/gvwstate.dat ] && \ + [ ! -s ${OCF_RESKEY_datadir}/gvwstate.dat ]; then + ocf_log warn "empty ${OCF_RESKEY_datadir}/gvwstate.dat detected, removing it to prevent PC recovery failure at next restart" + rm -f ${OCF_RESKEY_datadir}/gvwstate.dat + fi + + ocf_log info "attempting to detect last commit version by reading ${OCF_RESKEY_datadir}/grastate.dat" + last_commit="$(cat ${OCF_RESKEY_datadir}/grastate.dat | sed -n 's/^seqno.\s*\(.*\)\s*$/\1/p')" + if [ -z "$last_commit" ] || [ "$last_commit" = "-1" ]; then + local tmp=$(mktemp) + chown $OCF_RESKEY_user:$OCF_RESKEY_group $tmp + + # if we pass here because grastate.dat doesn't exist, + # try not to bootstrap from this node if possible + if [ ! -f ${OCF_RESKEY_datadir}/grastate.dat ]; then + set_no_grastate + fi + + ocf_log info "now attempting to detect last commit version using 'mysqld_safe --wsrep-recover'" + + $SU - $OCF_RESKEY_user -s /bin/sh -c \ + "${OCF_RESKEY_binary} $recover_args --wsrep-recover --log-error=$tmp 2>/dev/null" + + last_commit="$(cat $tmp | sed -n $recovered_position_regex | tail -1)" + if [ -z "$last_commit" ]; then + # Galera uses InnoDB's 2pc transactions internally. If + # server was stopped in the middle of a replication, the + # recovery may find a "prepared" XA transaction in the + # redo log, and mysql won't recover automatically + + local recovery_file="$(cat $tmp | sed -n $recovery_file_regex)" + if [ -e $recovery_file ]; then + cat $recovery_file | grep -q -E '\[ERROR\]\s+Found\s+[0-9]+\s+prepared\s+transactions!' 2>/dev/null + if [ $? -eq 0 ]; then + # we can only rollback the transaction, but that's OK + # since the DB will get resynchronized anyway + ocf_log warn "local node <${NODENAME}> was not shutdown properly. Rollback stuck transaction with --tc-heuristic-recover" + $SU - $OCF_RESKEY_user -s /bin/sh -c \ + "${OCF_RESKEY_binary} $recover_args --wsrep-recover \ + --tc-heuristic-recover=rollback --log-error=$tmp 2>/dev/null" + + last_commit="$(cat $tmp | sed -n $recovered_position_regex | tail -1)" + if [ ! -z "$last_commit" ]; then + ocf_log warn "State recovered. force SST at next restart for full resynchronization" + rm -f ${OCF_RESKEY_datadir}/grastate.dat + # try not to bootstrap from this node if possible + set_no_grastate + fi + fi + fi + fi + rm -f $tmp + fi + + if [ ! -z "$last_commit" ]; then + ocf_log info "Last commit version found: $last_commit" + set_last_commit $last_commit + return $OCF_SUCCESS + else + ocf_exit_reason "Unable to detect last known write sequence number" + clear_last_commit + return $OCF_ERR_GENERIC + fi +} + +# For galera, promote is really start +galera_promote() +{ + local rc + local extra_opts + local bootstrap + local safe_to_bootstrap + master_exists + if [ $? -eq 0 ]; then + # join without bootstrapping + extra_opts="--wsrep-cluster-address=${OCF_RESKEY_wsrep_cluster_address}" + else + bootstrap=$(is_bootstrap) + + if ocf_is_true $bootstrap; then + # The best node for bootstrapping wasn't cleanly shutdown. Allow + # bootstrapping anyways + if [ "$(get_safe_to_bootstrap)" = "0" ]; then + sed -ie 's/^\(safe_to_bootstrap:\) 0/\1 1/' ${OCF_RESKEY_datadir}/grastate.dat + ocf_log info "safe_to_bootstrap in ${OCF_RESKEY_datadir}/grastate.dat set to 1 on node ${NODENAME}" + fi + ocf_log info "Node <${NODENAME}> is bootstrapping the cluster" + extra_opts="--wsrep-cluster-address=gcomm://" + else + # We are being promoted without having the bootstrap + # attribute in the CIB, which means we are supposed to + # join a cluster; however if we end up here, there is no + # Master remaining right now, which means there is no + # cluster to join anymore. So force a demotion, and and + # let the RA decide later which node should be the next + # bootstrap node. + ocf_log warn "There is no running cluster to join, demoting ourself" + clear_master_score + return $OCF_SUCCESS + fi + fi + + galera_monitor + if [ $? -eq $OCF_RUNNING_MASTER ]; then + if ocf_is_true $bootstrap; then + promote_everyone + clear_bootstrap_node + ocf_log info "boostrap node already up, promoting the rest of the galera instances." + fi + clear_safe_to_bootstrap + clear_last_commit + return $OCF_SUCCESS + fi + + # last commit/safe_to_bootstrap flag are no longer relevant once promoted + clear_last_commit + clear_safe_to_bootstrap + + mysql_common_prepare_dirs + mysql_common_start "$extra_opts" + rc=$? + if [ $rc != $OCF_SUCCESS ]; then + return $rc + fi + + galera_monitor + rc=$? + if [ $rc != $OCF_SUCCESS -a $rc != $OCF_RUNNING_MASTER ]; then + ocf_exit_reason "Failed initial monitor action" + return $rc + fi + + is_readonly + if [ $? -eq 0 ]; then + ocf_exit_reason "Failure. Master instance started in read-only mode, check configuration." + return $OCF_ERR_GENERIC + fi + + is_primary + if [ $? -ne 0 ]; then + ocf_exit_reason "Failure. Master instance started, but is not in Primary mode." + return $OCF_ERR_GENERIC + fi + + if ocf_is_true $bootstrap; then + promote_everyone + clear_bootstrap_node + # clear attribute no-grastate. if last shutdown was + # not clean, we cannot be extra-cautious by requesting a SST + # since this is the bootstrap node + clear_no_grastate + ocf_log info "Bootstrap complete, promoting the rest of the galera instances." + else + # if this is not the bootstrap node, make sure this instance + # syncs with the rest of the cluster before promotion returns. + wait_for_sync + # sync is done, clear info about last startup + clear_no_grastate + fi + + ocf_log info "Galera started" + return $OCF_SUCCESS +} + +galera_demote() +{ + mysql_common_stop + rc=$? + if [ $rc -ne $OCF_SUCCESS ] && [ $rc -ne $OCF_NOT_RUNNING ]; then + ocf_exit_reason "Failed to stop Master galera instance during demotion to Master" + return $rc + fi + + # if this node was previously a bootstrap node, that is no longer the case. + clear_bootstrap_node + clear_last_commit + clear_no_grastate + clear_safe_to_bootstrap + + # Clear master score here rather than letting pacemaker do so once + # demote finishes. This way a promote cannot take place right + # after this demote even if pacemaker is requested to do so. It + # will first have to run a start/monitor op, to reprobe the state + # of the other galera nodes and act accordingly. + clear_master_score + + # record last commit for next promotion + detect_safe_to_bootstrap + detect_last_commit + rc=$? + return $rc +} + +galera_start() +{ + local rc + local galera_node + + galera_node=$(pcmk_to_galera_name $NODENAME) + if [ -z "$galera_node" ]; then + ocf_exit_reason "Could not determine galera name from pacemaker node <${NODENAME}>." + return $OCF_ERR_CONFIGURED + fi + + echo $OCF_RESKEY_wsrep_cluster_address | grep -q -F $galera_node + if [ $? -ne 0 ]; then + ocf_exit_reason "local node <${NODENAME}> (galera node <${galera_node}>) must be a member of the wsrep_cluster_address <${OCF_RESKEY_wsrep_cluster_address}> to start this galera instance" + return $OCF_ERR_CONFIGURED + fi + + galera_monitor + if [ $? -eq $OCF_RUNNING_MASTER ]; then + ocf_exit_reason "master galera instance started outside of the cluster's control" + return $OCF_ERR_GENERIC + fi + + mysql_common_prepare_dirs + + detect_safe_to_bootstrap + detect_last_commit + rc=$? + if [ $rc -ne $OCF_SUCCESS ]; then + return $rc + fi + + master_exists + if [ $? -eq 0 ]; then + ocf_log info "Master instances are already up, setting master score so this instance will join galera cluster." + set_master_score $NODENAME + else + clear_master_score + detect_first_master + fi + + return $OCF_SUCCESS +} + +galera_monitor() +{ + local rc + local galera_node + local status_loglevel="err" + + # Set loglevel to info during probe + if ocf_is_probe; then + status_loglevel="info" + fi + + mysql_common_status $status_loglevel + rc=$? + + if [ $rc -eq $OCF_NOT_RUNNING ]; then + last_commit=$(get_last_commit $node) + if [ -n "$last_commit" ]; then + # if last commit is set, this instance is considered started in slave mode + rc=$OCF_SUCCESS + master_exists + if [ $? -ne 0 ]; then + detect_first_master + else + # a master instance exists and is healthy, promote this + # local read only instance + # so it can join the master galera cluster. + set_master_score + fi + fi + return $rc + elif [ $rc -ne $OCF_SUCCESS ]; then + return $rc + fi + + # if we make it here, mysql is running. Check cluster status now. + galera_node=$(pcmk_to_galera_name $NODENAME) + if [ -z "$galera_node" ]; then + ocf_exit_reason "Could not determine galera name from pacemaker node <${NODENAME}>." + return $OCF_ERR_CONFIGURED + fi + + echo $OCF_RESKEY_wsrep_cluster_address | grep -q -F $galera_node + if [ $? -ne 0 ]; then + ocf_exit_reason "local node <${NODENAME}> (galera node <${galera_node}>) is started, but is not a member of the wsrep_cluster_address <${OCF_RESKEY_wsrep_cluster_address}>" + return $OCF_ERR_GENERIC + fi + + is_primary + if [ $? -eq 0 ]; then + + if ocf_is_probe; then + # restore master score during probe + # if we detect this is a master instance + set_master_score + fi + rc=$OCF_RUNNING_MASTER + else + ocf_exit_reason "local node <${NODENAME}> is started, but not in primary mode. Unknown state." + rc=$OCF_ERR_GENERIC + fi + + return $rc +} + +galera_stop() +{ + local rc + # make sure the process is stopped + mysql_common_stop + rc=$1 + + clear_safe_to_bootstrap + clear_last_commit + clear_master_score + clear_bootstrap_node + clear_no_grastate + return $rc +} + +galera_validate() +{ + if [ "$OCF_CHECK_LEVEL" -eq 10 ]; then + if ! ocf_is_ms; then + ocf_exit_reason "Galera must be configured as a multistate Master/Slave resource." + return $OCF_ERR_CONFIGURED + fi + fi + + if [ -z "$OCF_RESKEY_wsrep_cluster_address" ]; then + ocf_exit_reason "Galera must be configured with a wsrep_cluster_address value." + return $OCF_ERR_CONFIGURED + fi + + mysql_common_validate +} + +case "$1" in + meta-data) meta_data + exit $OCF_SUCCESS;; + usage|help) usage + exit $OCF_SUCCESS;; +esac + +[ "$__OCF_ACTION" = "start" ] && OCF_CHECK_LEVEL=10 +galera_validate +rc=$? +LSB_STATUS_STOPPED=3 +if [ $rc -ne 0 ]; then + case "$1" in + stop) exit $OCF_SUCCESS;; + monitor) exit $OCF_NOT_RUNNING;; + status) exit $LSB_STATUS_STOPPED;; + *) exit $rc;; + esac +fi + +if [ -z "${OCF_RESKEY_check_passwd}" ]; then + # This value is automatically sourced from /etc/sysconfig/checkcluster if available + OCF_RESKEY_check_passwd=${MYSQL_PASSWORD} +fi +if [ -z "${OCF_RESKEY_check_user}" ]; then + # This value is automatically sourced from /etc/sysconfig/checkcluster if available + OCF_RESKEY_check_user=${MYSQL_USERNAME} +fi +: ${OCF_RESKEY_check_user="root"} + +MYSQL_OPTIONS_CHECK="-nNE --user=${OCF_RESKEY_check_user}" + +if ocf_is_true "${OCF_RESKEY_check_passwd_use_empty}"; then + MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK --password=" +elif [ -n "${OCF_RESKEY_check_passwd}" ]; then + MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK --password=${OCF_RESKEY_check_passwd}" +fi + +# This value is automatically sourced from /etc/sysconfig/checkcluster if available +if [ -n "${MYSQL_HOST}" ]; then + MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK -h ${MYSQL_HOST}" +fi + +# This value is automatically sourced from /etc/sysconfig/checkcluster if available +if [ -n "${MYSQL_PORT}" ]; then + MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK -P ${MYSQL_PORT}" +fi + + + +# What kind of method was invoked? +case "$1" in + start) galera_start;; + stop) galera_stop;; + status) mysql_common_status err;; + monitor) galera_monitor;; + promote) galera_promote;; + demote) galera_demote;; + validate-all) exit $OCF_SUCCESS;; + + *) usage + exit $OCF_ERR_UNIMPLEMENTED;; +esac + +# vi:sw=4:ts=4:et: |