Adding upstream version 1:4.13.0.upstream/1%4.13.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1097 insertions, 0 deletions

diff --git a/heartbeat/galera.in b/heartbeat/galera.in
new file mode 100755
index 0000000..6aed3e4
--- /dev/null
+++ b/heartbeat/galera.in
@@ -0,0 +1,1097 @@
+#!@BASH_SHELL@
+#
+# Copyright (c) 2014 David Vossel <davidvossel@gmail.com>
+#                    All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+
+##
+# README.
+# 
+# This agent only supports being configured as a multistate Promoted
+# resource.
+#
+# Unpromoted vs Promoted role:
+#
+# During the 'Unpromoted' role, galera instances are in read-only mode and
+# will not attempt to connect to the cluster. This role exists only as
+# a means to determine which galera instance is the most up-to-date. The
+# most up-to-date node will be used to bootstrap a galera cluster that
+# has no current members.
+#
+# The galera instances will only begin to be promoted to the Promoted role
+# once all the nodes in the 'wsrep_cluster_address' connection address
+# have entered read-only mode. At that point the node containing the
+# database that is most current will be promoted to Promoted. Once the first
+# Promoted instance bootstraps the galera cluster, the other nodes will be
+# promoted to Promoted as well.
+#
+# Example: Create a galera cluster using nodes rhel7-node1 rhel7-node2 rhel7-node3
+#
+# pcs resource create db galera enable_creation=true \
+# wsrep_cluster_address="gcomm://rhel7-auto1,rhel7-auto2,rhel7-auto3" meta promoted-max=3 --promoted
+#
+# By setting the 'enable_creation' option, the database will be automatically 
+# generated at startup. The meta attribute 'promoted-max=3' means that all 3
+# nodes listed in the wsrep_cluster_address list will be allowed to connect
+# to the galera cluster and perform replication.
+#
+# NOTE: If you have more nodes in the pacemaker cluster then you wish
+# to have in the galera cluster, make sure to use location contraints to prevent
+# pacemaker from attempting to place a galera instance on a node that is
+# not in the 'wsrep_cluster_address" list. 
+#
+##
+
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+if [ "$__OCF_ACTION" != "meta-data" ]; then
+    . ${OCF_FUNCTIONS_DIR}/mysql-common.sh
+    NODENAME=$(ocf_attribute_target)
+fi
+
+# It is common for some galera instances to store
+# check user that can be used to query status
+# in this file
+if [ -f "/etc/sysconfig/clustercheck" ]; then
+    . /etc/sysconfig/clustercheck
+elif [ -f "/etc/default/clustercheck" ]; then
+    . /etc/default/clustercheck
+fi
+
+# Parameter defaults
+
+OCF_RESKEY_wsrep_cluster_address_default=""
+OCF_RESKEY_cluster_host_map_default=""
+OCF_RESKEY_check_user_default=""
+OCF_RESKEY_check_passwd_default=""
+OCF_RESKEY_two_node_mode_default="false"
+
+: ${OCF_RESKEY_wsrep_cluster_address=${OCF_RESKEY_wsrep_cluster_address_default}}
+: ${OCF_RESKEY_cluster_host_map=${OCF_RESKEY_cluster_host_map_default}}
+: ${OCF_RESKEY_check_user=${OCF_RESKEY_check_user_default}}
+: ${OCF_RESKEY_check_passwd=${OCF_RESKEY_check_passwd_default}}
+: ${OCF_RESKEY_two_node_mode=${OCF_RESKEY_two_node_mode_default}}
+
+#######################################################################
+# Defaults:
+
+OCF_RESKEY_check_passwd_use_empty_default=0
+
+: ${OCF_RESKEY_check_passwd_use_empty=${OCF_RESKEY_check_passwd_use_empty_default}}
+
+#######################################################################
+
+usage() {
+  cat <<UEND
+usage: $0 (start|stop|validate-all|meta-data|monitor|promote|demote)
+
+$0 manages a galera Database as an HA resource.
+
+The 'start' operation starts the database.
+The 'stop' operation stops the database.
+The 'status' operation reports whether the database is running
+The 'monitor' operation reports whether the database seems to be working
+The 'promote' operation makes this mysql server run as promoted
+The 'demote' operation makes this mysql server run as unpromoted
+The 'validate-all' operation reports whether the parameters are valid
+
+UEND
+}
+
+meta_data() {
+   cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="galera" version="1.0">
+<version>1.0</version>
+
+<longdesc lang="en">
+Resource script for managing galera database.
+</longdesc>
+<shortdesc lang="en">Manages a galera instance</shortdesc>
+<parameters>
+
+<parameter name="binary" unique="0" required="0">
+<longdesc lang="en">
+Location of the MySQL server binary
+</longdesc>
+<shortdesc lang="en">MySQL server binary</shortdesc>
+<content type="string" default="${OCF_RESKEY_binary_default}" />
+</parameter>
+
+<parameter name="client_binary" unique="0" required="0">
+<longdesc lang="en">
+Location of the MySQL client binary
+</longdesc>
+<shortdesc lang="en">MySQL client binary</shortdesc>
+<content type="string" default="${OCF_RESKEY_client_binary_default}" />
+</parameter>
+
+<parameter name="config" unique="0" required="0">
+<longdesc lang="en">
+Configuration file
+</longdesc>
+<shortdesc lang="en">MySQL config</shortdesc>
+<content type="string" default="${OCF_RESKEY_config_default}" />
+</parameter>
+
+<parameter name="datadir" unique="0" required="0">
+<longdesc lang="en">
+Directory containing databases
+</longdesc>
+<shortdesc lang="en">MySQL datadir</shortdesc>
+<content type="string" default="${OCF_RESKEY_datadir_default}" />
+</parameter>
+
+<parameter name="user" unique="0" required="0">
+<longdesc lang="en">
+User running MySQL daemon
+</longdesc>
+<shortdesc lang="en">MySQL user</shortdesc>
+<content type="string" default="${OCF_RESKEY_user_default}" />
+</parameter>
+
+<parameter name="group" unique="0" required="0">
+<longdesc lang="en">
+Group running MySQL daemon (for logfile and directory permissions)
+</longdesc>
+<shortdesc lang="en">MySQL group</shortdesc>
+<content type="string" default="${OCF_RESKEY_group_default}"/>
+</parameter>
+
+<parameter name="log" unique="0" required="0">
+<longdesc lang="en">
+The logfile to be used for mysqld.
+</longdesc>
+<shortdesc lang="en">MySQL log file</shortdesc>
+<content type="string" default="${OCF_RESKEY_log_default}"/>
+</parameter>
+
+<parameter name="pid" unique="0" required="0">
+<longdesc lang="en">
+The pidfile to be used for mysqld.
+</longdesc>
+<shortdesc lang="en">MySQL pid file</shortdesc>
+<content type="string" default="${OCF_RESKEY_pid_default}"/>
+</parameter>
+
+<parameter name="socket" unique="0" required="0">
+<longdesc lang="en">
+The socket to be used for mysqld.
+</longdesc>
+<shortdesc lang="en">MySQL socket</shortdesc>
+<content type="string" default="${OCF_RESKEY_socket_default}"/>
+</parameter>
+
+<parameter name="enable_creation" unique="0" required="0">
+<longdesc lang="en">
+If the MySQL database does not exist, it will be created
+</longdesc>
+<shortdesc lang="en">Create the database if it does not exist</shortdesc>
+<content type="boolean" default="${OCF_RESKEY_enable_creation_default}"/>
+</parameter>
+
+<parameter name="additional_parameters" unique="0" required="0">
+<longdesc lang="en">
+Additional parameters which are passed to the mysqld on startup.
+(e.g. --skip-external-locking or --skip-grant-tables)
+</longdesc>
+<shortdesc lang="en">Additional parameters to pass to mysqld</shortdesc>
+<content type="string" default="${OCF_RESKEY_additional_parameters_default}"/>
+</parameter>
+
+
+<parameter name="wsrep_cluster_address" unique="0" required="1">
+<longdesc lang="en">
+The galera cluster address. This takes the form of:
+gcomm://node,node,node
+
+Only nodes present in this node list will be allowed to start a galera instance.
+The galera node names listed in this address are expected to match valid
+pacemaker node names. If both names need to differ, you must provide a
+mapping in option cluster_host_map.
+</longdesc>
+<shortdesc lang="en">Galera cluster address</shortdesc>
+<content type="string" default="${OCF_RESKEY_wsrep_cluster_address_default}"/>
+</parameter>
+
+<parameter name="cluster_host_map" unique="0" required="0">
+<longdesc lang="en">
+A mapping of pacemaker node names to galera node names.
+
+To be used when both pacemaker and galera names need to differ,
+(e.g. when galera names map to IP from a specific network interface)
+This takes the form of:
+pcmk1:node.1.galera;pcmk2:node.2.galera;pcmk3:node.3.galera
+
+where the galera resource started on node pcmk1 would be named
+node.1.galera in the wsrep_cluster_address
+</longdesc>
+<shortdesc lang="en">Pacemaker to Galera name mapping</shortdesc>
+<content type="string" default="${OCF_RESKEY_cluster_host_map_default}"/>
+</parameter>
+
+<parameter name="check_user" unique="0" required="0">
+<longdesc lang="en">
+Cluster check user.
+</longdesc>
+<shortdesc lang="en">MySQL test user</shortdesc>
+<content type="string" default="${OCF_RESKEY_check_user_default}" />
+</parameter>
+
+<parameter name="check_passwd" unique="0" required="0">
+<longdesc lang="en">
+Cluster check user password. Empty passwords are ignored unless
+the parameter "check_passwd_use_empty" is set to 1.
+</longdesc>
+<shortdesc lang="en">check password</shortdesc>
+<content type="string" default="${OCF_RESKEY_check_passwd_default}" />
+</parameter>
+
+<parameter name="check_passwd_use_empty" unique="0" required="0">
+<longdesc lang="en">
+Use an empty "check_passwd" password. If this parameter is set to 1,
+"check_passwd" will be ignored and an empty password is used
+when calling the "mysql" client binary.
+</longdesc>
+<shortdesc lang="en">check password use empty</shortdesc>
+<content type="boolean" default="${OCF_RESKEY_check_passwd_use_empty_default}"/>
+</parameter>
+
+<parameter name="two_node_mode" unique="0" required="0">
+<longdesc lang="en">
+If running in a 2-node pacemaker cluster, rely on pacemaker quorum
+to allow automatic recovery even when the other node is unreachable.
+Use it with caution! (and fencing)
+</longdesc>
+<shortdesc lang="en">Special recovery when running on a 2-node cluster</shortdesc>
+<content type="boolean" default="${OCF_RESKEY_two_node_mode_default}"/>
+</parameter>
+
+</parameters>
+
+<actions>
+<action name="start" timeout="120s" />
+<action name="stop" timeout="120s" />
+<action name="status" timeout="60s" />
+<action name="monitor" depth="0" timeout="30s" interval="20s" />
+<action name="monitor" role="Promoted" depth="0" timeout="30s" interval="10s" />
+<action name="monitor" role="Unpromoted" depth="0" timeout="30s" interval="30s" />
+<action name="promote" timeout="300s" />
+<action name="demote" timeout="120s" />
+<action name="validate-all" timeout="5s" />
+<action name="meta-data" timeout="5s" />
+</actions>
+</resource-agent>
+END
+}
+
+get_option_variable()
+{
+    local key=$1
+
+    $MYSQL $MYSQL_OPTIONS_CHECK  -e "SHOW VARIABLES like '$key';" | tail -1
+}
+
+get_status_variable()
+{
+    local key=$1
+
+    $MYSQL $MYSQL_OPTIONS_CHECK -e "show status like '$key';" | tail -1
+}
+
+set_bootstrap_node()
+{
+    local node=$(ocf_attribute_target $1)
+
+    ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -v "true"
+}
+
+clear_bootstrap_node()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -D
+}
+
+is_bootstrap()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" --quiet 2>/dev/null
+
+}
+
+set_no_grastate()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" -v "true"
+}
+
+clear_no_grastate()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" -D
+}
+
+is_no_grastate()
+{
+    local node=$(ocf_attribute_target $1)
+    ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" --quiet 2>/dev/null
+}
+
+clear_last_commit()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -D
+}
+
+set_last_commit()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -v $1
+}
+
+get_last_commit()
+{
+    local node=$(ocf_attribute_target $1)
+
+    if [ -z "$node" ]; then
+       ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null
+    else 
+       ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null
+    fi
+}
+
+clear_safe_to_bootstrap()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" -D
+}
+
+set_safe_to_bootstrap()
+{
+    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" -v $1
+}
+
+get_safe_to_bootstrap()
+{
+    local node=$(ocf_attribute_target $1)
+
+    if [ -z "$node" ]; then
+        ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" --quiet 2>/dev/null
+    else
+        ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" --quiet 2>/dev/null
+    fi
+}
+
+wait_for_sync()
+{
+    local state=$(get_status_variable "wsrep_local_state")
+
+    ocf_log info "Waiting for database to sync with the cluster. "
+    while [ "$state" != "4" ]; do
+        sleep 1
+        state=$(get_status_variable "wsrep_local_state")
+    done
+    ocf_log info "Database synced."
+}
+
+is_primary()
+{
+    cluster_status=$(get_status_variable "wsrep_cluster_status")
+    if [ "$cluster_status" = "Primary" ]; then
+        return 0
+    fi
+
+    if [ -z "$cluster_status" ]; then
+        ocf_exit_reason "Unable to retrieve wsrep_cluster_status, verify check_user '$OCF_RESKEY_check_user' has permissions to view status"
+    else
+        ocf_log info "Galera instance wsrep_cluster_status=${cluster_status}"
+    fi
+    return 1
+}
+
+is_readonly()
+{
+    local res=$(get_option_variable "read_only")
+
+    if ! ocf_is_true "$res"; then
+        return 1
+    fi
+
+    cluster_status=$(get_status_variable "wsrep_cluster_status")
+    if ! [ "$cluster_status" = "Disconnected" ]; then
+        return 1
+    fi
+
+    return 0
+}
+
+is_two_node_mode_active()
+{
+    # crm_node or corosync-quorumtool cannot access various corosync
+    # flags when running inside a bundle, so only count the cluster
+    # members
+    ocf_is_true "$OCF_RESKEY_two_node_mode" && crm_mon_no_validation -1X | xmllint --xpath "count(//nodes/node[@type='member'])" - | grep -q -w 2
+}
+
+is_last_node_in_quorate_partition()
+{
+    # when a network split occurs in a 2-node cluster, pacemaker
+    # fences the other node and try to retain quorum. So until
+    # the fencing is resolved (and the status of the peer node
+    # is clean), we shouldn't consider ourself quorate.
+    local partition_members=$(${HA_SBIN_DIR}/crm_node -p | wc -w)
+    local quorate=$(${HA_SBIN_DIR}/crm_node -q)
+    local clean_members=$(crm_mon_no_validation -1X | xmllint --xpath 'count(//nodes/node[@type="member" and @unclean="false"])' -)
+
+    [ "$partition_members" = 1 ] && [ "$quorate" = 1 ] && [ "$clean_members" = 2 ]
+}
+
+master_exists()
+{
+    if [ "$__OCF_ACTION" = "demote" ]; then
+        # We don't want to detect master instances during demote.
+        # 1. we could be detecting ourselves as being master, which is no longer the case.
+        # 2. we could be detecting other master instances that are in the process of shutting down.
+        # by not detecting other master instances in "demote" we are deferring this check
+        # to the next recurring monitor operation which will be much more accurate
+        return 1
+    fi
+    # determine if a master instance is already up and is healthy
+    ocf_version_cmp "$OCF_RESKEY_crm_feature_set" "3.1.0"
+    res=$?
+    if [ -z "$OCF_RESKEY_crm_feature_set" ] || [ $res -eq 2 ]; then
+        XMLOPT="--output-as=xml"
+        ocf_version_cmp "$OCF_RESKEY_crm_feature_set" "3.2.0"
+        if [ $? -eq 1 ]; then
+            crm_mon_no_validation -1 $XMLOPT >/dev/null 2>&1
+            if [ $? -ne 0 ]; then
+                XMLOPT="--as-xml"
+            fi
+        fi
+    else
+        XMLOPT="--as-xml"
+    fi
+    crm_mon_no_validation -1 $XMLOPT | grep -q -i -E "resource.*id=\"${INSTANCE_ATTR_NAME}\".*role=\"(Promoted|Master)\".*active=\"true\".*orphaned=\"false\".*failed=\"false\""
+    return $?
+}
+
+clear_master_score()
+{
+    local node=$(ocf_attribute_target $1)
+    if [ -z "$node" ]; then
+        ocf_promotion_score -D
+    else 
+        ocf_promotion_score -D -N $node
+    fi
+}
+
+set_master_score()
+{
+    local node=$(ocf_attribute_target $1)
+
+    if [ -z "$node" ]; then
+        ocf_promotion_score -v 100
+    else 
+        ocf_promotion_score -N $node -v 100
+    fi
+}
+
+promote_everyone()
+{
+
+    for node in $(echo "$OCF_RESKEY_wsrep_cluster_address" | sed 's/gcomm:\/\///g' | tr -d ' ' | tr -s ',' ' '); do
+        local pcmk_node=$(galera_to_pcmk_name $node)
+        if [ -z "$pcmk_node" ]; then
+            ocf_log err "Could not determine pacemaker node from galera name <${node}>."
+            return
+        else
+            node=$pcmk_node
+        fi
+
+        set_master_score $node
+    done
+}
+
+greater_than_equal_long()
+{
+    # there are values we need to compare in this script
+    # that are too large for shell -gt to process
+    echo | awk -v n1="$1" -v n2="$2"  '{if (n1>=n2) printf ("true"); else printf ("false");}' |  grep -q "true"
+}
+
+galera_to_pcmk_name()
+{
+    local galera=$1
+    if [ -z "$OCF_RESKEY_cluster_host_map" ]; then
+        echo $galera
+    else
+        echo "$OCF_RESKEY_cluster_host_map" | tr ';' '\n' | tr -d ' ' | sed 's/:/ /' | awk -F' ' '$2=="'"$galera"'" {print $1;exit}'
+    fi
+}
+
+pcmk_to_galera_name()
+{
+    local pcmk=$1
+    if [ -z "$OCF_RESKEY_cluster_host_map" ]; then
+        echo $pcmk
+    else
+        echo "$OCF_RESKEY_cluster_host_map" | tr ';' '\n' | tr -d ' ' | sed 's/:/ /' | awk -F' ' '$1=="'"$pcmk"'" {print $2;exit}'
+    fi
+}
+
+
+detect_first_master()
+{
+    local best_commit=0
+    local last_commit=0
+    local missing_nodes=0
+    local nodes=""
+    local nodes_recovered=""
+    local all_nodes
+    local best_node_gcomm
+    local best_node
+    local safe_to_bootstrap
+
+    all_nodes=$(echo "$OCF_RESKEY_wsrep_cluster_address" | sed 's/gcomm:\/\///g' | tr -d ' ' | tr -s ',' ' ')
+    best_node_gcomm=$(echo "$all_nodes" | sed 's/^.* \(.*\)$/\1/')
+    best_node=$(galera_to_pcmk_name $best_node_gcomm)
+    if [ -z "$best_node" ]; then
+        ocf_log err "Could not determine initial best node from galera name <${best_node_gcomm}>."
+        return
+    fi
+
+    # avoid selecting a recovered node as bootstrap if possible
+    for node in $all_nodes; do
+        local pcmk_node=$(galera_to_pcmk_name $node)
+        if [ -z "$pcmk_node" ]; then
+            ocf_log err "Could not determine pacemaker node from galera name <${node}>."
+            return
+        else
+            node=$pcmk_node
+        fi
+
+        if is_no_grastate $node; then
+            nodes_recovered="$nodes_recovered $node"
+        else
+            nodes="$nodes $node"
+        fi
+    done
+
+    for node in $nodes_recovered $nodes; do
+        # On clean shutdown, galera sets the last stopped node as 'safe to bootstrap',
+        # so use this hint when we can
+        safe_to_bootstrap=$(get_safe_to_bootstrap $node)
+
+        # Special case for 2-node clusters: during a network split, rely on
+        # pacemaker's quorum to check whether we can restart galera
+        if [ "$safe_to_bootstrap" != "1" ] && [ "$node" = "$NODENAME" ] && is_two_node_mode_active; then
+            is_last_node_in_quorate_partition
+            if [ $? -eq 0 ]; then
+                ocf_log warn "Survived a split in a 2-node cluster, considering ourselves safe to bootstrap"
+                safe_to_bootstrap=1
+            fi
+        fi
+
+        if [ "$safe_to_bootstrap" = "1" ]; then
+            # Galera marked the node as safe to boostrap during shutdown. Let's just
+            # pick it as our bootstrap node.
+            ocf_log info "Node <${node}> is marked as safe to bootstrap."
+            best_node=$node
+
+            # We don't need to wait for the other nodes to report state in this case
+            missing_nodes=0
+            break
+        fi
+
+        last_commit=$(get_last_commit $node)
+
+        if [ -z "$last_commit" ]; then
+            ocf_log info "Waiting on node <${node}> to report database status before Master instances can start."
+            missing_nodes=1
+            continue
+        fi
+
+        # this means -1, or that no commit has occured yet.
+        if [ "$last_commit" = "18446744073709551615" ]; then
+            last_commit="0"
+        fi
+
+        greater_than_equal_long "$last_commit" "$best_commit"
+        if [ $? -eq 0 ]; then
+            best_node=$(ocf_attribute_target $node)
+            best_commit=$last_commit
+        fi
+
+    done
+
+    if [ $missing_nodes -eq 1 ]; then
+        return
+    fi
+
+    ocf_log info "Promoting $best_node to be our bootstrap node"
+    set_bootstrap_node $best_node
+    set_master_score $best_node
+}
+
+detect_safe_to_bootstrap()
+{
+    local safe_to_bootstrap=""
+    local uuid=""
+    local seqno=""
+
+    if [ -f ${OCF_RESKEY_datadir}/grastate.dat ]; then
+        ocf_log info "attempting to read safe_to_bootstrap flag from ${OCF_RESKEY_datadir}/grastate.dat"
+        safe_to_bootstrap=$(sed -n 's/^safe_to_bootstrap:\s*\(.*\)$/\1/p' < ${OCF_RESKEY_datadir}/grastate.dat)
+        uuid=$(sed -n 's/^uuid:\s*\(.*\)$/\1/p' < ${OCF_RESKEY_datadir}/grastate.dat)
+        seqno=$(sed -n 's/^seqno:\s*\(.*\)$/\1/p' < ${OCF_RESKEY_datadir}/grastate.dat)
+    fi
+
+    if [ -z "$uuid" ] || \
+       [ "$uuid" = "00000000-0000-0000-0000-000000000000" ]; then
+        clear_safe_to_bootstrap
+        return
+    fi
+    if [ "$safe_to_bootstrap" = "1" ]; then
+        if [ -z "$seqno" ] || [ "$seqno" = "-1" ]; then
+            clear_safe_to_bootstrap
+            return
+        fi
+    fi
+
+    if [ "$safe_to_bootstrap" = "1" ] || [ "$safe_to_bootstrap" = "0" ]; then
+        set_safe_to_bootstrap $safe_to_bootstrap
+    else
+        clear_safe_to_bootstrap
+    fi
+}
+
+detect_last_commit()
+{
+    local last_commit
+    local recover_args="--defaults-file=$OCF_RESKEY_config \
+                        --pid-file=$OCF_RESKEY_pid \
+                        --socket=$OCF_RESKEY_socket \
+                        --datadir=$OCF_RESKEY_datadir"
+    local recovery_file_regex='s/.*WSREP\:.*position\s*recovery.*--log_error='\''\([^'\'']*\)'\''.*/\1/p'
+    local recovered_position_regex='s/.*WSREP\:\s*[R|r]ecovered\s*position.*\:\(.*\)\s*$/\1/p'
+
+    # codership/galera#354
+    # Some ungraceful shutdowns can leave an empty gvwstate.dat on
+    # disk. This will prevent galera to join the cluster if it is
+    # configured to attempt PC recovery. Removing that file makes the
+    # node fall back to the normal, unoptimized joining process.
+    if [ -f ${OCF_RESKEY_datadir}/gvwstate.dat ] && \
+       [ ! -s ${OCF_RESKEY_datadir}/gvwstate.dat ]; then
+        ocf_log warn "empty ${OCF_RESKEY_datadir}/gvwstate.dat detected, removing it to prevent PC recovery failure at next restart"
+        rm -f ${OCF_RESKEY_datadir}/gvwstate.dat
+    fi
+
+    ocf_log info "attempting to detect last commit version by reading ${OCF_RESKEY_datadir}/grastate.dat"
+    last_commit="$(cat ${OCF_RESKEY_datadir}/grastate.dat | sed -n 's/^seqno.\s*\(.*\)\s*$/\1/p')"
+    if [ -z "$last_commit" ] || [ "$last_commit" = "-1" ]; then
+        local tmp=$(mktemp)
+        chown $OCF_RESKEY_user:$OCF_RESKEY_group $tmp
+
+        # if we pass here because grastate.dat doesn't exist,
+        # try not to bootstrap from this node if possible
+        if [ ! -f ${OCF_RESKEY_datadir}/grastate.dat ]; then
+            set_no_grastate
+        fi
+
+        ocf_log info "now attempting to detect last commit version using 'mysqld_safe --wsrep-recover'"
+
+        $SU - $OCF_RESKEY_user -s /bin/sh -c \
+        "${OCF_RESKEY_binary} $recover_args --wsrep-recover --log-error=$tmp 2>/dev/null"
+
+        last_commit="$(cat $tmp | sed -n $recovered_position_regex | tail -1)"
+        if [ -z "$last_commit" ]; then
+            # Galera uses InnoDB's 2pc transactions internally. If
+            # server was stopped in the middle of a replication, the
+            # recovery may find a "prepared" XA transaction in the
+            # redo log, and mysql won't recover automatically
+
+            local recovery_file="$(cat $tmp | sed -n $recovery_file_regex)"
+            if [ -e $recovery_file ]; then
+                cat $recovery_file | grep -q -E '\[ERROR\]\s+Found\s+[0-9]+\s+prepared\s+transactions!' 2>/dev/null
+                if [ $? -eq 0 ]; then
+                    # we can only rollback the transaction, but that's OK
+                    # since the DB will get resynchronized anyway
+                    ocf_log warn "local node <${NODENAME}> was not shutdown properly. Rollback stuck transaction with --tc-heuristic-recover"
+                    $SU - $OCF_RESKEY_user -s /bin/sh -c \
+                    "${OCF_RESKEY_binary} $recover_args --wsrep-recover \
+                                         --tc-heuristic-recover=rollback --log-error=$tmp 2>/dev/null"
+
+                    last_commit="$(cat $tmp | sed -n $recovered_position_regex | tail -1)"
+                    if [ ! -z "$last_commit" ]; then
+                        ocf_log warn "State recovered. force SST at next restart for full resynchronization"
+                        rm -f ${OCF_RESKEY_datadir}/grastate.dat
+                        # try not to bootstrap from this node if possible
+                        set_no_grastate
+                    fi
+                fi
+            fi
+        fi
+        rm -f $tmp
+    fi
+
+    if [ ! -z "$last_commit" ]; then
+        ocf_log info "Last commit version found:  $last_commit"
+        set_last_commit $last_commit
+        return $OCF_SUCCESS
+    else
+        ocf_exit_reason "Unable to detect last known write sequence number"
+        clear_last_commit
+        return $OCF_ERR_GENERIC
+    fi
+}
+
+# For galera, promote is really start
+galera_promote()
+{
+    local rc
+    local extra_opts
+    local bootstrap
+    local safe_to_bootstrap
+    master_exists
+    if [ $? -eq 0 ]; then
+        # join without bootstrapping
+        extra_opts="--wsrep-cluster-address=${OCF_RESKEY_wsrep_cluster_address}"
+    else
+        bootstrap=$(is_bootstrap)
+
+        if ocf_is_true $bootstrap; then
+            # The best node for bootstrapping wasn't cleanly shutdown. Allow
+            # bootstrapping anyways
+            if [ "$(get_safe_to_bootstrap)" = "0" ]; then
+                sed -ie 's/^\(safe_to_bootstrap:\) 0/\1 1/' ${OCF_RESKEY_datadir}/grastate.dat
+                ocf_log info "safe_to_bootstrap in ${OCF_RESKEY_datadir}/grastate.dat set to 1 on node ${NODENAME}"
+            fi
+            ocf_log info "Node <${NODENAME}> is bootstrapping the cluster"
+            extra_opts="--wsrep-cluster-address=gcomm://"
+        else
+            # We are being promoted without having the bootstrap
+            # attribute in the CIB, which means we are supposed to
+            # join a cluster; however if we end up here, there is no
+            # Master remaining right now, which means there is no
+            # cluster to join anymore. So force a demotion, and and
+            # let the RA decide later which node should be the next
+            # bootstrap node.
+            ocf_log warn "There is no running cluster to join, demoting ourself"
+            clear_master_score
+            return $OCF_SUCCESS
+        fi
+    fi
+
+    galera_monitor
+    if [ $? -eq $OCF_RUNNING_MASTER ]; then
+        if ocf_is_true $bootstrap; then
+            promote_everyone
+            clear_bootstrap_node
+            ocf_log info "boostrap node already up, promoting the rest of the galera instances."
+        fi
+        clear_safe_to_bootstrap
+        clear_last_commit
+        return $OCF_SUCCESS
+    fi
+
+    # last commit/safe_to_bootstrap flag are no longer relevant once promoted
+    clear_last_commit
+    clear_safe_to_bootstrap
+
+    mysql_common_prepare_dirs
+    mysql_common_start "$extra_opts"
+    rc=$?
+    if [ $rc != $OCF_SUCCESS ]; then
+        return $rc
+    fi
+
+    galera_monitor
+    rc=$?
+    if [ $rc != $OCF_SUCCESS -a $rc != $OCF_RUNNING_MASTER ]; then
+        ocf_exit_reason "Failed initial monitor action"
+        return $rc
+    fi
+
+    is_readonly
+    if [ $? -eq 0 ]; then
+        ocf_exit_reason "Failure. Master instance started in read-only mode, check configuration."
+        return $OCF_ERR_GENERIC
+    fi
+
+    is_primary
+    if [ $? -ne 0 ]; then
+        ocf_exit_reason "Failure. Master instance started, but is not in Primary mode."
+        return $OCF_ERR_GENERIC
+    fi
+
+    if ocf_is_true $bootstrap; then
+        promote_everyone
+        clear_bootstrap_node
+        # clear attribute no-grastate. if last shutdown was
+        # not clean, we cannot be extra-cautious by requesting a SST
+        # since this is the bootstrap node
+        clear_no_grastate
+        ocf_log info "Bootstrap complete, promoting the rest of the galera instances."
+    else
+        # if this is not the bootstrap node, make sure this instance
+        # syncs with the rest of the cluster before promotion returns.
+        wait_for_sync
+        # sync is done, clear info about last startup
+        clear_no_grastate
+    fi
+
+    ocf_log info "Galera started"
+    return $OCF_SUCCESS
+}
+
+galera_demote()
+{
+    mysql_common_stop
+    rc=$?
+    if [ $rc -ne $OCF_SUCCESS ] && [ $rc -ne $OCF_NOT_RUNNING ]; then
+        ocf_exit_reason "Failed to stop Master galera instance during demotion to Master"
+        return $rc
+    fi
+
+    # if this node was previously a bootstrap node, that is no longer the case.
+    clear_bootstrap_node
+    clear_last_commit
+    clear_no_grastate
+    clear_safe_to_bootstrap
+
+    # Clear master score here rather than letting pacemaker do so once
+    # demote finishes. This way a promote cannot take place right
+    # after this demote even if pacemaker is requested to do so. It
+    # will first have to run a start/monitor op, to reprobe the state
+    # of the other galera nodes and act accordingly.
+    clear_master_score
+
+    # record last commit for next promotion
+    detect_safe_to_bootstrap
+    detect_last_commit
+    rc=$?
+    return $rc
+}
+
+galera_start()
+{
+    local rc
+    local galera_node
+
+    galera_node=$(pcmk_to_galera_name $NODENAME)
+    if [ -z "$galera_node" ]; then
+        ocf_exit_reason "Could not determine galera name from pacemaker node <${NODENAME}>."
+        return $OCF_ERR_CONFIGURED
+    fi
+
+    echo $OCF_RESKEY_wsrep_cluster_address | grep -q -F $galera_node
+    if [ $? -ne 0 ]; then
+        ocf_exit_reason "local node <${NODENAME}> (galera node <${galera_node}>) must be a member of the wsrep_cluster_address <${OCF_RESKEY_wsrep_cluster_address}> to start this galera instance"
+        return $OCF_ERR_CONFIGURED
+    fi
+
+    galera_monitor
+    if [ $? -eq $OCF_RUNNING_MASTER ]; then
+        ocf_exit_reason "master galera instance started outside of the cluster's control"
+        return $OCF_ERR_GENERIC
+    fi
+
+    mysql_common_prepare_dirs
+
+    detect_safe_to_bootstrap
+    detect_last_commit
+    rc=$?
+    if [ $rc -ne $OCF_SUCCESS ]; then
+        return $rc
+    fi
+
+    master_exists
+    if [ $? -eq 0 ]; then
+        ocf_log info "Master instances are already up, setting master score so this instance will join galera cluster."
+        set_master_score $NODENAME
+    else
+        clear_master_score
+        detect_first_master
+    fi
+
+    return $OCF_SUCCESS
+}
+
+galera_monitor()
+{
+    local rc
+    local galera_node
+    local status_loglevel="err"
+
+    # Set loglevel to info during probe
+    if ocf_is_probe; then
+        status_loglevel="info"
+    fi
+
+    mysql_common_status $status_loglevel
+    rc=$?
+
+    if [ $rc -eq $OCF_NOT_RUNNING ]; then
+        last_commit=$(get_last_commit $node)
+        if [ -n "$last_commit" ]; then
+            # if last commit is set, this instance is considered started in slave mode
+            rc=$OCF_SUCCESS
+            master_exists
+            if [ $? -ne 0 ]; then
+                detect_first_master
+            else
+                # a master instance exists and is healthy, promote this
+                # local read only instance
+                # so it can join the master galera cluster.
+                set_master_score
+            fi
+        fi
+        return $rc
+    elif [ $rc -ne $OCF_SUCCESS ]; then
+        return $rc
+    fi
+
+    # if we make it here, mysql is running. Check cluster status now.
+    galera_node=$(pcmk_to_galera_name $NODENAME)
+    if [ -z "$galera_node" ]; then
+        ocf_exit_reason "Could not determine galera name from pacemaker node <${NODENAME}>."
+        return $OCF_ERR_CONFIGURED
+    fi
+
+    echo $OCF_RESKEY_wsrep_cluster_address | grep -q -F $galera_node
+    if [ $? -ne 0 ]; then
+        ocf_exit_reason "local node <${NODENAME}> (galera node <${galera_node}>) is started, but is not a member of the wsrep_cluster_address <${OCF_RESKEY_wsrep_cluster_address}>"
+        return $OCF_ERR_GENERIC
+    fi
+
+    is_primary
+    if [ $? -eq 0 ]; then
+
+        if ocf_is_probe; then
+            # restore master score during probe
+            # if we detect this is a master instance
+            set_master_score
+        fi
+        rc=$OCF_RUNNING_MASTER
+    else
+        ocf_exit_reason "local node <${NODENAME}> is started, but not in primary mode. Unknown state."
+        rc=$OCF_ERR_GENERIC
+    fi
+
+    return $rc
+}
+
+galera_stop()
+{
+    local rc
+    # make sure the process is stopped
+    mysql_common_stop
+    rc=$1
+
+    clear_safe_to_bootstrap
+    clear_last_commit
+    clear_master_score
+    clear_bootstrap_node
+    clear_no_grastate
+    return $rc
+}
+
+galera_validate()
+{
+    if [ "$OCF_CHECK_LEVEL" -eq 10 ]; then
+        if ! ocf_is_ms; then
+            ocf_exit_reason "Galera must be configured as a multistate Master/Slave resource."
+            return $OCF_ERR_CONFIGURED
+        fi
+    fi
+
+    if [ -z "$OCF_RESKEY_wsrep_cluster_address" ]; then
+        ocf_exit_reason "Galera must be configured with a wsrep_cluster_address value."
+        return $OCF_ERR_CONFIGURED
+    fi
+
+    mysql_common_validate
+}
+
+case "$1" in
+  meta-data)    meta_data
+        exit $OCF_SUCCESS;;
+  usage|help)   usage
+        exit $OCF_SUCCESS;;
+esac
+
+[ "$__OCF_ACTION" = "start" ] && OCF_CHECK_LEVEL=10
+galera_validate
+rc=$?
+LSB_STATUS_STOPPED=3
+if [ $rc -ne 0 ]; then
+    case "$1" in
+        stop) exit $OCF_SUCCESS;;
+        monitor) exit $OCF_NOT_RUNNING;;
+        status) exit $LSB_STATUS_STOPPED;;
+        *) exit $rc;;
+    esac
+fi
+
+if [ -z "${OCF_RESKEY_check_passwd}" ]; then
+    # This value is automatically sourced from /etc/sysconfig/checkcluster if available
+    OCF_RESKEY_check_passwd=${MYSQL_PASSWORD}
+fi
+if [ -z "${OCF_RESKEY_check_user}" ]; then
+    # This value is automatically sourced from /etc/sysconfig/checkcluster if available
+    OCF_RESKEY_check_user=${MYSQL_USERNAME}
+fi
+: ${OCF_RESKEY_check_user="root"}
+
+MYSQL_OPTIONS_CHECK="-nNE --user=${OCF_RESKEY_check_user}"
+
+if ocf_is_true "${OCF_RESKEY_check_passwd_use_empty}"; then
+    MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK --password="
+elif [ -n "${OCF_RESKEY_check_passwd}" ]; then
+    MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK --password=${OCF_RESKEY_check_passwd}"
+fi
+
+# This value is automatically sourced from /etc/sysconfig/checkcluster if available
+if [ -n "${MYSQL_HOST}" ]; then
+    MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK -h ${MYSQL_HOST}"
+fi
+
+# This value is automatically sourced from /etc/sysconfig/checkcluster if available
+if [ -n "${MYSQL_PORT}" ]; then
+    MYSQL_OPTIONS_CHECK="$MYSQL_OPTIONS_CHECK -P ${MYSQL_PORT}"
+fi
+
+
+
+# What kind of method was invoked?
+case "$1" in
+  start)    galera_start;;
+  stop)     galera_stop;;
+  status)   mysql_common_status err;;
+  monitor)  galera_monitor;;
+  promote)  galera_promote;;
+  demote)   galera_demote;;
+  validate-all) exit $OCF_SUCCESS;;
+
+ *)     usage
+        exit $OCF_ERR_UNIMPLEMENTED;;
+esac
+
+# vi:sw=4:ts=4:et: