#!@BASH_PATH@
#
# ocf:pacemaker:ifspeed resource agent
#
# Copyright 2011-2023 the Pacemaker project contributors
#
# The version control history for this file may have further details.
#
# This source code is licensed under the GNU General Public License version 2
# or later (GPLv2+) WITHOUT ANY WARRANTY.
#

#
# Record speed of a network interface as a node attribute, based on the sum of
# speeds of its active (up, link detected, not blocked) underlying interfaces.
#
# Originally based on ocf:pacemaker:ping agent
#

: ${OCF_FUNCTIONS:="${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs"}

# If these aren't available, we can still show help,
# which is all that is needed to build the man pages.
[ -r "${OCF_FUNCTIONS}" ] && . "${OCF_FUNCTIONS}"
[ -r "${OCF_FUNCTIONS_DIR}/findif.sh" ] && . "${OCF_FUNCTIONS_DIR}/findif.sh"
: ${OCF_SUCCESS:=0}

: ${__OCF_ACTION:=$1}

FINDIF=findif

# Defaults
OCF_RESKEY_name_default="ifspeed"
OCF_RESKEY_bridge_ports_default="detect"
OCF_RESKEY_weight_base_default=1000
OCF_RESKEY_dampen_default=5

# Explicitly list all environment variables used, to make static analysis happy
: ${OCF_RESKEY_name:=${OCF_RESKEY_name_default}}
: ${OCF_RESKEY_bridge_ports:=${OCF_RESKEY_bridge_ports_default}}
: ${OCF_RESKEY_weight_base:=${OCF_RESKEY_weight_base_default}}
: ${OCF_RESKEY_dampen:=${OCF_RESKEY_dampen_default}}
: ${OCF_RESKEY_iface:=""}
: ${OCF_RESKEY_ip:=""}
: ${OCF_RESKEY_debug:="false"}

meta_data() {
        cat <<END
<?xml version="1.0"?>
<resource-agent name="ifspeed" version="@VERSION@">
<version>1.1</version>

<longdesc lang="en">
This agent's monitor action records the speed of a specified network interface
as a node attribute. The attribute can be used in rules to prefer nodes based
on network speeds.

This agent can monitor physical interfaces, bonded interfaces, bridges, VLANs,
or any combination thereof. For example:

*) Bridge on top of one 10Gbps interface (eth2) and 802.3ad bonding (bond0) built
   on two 1Gbps interfaces (eth0 and eth1).
*) Active-backup bonding built on top of one physical interface and one VLAN on
   another interface.

For STP-enabled bridges, this agent tries to determine the network topology, and
by default looks only on ports which are connected to an upstream switch. This
can be overridden by 'bridge_ports' parameter. Active interfaces in this case
are those in "forwarding" state.

For balancing bonded interfaces, this agent uses 80% of the sum of the speeds of
underlying "up" ports.

For non-balancing bonded interfaces ("active-backup" and probably "broadcast"),
only the speed of the currently active port is considered.
</longdesc>
<shortdesc lang="en">Network interface speed monitor</shortdesc>

<parameters>

<parameter name="name" unique-group="name">
<longdesc lang="en">
Name of the node attribute to set
</longdesc>
<shortdesc lang="en">Attribute name</shortdesc>
<content type="string" default="${OCF_RESKEY_name_default}"/>
</parameter>

<parameter name="iface" unique-group="iface">
<longdesc lang="en">
If this is set, monitor this network interface. One of iface or ip must be set.
</longdesc>
<shortdesc lang="en">Network interface</shortdesc>
<content type="string" default=""/>
</parameter>

<parameter name="ip" unique-group="ip">
<longdesc lang="en">
If this is set instead of iface, monitor the interface that holds this IP
address. The address may be specified in dotted-quad notation for IPv4 (for
example, 192.168.1.1) or hexadecimal notation for IPv6 (for example,
2001:db8:DC28:0:0:FC57:D4C8:1FFF). One of iface or ip must be set.
</longdesc>
<shortdesc lang="en">IPv4 or IPv6 address</shortdesc>
<content type="string" default="" />
</parameter>

<parameter name="bridge_ports" unique-group="iface">
<longdesc lang="en">
If set and iface is a bridge, consider these bridge ports (by default, all ports
which have designated_bridge=root_id)
</longdesc>
<shortdesc lang="en">Bridge ports</shortdesc>
<content type="string" default="${OCF_RESKEY_bridge_ports_default}"/>
</parameter>

<parameter name="weight_base">
<longdesc lang="en">
Relative weight of 1Gbps in interface speed.
Can be used to tune how big attribute value will be.
</longdesc>
<shortdesc lang="en">Weight of 1Gbps</shortdesc>
<content type="integer" default="${OCF_RESKEY_weight_base_default}"/>
</parameter>

<parameter name="dampen">
<longdesc lang="en">
The time to wait (dampening) for further changes to occur.
</longdesc>
<shortdesc lang="en">Dampening interval</shortdesc>
<content type="integer" default="${OCF_RESKEY_dampen_default}"/>
</parameter>

<parameter name="debug">
<longdesc lang="en">
Log more verbosely.
</longdesc>
<shortdesc lang="en">Verbose logging</shortdesc>
<content type="string" default="false"/>
</parameter>

</parameters>

<actions>
<action name="start"   timeout="30s" />
<action name="stop"    timeout="30s" />
<action name="monitor" depth="0"  timeout="30s" interval="10s"/>
<action name="meta-data"  timeout="5s" />
<action name="validate-all"  timeout="30s" depth="0" />
</actions>
</resource-agent>
END
}

usage() {
    cat <<END
Usage: $0 {start|stop|monitor|validate-all|meta-data}

Expects to have a fully populated OCF RA-compliant environment set.
END
}

start() {
    monitor
    if [ $? -eq $OCF_SUCCESS ]; then
        return $OCF_SUCCESS
    fi
    ha_pseudo_resource ${ha_pseudo_resource_name} start
    update
    return $?
}

stop() {
    ha_pseudo_resource "${ha_pseudo_resource_name}" stop
    attrd_updater -D -n "${OCF_RESKEY_name}" -d "${OCF_RESKEY_dampen}" ${attrd_options}
    return $OCF_SUCCESS
}

monitor() {
    local ret

    ha_pseudo_resource "${ha_pseudo_resource_name}" monitor
    ret=$?
    if [ ${ret} -eq $OCF_SUCCESS ] ; then
        update
    fi
    return ${ret}
}

get_nic_name_by_ip() {
    # $FINDIF takes its parameters from the environment.
    # Its output is as follows:
    # [NIC_NAME] netmask [NETMASK] broadcast [BROADCAST}
    NICINFO=$( "${FINDIF}" )
    rc=$?
    if [ $rc -eq 0 ];then
        # Get NIC_NAME part of findif function output.
        echo "${NICINFO%% *}"
    else
        echo ""
    fi
}

validate() {
    if [ -z "${OCF_RESKEY_iface}" ]; then
        if [ -z "${OCF_RESKEY_ip}" ]; then
            ocf_log err "Must specify either an interface name or valid IP address"
            ocf_exit_reason "no interface or IP address specified"
            exit $OCF_ERR_CONFIGURED
        else
            ipcheck_ipv4 "${OCF_RESKEY_ip}"
            if [ $? -eq 1 ] ; then
                ipcheck_ipv6 "${OCF_RESKEY_ip}"
                if [ $? -eq 1 ] ; then
                  ocf_exit_reason "'${OCF_RESKEY_ip}' is not a valid IP"
                  exit $OCF_ERR_CONFIGURED
                fi
            fi
        fi
    fi

    # Host-specific checks
    if [ "$1" = "10" ]; then
        if [ "$(uname)" != "Linux" ] ; then
            ocf_log err "This resource agent works only on Linux"
            ocf_exit_reason "not Linux"
            exit $OCF_ERR_INSTALLED
        fi
    fi
    return $OCF_SUCCESS
}

iface_get_speed() {
    local iface="$1"
    local operstate
    local carrier
    local speed

    if [ ! -e "/sys/class/net/${iface}" ] ; then
        echo "0"
    elif iface_is_bridge "${iface}" ; then # bridges do not have operstate
        read carrier < "/sys/class/net/${iface}/carrier"

        if [ "${carrier}" != "1" ] ; then
            echo "0"
        else
            bridge_get_speed "${iface}"
        fi
    else
        read operstate < "/sys/class/net/${iface}/operstate"
        read carrier < "/sys/class/net/${iface}/carrier"

        if [ "${operstate}" != "up" ] || [ "${carrier}" != "1" ] ; then
            echo "0"
        elif iface_is_bond "${iface}" ; then
            bond_get_speed "${iface}"
        elif iface_is_vlan "${iface}" ; then
            iface_get_speed "$(vlan_get_phy "${iface}")"
        elif iface_is_hfi1 "${iface}" ; then
            hfi1_get_speed "${iface}"
        else
            read speed < "/sys/class/net/${iface}/speed"
            echo "${speed}"
        fi
    fi
}

iface_is_vlan() {
    local iface="$1"

    [ -e "/proc/net/vlan/${iface}" ] && return 0 || return 1
}

iface_is_bridge() {
    local iface="$1"

    [ -e "/sys/class/net/${iface}/bridge" ] && return 0 || return 1
}

iface_is_bond() {
    local iface="$1"

    [ -e "/sys/class/net/${iface}/bonding" ] && return 0 || return 1
}

iface_is_hfi1() {
    local iface="$1"

    driver=$(readlink "/sys/class/net/${iface}/device/driver")
    [[ $(basename "${driver}") =~ "hfi1" ]] && return 0 || return 1
}

vlan_get_phy() {
    local iface="$1"

    sed -ne "s/^${iface} .*| *//p" < /proc/net/vlan/config
}

bridge_is_stp_enabled() {
    local iface="$1"
    local stp

    read stp < "/sys/class/net/${iface}/bridge/stp_state"
    [ "${stp}" = "1" ] && return 0 || return 1
}

bridge_get_root_ports() {
    local bridge="$1"
    local root_id
    local root_ports=""
    local bridge_id

    read root_id < "/sys/class/net/${bridge}/bridge/root_id"

    for port in /sys/class/net/${bridge}/brif/* ; do
        read bridge_id < "${port}/designated_bridge"
        if [ "${bridge_id}" = "${root_id}" ] ; then
            root_ports="${root_ports} ${port##*/}"
        fi
    done

    root_ports=${root_ports# }

    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${root_ports}"
    else # Expect sub-shell
        echo ${root_ports}
    fi
}

# From /usr/include/linux/if_bridge.h:
#define BR_STATE_DISABLED 0
#define BR_STATE_LISTENING 1
#define BR_STATE_LEARNING 2
#define BR_STATE_FORWARDING 3
#define BR_STATE_BLOCKING 4

bridge_get_active_ports() {
    local bridge="$1"
    shift 1
    local ports="$*"
    local active_ports=""
    local port_state
    local stp_state
    local warn=0

    bridge_is_stp_enabled "${bridge}"
    stp_state=$?

    if [ -z "${ports}" ] || [ "${ports}" = "detect" ] ; then
        bridge_get_root_ports "${bridge}" ports
    fi

    for port in $ports ; do
        if [ ! -e "/sys/class/net/${bridge}/brif/${port}" ] ; then
            ocf_log warning "Port ${port} doesn't belong to bridge ${bridge}"
            continue
        fi
        read port_state < "/sys/class/net/${bridge}/brif/${port}/state"
        if [ "${port_state}" = "3" ] ; then
            if [ -n "${active_ports}" ] && ${stp_state} ; then
                warn=1
            fi
            active_ports="${active_ports} ${port}"
        fi
    done
    if [ ${warn} -eq 1 ] ; then
        ocf_log warning "More then one upstream port in bridge '${bridge}' is in forwarding state while STP is enabled: ${active_ports}"
    fi
    echo "${active_ports# }"
}

bridge_get_speed() {
    local iface="$1"
    local aggregate_speed=0

    if ! iface_is_bridge "${iface}" ; then
        echo 0
        return
    fi

    BGS_PORTS=$( bridge_get_active_ports "${iface}" "${OCF_RESKEY_bridge_ports}" )
    for port in ${BGS_PORTS} ; do
        : $(( aggregate_speed += $( iface_get_speed "${port}" ) ))
    done
    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${aggregate_speed}"
    else # Expect sub-shell
        echo ${aggregate_speed}
    fi
}

hfi1_get_speed() {
    local iface="$1"
    local hfi1_speed
    local hfi1_value
    local hfi1_desc

    # At least as of 9/14/2017 Intel Omni Path v10.5.0.0.155, Intel doesn't have
    # dual- or multiple-port Host Channel Adapters, and it's safe to use this
    # method to get the speed. Example output:
    # [root@es-host0 ~]# cat /sys/class/net/ib0/device/infiniband/*/ports/*/rate
    # 100 Gb/sec (4X EDR)
    read hfi1_speed hfi1_value hfi1_desc < "/sys/class/net/${iface}/device/infiniband"/*/ports/*/rate
    ocf_is_true "${OCF_RESKEY_debug}" && ocf_log debug "Detected speed $hfi1_speed $hfi1_value $hfi1_desc"

    # hfi1_value always in Gb/sec, so we need to convert hfi1_speed in Mb/sec
    echo $(( hfi1_speed * 1000 ))
}

bond_get_ports() {
    local iface="$1"
    local ports

    read ports < "/sys/class/net/${iface}/bonding/slaves"
    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${ports}"
    else # Expect sub-shell
        echo ${ports}
    fi
}

bond_get_active_iface() {
    local iface="$1"
    local active

    read active < "/sys/class/net/${iface}/bonding/active_slave"
    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${active}"
    else # Expect sub-shell
        echo ${active}
    fi
}

bond_is_balancing() {
    local iface="$1"
    read mode mode_index < "/sys/class/net/${iface}/bonding/mode"
    ocf_is_true "${OCF_RESKEY_debug}" && ocf_log debug "Detected balancing $mode $mode_index"
    case "${mode}" in
        "balance-rr"|"balance-xor"|"802.3ad"|"balance-tlb"|"balance-alb")
            return 0
            ;;
        *)
            return 1
            ;;
    esac
}

bond_get_speed() {
    local iface="$1"
    local aggregate_speed=0
    local active_iface
    local bond_ports

    if ! iface_is_bond "${iface}" ; then
        echo 0
        return
    fi

    bond_get_ports "${iface}" bond_ports

    if bond_is_balancing "${iface}" ; then
        for port in ${bond_ports} ; do
            : $(( aggregate_speed += $( iface_get_speed "${port}" ) ))
        done
        # Bonding is unable to get speed*n
        : $(( aggregate_speed = aggregate_speed * 8 / 10 ))
    else
        bond_get_active_iface "${iface}" "active_iface"
        aggregate_speed=$( iface_get_speed "$active_iface" )
    fi
    if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell.
        eval "$2=\${aggregate_speed}"
    else # Expect sub-shell
        echo ${aggregate_speed}
    fi
}

update() {
    local speed;
    local nic="${OCF_RESKEY_iface}";

    if [ -z "${OCF_RESKEY_iface}" ]; then
        nic=$( get_nic_name_by_ip )
        if [ -z "${nic}" ];then
            ocf_log err "Could not determine network interface name from IP address (${OCF_RESKEY_ip})"
            ocf_exit_reason "unable to determine interface name"
            exit $OCF_ERR_GENERIC
        fi
    fi
    speed=$( iface_get_speed "${nic}" )

    : $(( score = speed * ${OCF_RESKEY_weight_base} / 1000 ))
    if [ "$__OCF_ACTION" = "start" ] ; then
        attrd_updater -n "${OCF_RESKEY_name}" -B "${score}" -d "${OCF_RESKEY_dampen}" ${attrd_options}
    else
        attrd_updater -n "${OCF_RESKEY_name}" -v "${score}" -d "${OCF_RESKEY_dampen}" ${attrd_options}
    fi
    rc=$?
    case ${rc} in
        0)
            ocf_is_true "${OCF_RESKEY_debug}" && ocf_log debug "Updated ${OCF_RESKEY_name} = ${score}"
            ;;
        *)
            ocf_log warn "Could not update ${OCF_RESKEY_name} = ${score}: rc=${rc}"
            ;;
    esac
    return ${rc}
}

case $__OCF_ACTION in
    meta-data)
        meta_data
        exit $OCF_SUCCESS
        ;;
    usage|help)
        usage
        exit $OCF_SUCCESS
        ;;
esac

: ${ha_pseudo_resource_name:="ifspeed-${OCF_RESOURCE_INSTANCE}"}

attrd_options=''
if ocf_is_true "${OCF_RESKEY_debug}" ; then
    attrd_options='-VV'
fi

case "$__OCF_ACTION" in
    start)
        validate 10
        start
        ;;
    stop)
        validate 10
        stop
        ;;
    monitor)
        validate 10
        monitor
        ;;
    validate-all)
        validate "$OCF_CHECK_LEVEL"
        ;;
    *)
        usage
        exit $OCF_ERR_UNIMPLEMENTED
        ;;
esac

exit $?

# vim: set filetype=sh expandtab tabstop=4 softtabstop=4 shiftwidth=4 textwidth=80: