path: root/cts/lxc_autogen.sh.in

#!@BASH_PATH@
#
# Copyright 2013-2022 the Pacemaker project contributors
#
# The version control history for this file may have further details.
#
# This source code is licensed under the GNU General Public License version 2
# or later (GPLv2+) WITHOUT ANY WARRANTY.
#

containers="2"
download=0
share_configs=0
# different than default libvirt network in case this is run nested in a KVM instance
addr="192.168.123.1"
restore=0
restore_pcmk=0
restore_all=0
generate=0
key_gen=0
cib=0
anywhere=0
add_clone=0
verify=0
working_dir="@CRM_PACEMAKER_DIR@/cts/lxc"
run_dirs="/run /var/run /usr/var/run"

# must be on one line b/c used inside quotes
SSH_RSYNC_OPTS="-o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no"

function helptext() {
    echo "lxc_autogen.sh - generate libvirt LXC containers for testing purposes"
    echo ""
    echo "Usage: lxc-autogen [options]"
    echo ""
    echo "Options:"
    echo "-g, --generate         Generate libvirt LXC environment in directory this script is run from"
    echo "-k, --key-gen          Generate Pacemaker Remote key only"
    echo "-r, --restore-libvirt  Restore the default network and libvirt config to before this script ran"
    echo "-p, --restore-cib      Remove CIB entries this script generated"
    echo "-R, --restore-all      Restore both libvirt and CIB, and clean working directory"
    echo "                       (libvirt xml files are not removed, so resource can be stopped properly)"
    echo ""
    echo "-A, --allow-anywhere   Allow the containers to live anywhere in the cluster"
    echo "-a, --add-cib          Add CIB entries to create a guest node for each LXC instance"
    echo "-C, --add-clone        Add promotable clone resource shared between LXC guest nodes"
    echo "-d, --download-agent   Download and install latest VirtualDomain agent"
    echo "-s, --share-configs    Synchronize on all known cluster nodes"
    echo "-c, --containers       Specify number of containers to generate (default $containers; used with -g)"
    echo "-n, --network          Network to override libvirt default (example: -n 192.168.123.1; used with -g)"
    echo "-v, --verify           Verify environment is capable of running LXC"
    echo ""
    exit "$1"
}

while true ; do
    case "$1" in
    --help|-h|-\?) helptext 0;;
    -c|--containers) containers="$2"; shift; shift;;
    -d|--download-agent) download=1; shift;;
    -s|--share-configs) share_configs=1; shift;;
    -n|--network) addr="$2"; shift; shift;;
    -r|--restore-libvirt) restore=1; shift;;
    -p|--restore-cib) restore_pcmk=1; shift;;
    -R|--restore-all)
        restore_all=1
        restore=1
        restore_pcmk=1
        shift;;
    -g|--generate) generate=1; key_gen=1; shift;;
    -k|--key-gen) key_gen=1; shift;;
    -a|--add-cib) cib=1; shift;;
    -A|--allow-anywhere) anywhere=1; shift;;
    -C|--add-clone) add_clone=1; shift;;
    -m|--add-master)
        echo "$1 is deprecated (use -C/--add-clone instead)"
        echo
        add_clone=1
        shift
        ;;
    -v|--verify) verify=1; shift;;
    "") break;;
    *) helptext 1;;
    esac
done

if [ $verify -eq 1 ]; then
    # verify virsh tool is available and that 
    # we can connect to lxc driver.
    virsh -c lxc:/// list --all > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo "libvirt LXC driver must be installed (could not connect 'virsh -c lxc:///')"
        # yum install -y libvirt-daemon-driver-lxc libvirt-daemon-lxc libvirt-login-shell
        exit 1
    fi

    SELINUX=$(getenforce)
    if [ "$SELINUX" != "Enforcing" ] && [ "$SELINUX" != "Permissive" ]; then
        echo "SELINUX must be set to permissive or enforcing mode"
        exit 1
    fi

    ps ax | grep "[l]ibvirtd"
    if [ $? -ne 0 ]; then
        echo "libvirtd must be running"
        exit 1
    fi

    which rsync > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo "rsync must be installed"
    fi

    which pacemaker-remoted > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo "pacemaker-remoted must be installed"
    fi
fi

#strip last digits off addr
addr="$(echo "$addr" | awk -F. '{print $1"."$2"."$3}')"

node_exec() {
    ssh -o StrictHostKeyChecking=no \
        -o ConnectTimeout=30 \
        -o BatchMode=yes \
        -l root -T "$@"
}

this_node()
{
    crm_node -n
}

other_nodes()
{
    crm_node -l | awk "\$2 != \"$(this_node)\" {print \$2}"
}

make_directory()
{
    # argument must be full path
    DIR="$1"

    mkdir -p "$DIR"
    if [ $share_configs -eq 1 ]; then
        for node in $(other_nodes); do
            node_exec "$node" mkdir -p "$DIR"
        done
    fi
}

sync_file()
{
    TARGET="$1"

    if [ $share_configs -eq 1 ]; then
        for node in $(other_nodes); do
            rsync -ave "ssh $SSH_RSYNC_OPTS" "$TARGET" "${node}:${TARGET}"
        done
    fi
}

download_agent()
{
    wget https://raw.github.com/ClusterLabs/resource-agents/main/heartbeat/VirtualDomain
    chmod 755 VirtualDomain
    mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
    sync_file /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
}

set_network()
{
    rm -f cur_network.xml
    cat << END >> cur_network.xml
<network>
  <name>default</name>
  <uuid>41ebdb84-7134-1111-a136-91f0f1119225</uuid>
  <forward mode='nat'/>
  <bridge name='virbr0' stp='on' delay='0' />
  <mac address='52:54:00:A8:12:35'/>
  <ip address='$addr.1' netmask='255.255.255.0'>
    <dhcp>
      <range start='$addr.2' end='$addr.254' />
    </dhcp>
  </ip>
</network>
END
    sync_file "${working_dir}"/cur_network.xml
}

distribute_configs()
{
    for node in $(other_nodes); do
        rsync -ave "ssh $SSH_RSYNC_OPTS" "${working_dir}"/lxc*.xml "${node}:${working_dir}"
        rsync -ave "ssh $SSH_RSYNC_OPTS" "${working_dir}"/lxc*-filesystem "${node}:${working_dir}"
    done
}

start_network()
{
    NODE="$1"

    node_exec "$NODE" <<-EOF
    cd "$working_dir"
    virsh net-info default >/dev/null 2>&1
    if [ \$? -eq 0 ]; then
        if [ ! -f restore_default.xml ]; then
            virsh net-dumpxml default > restore_default.xml
        fi
        virsh net-destroy default
        virsh net-undefine default
    fi
    virsh net-define cur_network.xml
    virsh net-start default
    virsh net-autostart default
EOF
}

start_network_all()
{
    start_network "$(this_node)"
    if [ $share_configs -eq 1 ]; then
        for node in $(other_nodes); do
            start_network "$node"
        done
    fi
}

add_hosts_entry()
{
    IP="$1"
    HNAME="$2"

    echo "$IP $HNAME" >>/etc/hosts
    if [ $share_configs -eq 1 ]; then
        for node in $(other_nodes); do
            node_exec "$node" "echo $IP $HNAME >>/etc/hosts"
        done
    fi
}

generate_key()
{
    if [ ! -e /etc/pacemaker/authkey ]; then
        make_directory /etc/pacemaker
        dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1
        sync_file /etc/pacemaker/authkey
    fi
}

generate()
{
    set_network

    # Generate libvirt domains in xml
    for (( c=1; c <= containers; c++ ))
    do
        # Clean any previous definition
        rm -rf "lxc$c.xml" "lxc$c-filesystem"

        # Create a basic filesystem with run directories
        for dir in $run_dirs; do
            mkdir -p "lxc$c-filesystem/$dir"
        done

        # Create libvirt definition
        suffix=$((10 + c))
        prefix="$(echo "$addr" | awk -F. '{print $1"."$2}')"
        subnet="$(echo "$addr" | awk -F. '{print $3}')"
        while [ $suffix -gt 255 ]; do
            subnet=$((subnet + 1))
            suffix=$((subnet - 255))
        done
        cip="$prefix.$subnet.$suffix"

        cat << END >> lxc$c.xml
<domain type='lxc'>
  <name>lxc$c</name>
  <memory unit='KiB'>200704</memory>
  <os>
    <type>exe</type>
    <init>$working_dir/lxc$c-filesystem/launch-helper</init>
  </os>
  <devices>
    <console type='pty'/>
    <filesystem type='ram'>
        <source usage='150528'/>
        <target dir='/dev/shm'/>
    </filesystem>
END
        for dir in $run_dirs; do
            cat << END >> lxc$c.xml
    <filesystem type='mount'>
      <source dir='$working_dir/lxc$c-filesystem${dir}'/>
      <target dir='$dir'/>
    </filesystem>
END
        done
        cat << END >> lxc$c.xml
    <interface type='network'>
      <mac address='52:54:$((RANDOM % 9))$((RANDOM % 9)):$((RANDOM % 9))$((RANDOM % 9)):$((RANDOM % 9))$((RANDOM % 9)):$((RANDOM % 9))$((RANDOM % 9))'/>
      <source network='default'/>
    </interface>
  </devices>
</domain>
END

        # Create CIB definition
        rm -f "container$c.cib"
        cat << END >> "container$c.cib"
      <primitive class="ocf" id="container$c" provider="heartbeat" type="VirtualDomain">
        <instance_attributes id="container$c-instance_attributes">
          <nvpair id="container$c-instance_attributes-force_stop" name="force_stop" value="true"/>
          <nvpair id="container$c-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
          <nvpair id="container$c-instance_attributes-config" name="config" value="$working_dir/lxc$c.xml"/>
        </instance_attributes>
        <utilization id="container$c-utilization">
          <nvpair id="container$c-utilization-cpu" name="cpu" value="1"/>
          <nvpair id="container$c-utilization-hv_memory" name="hv_memory" value="100"/>
        </utilization>
        <meta_attributes id="container$c-meta_attributes">
          <nvpair id="container$c-meta_attributes-remote-node" name="remote-node" value="lxc$c"/>
        </meta_attributes>
        <operations>
          <op id="container$c-monitor-20s" interval="20s" name="monitor"/>
        </operations>
      </primitive>
END

        # Create container init
        rm -f "lxc$c-filesystem/launch-helper"
        cat << END >> "lxc$c-filesystem/launch-helper"
#!@BASH_PATH@
ip -f inet addr add "$cip/24" dev eth0
ip link set eth0 up
ip route add default via "$addr.1"
hostname "lxc$c"
df > "$working_dir/lxc$c-filesystem/disk_usage.txt"
export PCMK_debugfile="@CRM_LOG_DIR@/pacemaker_remote_lxc$c.log"
/usr/sbin/pacemaker-remoted
END
        chmod 711 "lxc$c-filesystem/launch-helper"

        add_hosts_entry "$cip" "lxc$c"
    done

    # Create CIB fragment for a promotable clone resource
    cat << END > lxc-clone.cib
      <clone id="lxc-clone">
        <primitive class="ocf" id="lxc-rsc" provider="pacemaker" type="Stateful">
          <instance_attributes id="lxc-rsc-instance_attributes"/>
          <operations>
            <op id="lxc-rsc-monitor-interval-10s" interval="10s" name="monitor" role="Promoted" timeout="20s"/>
            <op id="lxc-rsc-monitor-interval-11s" interval="11s" name="monitor" role="Unpromoted" timeout="20s"/>
          </operations>
        </primitive>
        <meta_attributes id="lxc-clone-meta_attributes">
          <nvpair id="lxc-clone-meta_attributes-promotable" name="promotable" value="true"/>
          <nvpair id="lxc-clone-meta_attributes-promoted-max" name="promoted-max" value="1"/>
          <nvpair id="lxc-clone-meta_attributes-clone-max" name="clone-max" value="$containers"/>
        </meta_attributes>
      </clone>
END
}

container_names() {
    find . -maxdepth 1 -name "lxc*.xml" -exec basename -s .xml "{}" ";"
}

apply_cib_clone()
{
    cibadmin -Q > cur.cib
    export CIB_file=cur.cib

    cibadmin -o resources -Mc -x lxc-clone.cib
    for tmp in $(container_names); do
        echo "<rsc_location id=\"lxc-clone-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-clone\" score=\"INFINITY\"/>" > tmp_constraint
        cibadmin -o constraints -Mc -x tmp_constraint
    done
    # Make sure the version changes even if the content doesn't
    cibadmin -B
    unset CIB_file

    cibadmin --replace -o configuration --xml-file cur.cib
    rm -f cur.cib
}

apply_cib_entries()
{
    cibadmin -Q > cur.cib
    export CIB_file=cur.cib
    for tmp in container*.cib; do
        cibadmin -o resources -Mc -x "$tmp"

        remote_node="$(grep remote-node "${tmp}" | sed -n -e 's/^.*value=\"\(.*\)\".*/\1/p')"
        if [ $anywhere -eq 0 ]; then
            crm_resource -M -r "${tmp//\.cib/}" -H "$(this_node)"
        fi
        echo "<rsc_location id=\"lxc-ping-location-${remote_node}\" node=\"${remote_node}\" rsc=\"Connectivity\" score=\"-INFINITY\"/>" > tmp_constraint
        # Ignore any failure; this constraint is just to help with CTS when the
        # connectivity resources (which fail the guest nodes) are in use.
        cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1

        for rsc in $(crm_resource -l | grep rsc_ ); do
            echo "<rsc_location id=\"lxc-${rsc}-location-${remote_node}\" node=\"${remote_node}\" rsc=\"${rsc}\" score=\"-INFINITY\"/>" > tmp_constraint
            cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1
        done

        rm -f tmp_constraint
    done

    # Make sure the version changes even if the content doesn't
    cibadmin -B

    unset CIB_file

    cibadmin --replace -o configuration --xml-file cur.cib
    rm -f cur.cib
}

restore_cib()
{
    cibadmin -Q > cur.cib
    export CIB_file=cur.cib

    for tmp in $(container_names); do
        echo "<rsc_location id=\"lxc-clone-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-clone\" score=\"INFINITY\"/>" > tmp_constraint
        cibadmin -o constraints -D -x tmp_constraint
        echo "<rsc_location id=\"lxc-ping-location-${tmp}\" node=\"${tmp}\" rsc=\"Connectivity\" score=\"-INFINITY\"/>" > tmp_constraint
        cibadmin -o constraints -D -x tmp_constraint

        for rsc in $(crm_resource -l | grep rsc_ ); do
            echo "<rsc_location id=\"lxc-${rsc}-location-${tmp}\" node=\"${tmp}\" rsc=\"${rsc}\" score=\"-INFINITY\"/>" > tmp_constraint
            cibadmin -o constraints -D -x tmp_constraint
        done
        rm -f tmp_constraint
    done
    cibadmin -o resources -D -x lxc-clone.cib

    for tmp in container*.cib; do
        tmp="${tmp//\.cib/}" 
        crm_resource -U -r "$tmp" -H "$(this_node)"
        crm_resource -D -r "$tmp" -t primitive
    done
    # Make sure the version changes even if the content doesn't
    cibadmin -B
    unset CIB_file

    cibadmin --replace -o configuration --xml-file cur.cib
    rm -f  cur.cib 

    # Allow the cluster to stabilize before continuing
    crm_resource --wait

    # Purge nodes from caches and CIB status section
    for tmp in $(container_names); do
        crm_node --force --remove "$tmp"
    done
}

restore_network()
{
    NODE="$1"

    node_exec "$NODE" <<-EOF
    cd "$working_dir"
    for tmp in \$(ls lxc*.xml | sed -e 's/\.xml//g'); do
        virsh -c lxc:/// destroy "\$tmp" >/dev/null 2>&1
        virsh -c lxc:/// undefine "\$tmp" >/dev/null 2>&1
        sed -i.bak "/...\....\....\..* \${tmp}/d" /etc/hosts
    done
    virsh net-destroy default >/dev/null 2>&1
    virsh net-undefine default >/dev/null 2>&1
    if [ -f restore_default.xml ]; then
        virsh net-define restore_default.xml
        virsh net-start default
        rm restore_default.xml
    fi
EOF
    echo "Containers destroyed and default network restored on $NODE"
}

restore_libvirt()
{
    restore_network "$(this_node)"
    if [ $share_configs -eq 1 ]; then
        for node in $(other_nodes); do
            restore_network "$node"
        done
    fi
}

restore_files()
{
    find . -maxdepth 1 -not -name "lxc*.xml" -a -not -name . -exec rm -rf "{}" ";"
    if [ $share_configs -eq 1 ]; then
        for node in $(other_nodes); do
            node_exec "$node" rm -rf \
                "$working_dir"/lxc*-filesystem \
                "$working_dir"/cur_network.xml
        done
    fi
}

make_directory "$working_dir"
cd "$working_dir" || exit 1

# Generate files as requested
if [ $download -eq 1 ]; then
    download_agent
fi
if [ $key_gen -eq 1 ]; then
    generate_key
fi
if [ $generate -eq 1 ]; then
    generate
fi
if [ $share_configs -eq 1 ]; then
    distribute_configs
fi
if [ $generate -eq 1 ]; then
    start_network_all
fi

# Update cluster as requested
if [ $cib -eq 1 ]; then
    apply_cib_entries
fi
if [ $add_clone -eq 1 ]; then
    apply_cib_clone
fi

# Restore original state as requested
if [ $restore_pcmk -eq 1 ]; then
    restore_cib
fi
if [ $restore -eq 1 ]; then
    restore_libvirt
fi
if [ $restore_all -eq 1 ]; then
    restore_files
fi

# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80: