summaryrefslogtreecommitdiffstats
path: root/cts/lxc_autogen.sh.in
diff options
context:
space:
mode:
Diffstat (limited to 'cts/lxc_autogen.sh.in')
-rw-r--r--cts/lxc_autogen.sh.in545
1 files changed, 545 insertions, 0 deletions
diff --git a/cts/lxc_autogen.sh.in b/cts/lxc_autogen.sh.in
new file mode 100644
index 0000000..195d3f9
--- /dev/null
+++ b/cts/lxc_autogen.sh.in
@@ -0,0 +1,545 @@
+#!@BASH_PATH@
+#
+# Copyright 2013-2022 the Pacemaker project contributors
+#
+# The version control history for this file may have further details.
+#
+# This source code is licensed under the GNU General Public License version 2
+# or later (GPLv2+) WITHOUT ANY WARRANTY.
+#
+
+containers="2"
+download=0
+share_configs=0
+# different than default libvirt network in case this is run nested in a KVM instance
+addr="192.168.123.1"
+restore=0
+restore_pcmk=0
+restore_all=0
+generate=0
+key_gen=0
+cib=0
+anywhere=0
+add_clone=0
+verify=0
+working_dir="@CRM_PACEMAKER_DIR@/cts/lxc"
+run_dirs="/run /var/run /usr/var/run"
+
+# must be on one line b/c used inside quotes
+SSH_RSYNC_OPTS="-o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no"
+
+function helptext() {
+ echo "lxc_autogen.sh - generate libvirt LXC containers for testing purposes"
+ echo ""
+ echo "Usage: lxc-autogen [options]"
+ echo ""
+ echo "Options:"
+ echo "-g, --generate Generate libvirt LXC environment in directory this script is run from"
+ echo "-k, --key-gen Generate Pacemaker Remote key only"
+ echo "-r, --restore-libvirt Restore the default network and libvirt config to before this script ran"
+ echo "-p, --restore-cib Remove CIB entries this script generated"
+ echo "-R, --restore-all Restore both libvirt and CIB, and clean working directory"
+ echo " (libvirt xml files are not removed, so resource can be stopped properly)"
+ echo ""
+ echo "-A, --allow-anywhere Allow the containers to live anywhere in the cluster"
+ echo "-a, --add-cib Add CIB entries to create a guest node for each LXC instance"
+ echo "-C, --add-clone Add promotable clone resource shared between LXC guest nodes"
+ echo "-d, --download-agent Download and install latest VirtualDomain agent"
+ echo "-s, --share-configs Synchronize on all known cluster nodes"
+ echo "-c, --containers Specify number of containers to generate (default $containers; used with -g)"
+ echo "-n, --network Network to override libvirt default (example: -n 192.168.123.1; used with -g)"
+ echo "-v, --verify Verify environment is capable of running LXC"
+ echo ""
+ exit "$1"
+}
+
+while true ; do
+ case "$1" in
+ --help|-h|-\?) helptext 0;;
+ -c|--containers) containers="$2"; shift; shift;;
+ -d|--download-agent) download=1; shift;;
+ -s|--share-configs) share_configs=1; shift;;
+ -n|--network) addr="$2"; shift; shift;;
+ -r|--restore-libvirt) restore=1; shift;;
+ -p|--restore-cib) restore_pcmk=1; shift;;
+ -R|--restore-all)
+ restore_all=1
+ restore=1
+ restore_pcmk=1
+ shift;;
+ -g|--generate) generate=1; key_gen=1; shift;;
+ -k|--key-gen) key_gen=1; shift;;
+ -a|--add-cib) cib=1; shift;;
+ -A|--allow-anywhere) anywhere=1; shift;;
+ -C|--add-clone) add_clone=1; shift;;
+ -m|--add-master)
+ echo "$1 is deprecated (use -C/--add-clone instead)"
+ echo
+ add_clone=1
+ shift
+ ;;
+ -v|--verify) verify=1; shift;;
+ "") break;;
+ *) helptext 1;;
+ esac
+done
+
+if [ $verify -eq 1 ]; then
+ # verify virsh tool is available and that
+ # we can connect to lxc driver.
+ virsh -c lxc:/// list --all > /dev/null 2>&1
+ if [ $? -ne 0 ]; then
+ echo "libvirt LXC driver must be installed (could not connect 'virsh -c lxc:///')"
+ # yum install -y libvirt-daemon-driver-lxc libvirt-daemon-lxc libvirt-login-shell
+ exit 1
+ fi
+
+ SELINUX=$(getenforce)
+ if [ "$SELINUX" != "Enforcing" ] && [ "$SELINUX" != "Permissive" ]; then
+ echo "SELINUX must be set to permissive or enforcing mode"
+ exit 1
+ fi
+
+ ps ax | grep "[l]ibvirtd"
+ if [ $? -ne 0 ]; then
+ echo "libvirtd must be running"
+ exit 1
+ fi
+
+ which rsync > /dev/null 2>&1
+ if [ $? -ne 0 ]; then
+ echo "rsync must be installed"
+ fi
+
+ which pacemaker-remoted > /dev/null 2>&1
+ if [ $? -ne 0 ]; then
+ echo "pacemaker-remoted must be installed"
+ fi
+fi
+
+#strip last digits off addr
+addr="$(echo "$addr" | awk -F. '{print $1"."$2"."$3}')"
+
+node_exec() {
+ ssh -o StrictHostKeyChecking=no \
+ -o ConnectTimeout=30 \
+ -o BatchMode=yes \
+ -l root -T "$@"
+}
+
+this_node()
+{
+ crm_node -n
+}
+
+other_nodes()
+{
+ crm_node -l | awk "\$2 != \"$(this_node)\" {print \$2}"
+}
+
+make_directory()
+{
+ # argument must be full path
+ DIR="$1"
+
+ mkdir -p "$DIR"
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ node_exec "$node" mkdir -p "$DIR"
+ done
+ fi
+}
+
+sync_file()
+{
+ TARGET="$1"
+
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ rsync -ave "ssh $SSH_RSYNC_OPTS" "$TARGET" "${node}:${TARGET}"
+ done
+ fi
+}
+
+download_agent()
+{
+ wget https://raw.github.com/ClusterLabs/resource-agents/main/heartbeat/VirtualDomain
+ chmod 755 VirtualDomain
+ mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
+ sync_file /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
+}
+
+set_network()
+{
+ rm -f cur_network.xml
+ cat << END >> cur_network.xml
+<network>
+ <name>default</name>
+ <uuid>41ebdb84-7134-1111-a136-91f0f1119225</uuid>
+ <forward mode='nat'/>
+ <bridge name='virbr0' stp='on' delay='0' />
+ <mac address='52:54:00:A8:12:35'/>
+ <ip address='$addr.1' netmask='255.255.255.0'>
+ <dhcp>
+ <range start='$addr.2' end='$addr.254' />
+ </dhcp>
+ </ip>
+</network>
+END
+ sync_file "${working_dir}"/cur_network.xml
+}
+
+distribute_configs()
+{
+ for node in $(other_nodes); do
+ rsync -ave "ssh $SSH_RSYNC_OPTS" "${working_dir}"/lxc*.xml "${node}:${working_dir}"
+ rsync -ave "ssh $SSH_RSYNC_OPTS" "${working_dir}"/lxc*-filesystem "${node}:${working_dir}"
+ done
+}
+
+start_network()
+{
+ NODE="$1"
+
+ node_exec "$NODE" <<-EOF
+ cd "$working_dir"
+ virsh net-info default >/dev/null 2>&1
+ if [ \$? -eq 0 ]; then
+ if [ ! -f restore_default.xml ]; then
+ virsh net-dumpxml default > restore_default.xml
+ fi
+ virsh net-destroy default
+ virsh net-undefine default
+ fi
+ virsh net-define cur_network.xml
+ virsh net-start default
+ virsh net-autostart default
+EOF
+}
+
+start_network_all()
+{
+ start_network "$(this_node)"
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ start_network "$node"
+ done
+ fi
+}
+
+add_hosts_entry()
+{
+ IP="$1"
+ HNAME="$2"
+
+ echo "$IP $HNAME" >>/etc/hosts
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ node_exec "$node" "echo $IP $HNAME >>/etc/hosts"
+ done
+ fi
+}
+
+generate_key()
+{
+ if [ ! -e /etc/pacemaker/authkey ]; then
+ make_directory /etc/pacemaker
+ dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1
+ sync_file /etc/pacemaker/authkey
+ fi
+}
+
+generate()
+{
+ set_network
+
+ # Generate libvirt domains in xml
+ for (( c=1; c <= containers; c++ ))
+ do
+ # Clean any previous definition
+ rm -rf "lxc$c.xml" "lxc$c-filesystem"
+
+ # Create a basic filesystem with run directories
+ for dir in $run_dirs; do
+ mkdir -p "lxc$c-filesystem/$dir"
+ done
+
+ # Create libvirt definition
+ suffix=$((10 + c))
+ prefix="$(echo "$addr" | awk -F. '{print $1"."$2}')"
+ subnet="$(echo "$addr" | awk -F. '{print $3}')"
+ while [ $suffix -gt 255 ]; do
+ subnet=$((subnet + 1))
+ suffix=$((subnet - 255))
+ done
+ cip="$prefix.$subnet.$suffix"
+
+ cat << END >> lxc$c.xml
+<domain type='lxc'>
+ <name>lxc$c</name>
+ <memory unit='KiB'>200704</memory>
+ <os>
+ <type>exe</type>
+ <init>$working_dir/lxc$c-filesystem/launch-helper</init>
+ </os>
+ <devices>
+ <console type='pty'/>
+ <filesystem type='ram'>
+ <source usage='150528'/>
+ <target dir='/dev/shm'/>
+ </filesystem>
+END
+ for dir in $run_dirs; do
+ cat << END >> lxc$c.xml
+ <filesystem type='mount'>
+ <source dir='$working_dir/lxc$c-filesystem${dir}'/>
+ <target dir='$dir'/>
+ </filesystem>
+END
+ done
+ cat << END >> lxc$c.xml
+ <interface type='network'>
+ <mac address='52:54:$((RANDOM % 9))$((RANDOM % 9)):$((RANDOM % 9))$((RANDOM % 9)):$((RANDOM % 9))$((RANDOM % 9)):$((RANDOM % 9))$((RANDOM % 9))'/>
+ <source network='default'/>
+ </interface>
+ </devices>
+</domain>
+END
+
+ # Create CIB definition
+ rm -f "container$c.cib"
+ cat << END >> "container$c.cib"
+ <primitive class="ocf" id="container$c" provider="heartbeat" type="VirtualDomain">
+ <instance_attributes id="container$c-instance_attributes">
+ <nvpair id="container$c-instance_attributes-force_stop" name="force_stop" value="true"/>
+ <nvpair id="container$c-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
+ <nvpair id="container$c-instance_attributes-config" name="config" value="$working_dir/lxc$c.xml"/>
+ </instance_attributes>
+ <utilization id="container$c-utilization">
+ <nvpair id="container$c-utilization-cpu" name="cpu" value="1"/>
+ <nvpair id="container$c-utilization-hv_memory" name="hv_memory" value="100"/>
+ </utilization>
+ <meta_attributes id="container$c-meta_attributes">
+ <nvpair id="container$c-meta_attributes-remote-node" name="remote-node" value="lxc$c"/>
+ </meta_attributes>
+ <operations>
+ <op id="container$c-monitor-20s" interval="20s" name="monitor"/>
+ </operations>
+ </primitive>
+END
+
+ # Create container init
+ rm -f "lxc$c-filesystem/launch-helper"
+ cat << END >> "lxc$c-filesystem/launch-helper"
+#!@BASH_PATH@
+ip -f inet addr add "$cip/24" dev eth0
+ip link set eth0 up
+ip route add default via "$addr.1"
+hostname "lxc$c"
+df > "$working_dir/lxc$c-filesystem/disk_usage.txt"
+export PCMK_debugfile="@CRM_LOG_DIR@/pacemaker_remote_lxc$c.log"
+/usr/sbin/pacemaker-remoted
+END
+ chmod 711 "lxc$c-filesystem/launch-helper"
+
+ add_hosts_entry "$cip" "lxc$c"
+ done
+
+ # Create CIB fragment for a promotable clone resource
+ cat << END > lxc-clone.cib
+ <clone id="lxc-clone">
+ <primitive class="ocf" id="lxc-rsc" provider="pacemaker" type="Stateful">
+ <instance_attributes id="lxc-rsc-instance_attributes"/>
+ <operations>
+ <op id="lxc-rsc-monitor-interval-10s" interval="10s" name="monitor" role="Promoted" timeout="20s"/>
+ <op id="lxc-rsc-monitor-interval-11s" interval="11s" name="monitor" role="Unpromoted" timeout="20s"/>
+ </operations>
+ </primitive>
+ <meta_attributes id="lxc-clone-meta_attributes">
+ <nvpair id="lxc-clone-meta_attributes-promotable" name="promotable" value="true"/>
+ <nvpair id="lxc-clone-meta_attributes-promoted-max" name="promoted-max" value="1"/>
+ <nvpair id="lxc-clone-meta_attributes-clone-max" name="clone-max" value="$containers"/>
+ </meta_attributes>
+ </clone>
+END
+}
+
+container_names() {
+ find . -maxdepth 1 -name "lxc*.xml" -exec basename -s .xml "{}" ";"
+}
+
+apply_cib_clone()
+{
+ cibadmin -Q > cur.cib
+ export CIB_file=cur.cib
+
+ cibadmin -o resources -Mc -x lxc-clone.cib
+ for tmp in $(container_names); do
+ echo "<rsc_location id=\"lxc-clone-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-clone\" score=\"INFINITY\"/>" > tmp_constraint
+ cibadmin -o constraints -Mc -x tmp_constraint
+ done
+ # Make sure the version changes even if the content doesn't
+ cibadmin -B
+ unset CIB_file
+
+ cibadmin --replace -o configuration --xml-file cur.cib
+ rm -f cur.cib
+}
+
+apply_cib_entries()
+{
+ cibadmin -Q > cur.cib
+ export CIB_file=cur.cib
+ for tmp in container*.cib; do
+ cibadmin -o resources -Mc -x "$tmp"
+
+ remote_node="$(grep remote-node "${tmp}" | sed -n -e 's/^.*value=\"\(.*\)\".*/\1/p')"
+ if [ $anywhere -eq 0 ]; then
+ crm_resource -M -r "${tmp//\.cib/}" -H "$(this_node)"
+ fi
+ echo "<rsc_location id=\"lxc-ping-location-${remote_node}\" node=\"${remote_node}\" rsc=\"Connectivity\" score=\"-INFINITY\"/>" > tmp_constraint
+ # Ignore any failure; this constraint is just to help with CTS when the
+ # connectivity resources (which fail the guest nodes) are in use.
+ cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1
+
+ for rsc in $(crm_resource -l | grep rsc_ ); do
+ echo "<rsc_location id=\"lxc-${rsc}-location-${remote_node}\" node=\"${remote_node}\" rsc=\"${rsc}\" score=\"-INFINITY\"/>" > tmp_constraint
+ cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1
+ done
+
+ rm -f tmp_constraint
+ done
+
+ # Make sure the version changes even if the content doesn't
+ cibadmin -B
+
+ unset CIB_file
+
+ cibadmin --replace -o configuration --xml-file cur.cib
+ rm -f cur.cib
+}
+
+restore_cib()
+{
+ cibadmin -Q > cur.cib
+ export CIB_file=cur.cib
+
+ for tmp in $(container_names); do
+ echo "<rsc_location id=\"lxc-clone-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-clone\" score=\"INFINITY\"/>" > tmp_constraint
+ cibadmin -o constraints -D -x tmp_constraint
+ echo "<rsc_location id=\"lxc-ping-location-${tmp}\" node=\"${tmp}\" rsc=\"Connectivity\" score=\"-INFINITY\"/>" > tmp_constraint
+ cibadmin -o constraints -D -x tmp_constraint
+
+ for rsc in $(crm_resource -l | grep rsc_ ); do
+ echo "<rsc_location id=\"lxc-${rsc}-location-${tmp}\" node=\"${tmp}\" rsc=\"${rsc}\" score=\"-INFINITY\"/>" > tmp_constraint
+ cibadmin -o constraints -D -x tmp_constraint
+ done
+ rm -f tmp_constraint
+ done
+ cibadmin -o resources -D -x lxc-clone.cib
+
+ for tmp in container*.cib; do
+ tmp="${tmp//\.cib/}"
+ crm_resource -U -r "$tmp" -H "$(this_node)"
+ crm_resource -D -r "$tmp" -t primitive
+ done
+ # Make sure the version changes even if the content doesn't
+ cibadmin -B
+ unset CIB_file
+
+ cibadmin --replace -o configuration --xml-file cur.cib
+ rm -f cur.cib
+
+ # Allow the cluster to stabilize before continuing
+ crm_resource --wait
+
+ # Purge nodes from caches and CIB status section
+ for tmp in $(container_names); do
+ crm_node --force --remove "$tmp"
+ done
+}
+
+restore_network()
+{
+ NODE="$1"
+
+ node_exec "$NODE" <<-EOF
+ cd "$working_dir"
+ for tmp in \$(ls lxc*.xml | sed -e 's/\.xml//g'); do
+ virsh -c lxc:/// destroy "\$tmp" >/dev/null 2>&1
+ virsh -c lxc:/// undefine "\$tmp" >/dev/null 2>&1
+ sed -i.bak "/...\....\....\..* \${tmp}/d" /etc/hosts
+ done
+ virsh net-destroy default >/dev/null 2>&1
+ virsh net-undefine default >/dev/null 2>&1
+ if [ -f restore_default.xml ]; then
+ virsh net-define restore_default.xml
+ virsh net-start default
+ rm restore_default.xml
+ fi
+EOF
+ echo "Containers destroyed and default network restored on $NODE"
+}
+
+restore_libvirt()
+{
+ restore_network "$(this_node)"
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ restore_network "$node"
+ done
+ fi
+}
+
+restore_files()
+{
+ find . -maxdepth 1 -not -name "lxc*.xml" -a -not -name . -exec rm -rf "{}" ";"
+ if [ $share_configs -eq 1 ]; then
+ for node in $(other_nodes); do
+ node_exec "$node" rm -rf \
+ "$working_dir"/lxc*-filesystem \
+ "$working_dir"/cur_network.xml
+ done
+ fi
+}
+
+make_directory "$working_dir"
+cd "$working_dir" || exit 1
+
+# Generate files as requested
+if [ $download -eq 1 ]; then
+ download_agent
+fi
+if [ $key_gen -eq 1 ]; then
+ generate_key
+fi
+if [ $generate -eq 1 ]; then
+ generate
+fi
+if [ $share_configs -eq 1 ]; then
+ distribute_configs
+fi
+if [ $generate -eq 1 ]; then
+ start_network_all
+fi
+
+# Update cluster as requested
+if [ $cib -eq 1 ]; then
+ apply_cib_entries
+fi
+if [ $add_clone -eq 1 ]; then
+ apply_cib_clone
+fi
+
+# Restore original state as requested
+if [ $restore_pcmk -eq 1 ]; then
+ restore_cib
+fi
+if [ $restore -eq 1 ]; then
+ restore_libvirt
+fi
+if [ $restore_all -eq 1 ]; then
+ restore_files
+fi
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80:
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-21 17:03:57 +0000