diff options
Diffstat (limited to 'qa/workunits/rbd/rbd_mirror_helpers.sh')
| -rwxr-xr-x | qa/workunits/rbd/rbd_mirror_helpers.sh | 1488 |
1 files changed, 1488 insertions, 0 deletions
diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh new file mode 100755 index 000000000..f4961b925 --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -0,0 +1,1488 @@ +#!/bin/sh +# +# rbd_mirror_helpers.sh - shared rbd-mirror daemon helper functions +# +# The scripts starts two ("local" and "remote") clusters using mstart.sh script, +# creates a temporary directory, used for cluster configs, daemon logs, admin +# socket, temporary files, and launches rbd-mirror daemon. +# +# There are several env variables useful when troubleshooting a test failure: +# +# RBD_MIRROR_NOCLEANUP - if not empty, don't run the cleanup (stop processes, +# destroy the clusters and remove the temp directory) +# on exit, so it is possible to check the test state +# after failure. +# RBD_MIRROR_TEMDIR - use this path when creating the temporary directory +# (should not exist) instead of running mktemp(1). +# RBD_MIRROR_ARGS - use this to pass additional arguments to started +# rbd-mirror daemons. +# RBD_MIRROR_VARGS - use this to pass additional arguments to vstart.sh +# when starting clusters. +# RBD_MIRROR_INSTANCES - number of daemons to start per cluster +# RBD_MIRROR_CONFIG_KEY - if not empty, use config-key for remote cluster +# secrets +# The cleanup can be done as a separate step, running the script with +# `cleanup ${RBD_MIRROR_TEMDIR}' arguments. +# +# Note, as other workunits tests, rbd_mirror_journal.sh expects to find ceph binaries +# in PATH. +# +# Thus a typical troubleshooting session: +# +# From Ceph src dir (CEPH_SRC_PATH), start the test in NOCLEANUP mode and with +# TEMPDIR pointing to a known location: +# +# cd $CEPH_SRC_PATH +# PATH=$CEPH_SRC_PATH:$PATH +# RBD_MIRROR_NOCLEANUP=1 RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ +# ../qa/workunits/rbd/rbd_mirror_journal.sh +# +# After the test failure cd to TEMPDIR and check the current state: +# +# cd /tmp/tmp.rbd_mirror +# ls +# less rbd-mirror.cluster1_daemon.$pid.log +# ceph --cluster cluster1 -s +# ceph --cluster cluster1 -s +# rbd --cluster cluster2 -p mirror ls +# rbd --cluster cluster2 -p mirror journal status --image test +# ceph --admin-daemon rbd-mirror.cluster1_daemon.cluster1.$pid.asok help +# ... +# +# Also you can execute commands (functions) from the script: +# +# cd $CEPH_SRC_PATH +# export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror +# ../qa/workunits/rbd/rbd_mirror_journal.sh status +# ../qa/workunits/rbd/rbd_mirror_journal.sh stop_mirror cluster1 +# ../qa/workunits/rbd/rbd_mirror_journal.sh start_mirror cluster2 +# ../qa/workunits/rbd/rbd_mirror_journal.sh flush cluster2 +# ... +# +# Eventually, run the cleanup: +# +# cd $CEPH_SRC_PATH +# RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ +# ../qa/workunits/rbd/rbd_mirror_journal.sh cleanup +# + +if type xmlstarlet > /dev/null 2>&1; then + XMLSTARLET=xmlstarlet +elif type xml > /dev/null 2>&1; then + XMLSTARLET=xml +else + echo "Missing xmlstarlet binary!" + exit 1 +fi + +RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-2} + +CLUSTER1=cluster1 +CLUSTER2=cluster2 +PEER_CLUSTER_SUFFIX= +POOL=mirror +PARENT_POOL=mirror_parent +NS1=ns1 +NS2=ns2 +TEMPDIR= +CEPH_ID=${CEPH_ID:-mirror} +RBD_IMAGE_FEATURES=${RBD_IMAGE_FEATURES:-layering,exclusive-lock,journaling} +MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.} +MIRROR_POOL_MODE=${MIRROR_POOL_MODE:-pool} +MIRROR_IMAGE_MODE=${MIRROR_IMAGE_MODE:-journal} + +export CEPH_ARGS="--id ${CEPH_ID}" + +LAST_MIRROR_INSTANCE=$((${RBD_MIRROR_INSTANCES} - 1)) + +CEPH_ROOT=$(readlink -f $(dirname $0)/../../../src) +CEPH_BIN=. +CEPH_SRC=. +if [ -e CMakeCache.txt ]; then + CEPH_SRC=${CEPH_ROOT} + CEPH_ROOT=${PWD} + CEPH_BIN=./bin + + # needed for ceph CLI under cmake + export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH} + export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind:${CEPH_ROOT}/lib/cython_modules/lib.3 +fi + +# These vars facilitate running this script in an environment with +# ceph installed from packages, like teuthology. These are not defined +# by default. +# +# RBD_MIRROR_USE_EXISTING_CLUSTER - if set, do not start and stop ceph clusters +# RBD_MIRROR_USE_RBD_MIRROR - if set, use an existing instance of rbd-mirror +# running as ceph client $CEPH_ID. If empty, +# this script will start and stop rbd-mirror + +# +# Functions +# + +# Parse a value in format cluster[:instance] and set cluster and instance vars. +set_cluster_instance() +{ + local val=$1 + local cluster_var_name=$2 + local instance_var_name=$3 + + cluster=${val%:*} + instance=${val##*:} + + if [ "${instance}" = "${val}" ]; then + # instance was not specified, use default + instance=0 + fi + + eval ${cluster_var_name}=${cluster} + eval ${instance_var_name}=${instance} +} + +daemon_asok_file() +{ + local local_cluster=$1 + local cluster=$2 + local instance + + set_cluster_instance "${local_cluster}" local_cluster instance + + echo $(ceph-conf --cluster $local_cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'admin socket') +} + +daemon_pid_file() +{ + local cluster=$1 + local instance + + set_cluster_instance "${cluster}" cluster instance + + echo $(ceph-conf --cluster $cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'pid file') +} + +testlog() +{ + echo $(date '+%F %T') $@ | tee -a "${TEMPDIR}/rbd-mirror.test.log" >&2 +} + +expect_failure() +{ + local expected="$1" ; shift + local out=${TEMPDIR}/expect_failure.out + + if "$@" > ${out} 2>&1 ; then + cat ${out} >&2 + return 1 + fi + + if [ -z "${expected}" ]; then + return 0 + fi + + if ! grep -q "${expected}" ${out} ; then + cat ${out} >&2 + return 1 + fi + + return 0 +} + +mkfname() +{ + echo "$@" | sed -e 's|[/ ]|_|g' +} + +create_users() +{ + local cluster=$1 + + CEPH_ARGS='' ceph --cluster "${cluster}" \ + auth get-or-create client.${CEPH_ID} \ + mon 'profile rbd' osd 'profile rbd' mgr 'profile rbd' >> \ + ${CEPH_ROOT}/run/${cluster}/keyring + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + CEPH_ARGS='' ceph --cluster "${cluster}" \ + auth get-or-create client.${MIRROR_USER_ID_PREFIX}${instance} \ + mon 'profile rbd-mirror' osd 'profile rbd' mgr 'profile rbd' >> \ + ${CEPH_ROOT}/run/${cluster}/keyring + done +} + +setup_cluster() +{ + local cluster=$1 + + CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${cluster} -n ${RBD_MIRROR_VARGS} + + cd ${CEPH_ROOT} + rm -f ${TEMPDIR}/${cluster}.conf + ln -s $(readlink -f run/${cluster}/ceph.conf) \ + ${TEMPDIR}/${cluster}.conf + + cd ${TEMPDIR} + create_users "${cluster}" + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + cat<<EOF >> ${TEMPDIR}/${cluster}.conf +[client.${MIRROR_USER_ID_PREFIX}${instance}] + admin socket = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.asok + pid file = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.pid + log file = ${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.log +EOF + done +} + +peer_add() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local client_cluster=$1 ; shift + local remote_cluster="${client_cluster##*@}" + + local uuid_var_name + if [ -n "$1" ]; then + uuid_var_name=$1 ; shift + fi + + local error_code + local peer_uuid + + for s in 1 2 4 8 16 32; do + set +e + peer_uuid=$(rbd --cluster ${cluster} mirror pool peer add \ + ${pool} ${client_cluster} $@) + error_code=$? + set -e + + if [ $error_code -eq 17 ]; then + # raced with a remote heartbeat ping -- remove and retry + sleep $s + peer_uuid=$(rbd mirror pool info --cluster ${cluster} --pool ${pool} --format xml | \ + xmlstarlet sel -t -v "//peers/peer[site_name='${remote_cluster}']/uuid") + + CEPH_ARGS='' rbd --cluster ${cluster} --pool ${pool} mirror pool peer remove ${peer_uuid} + else + test $error_code -eq 0 + if [ -n "$uuid_var_name" ]; then + eval ${uuid_var_name}=${peer_uuid} + fi + return 0 + fi + done + + return 1 +} + +setup_pools() +{ + local cluster=$1 + local remote_cluster=$2 + local mon_map_file + local mon_addr + local admin_key_file + local uuid + + CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${POOL} 64 64 + CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${PARENT_POOL} 64 64 + + CEPH_ARGS='' rbd --cluster ${cluster} pool init ${POOL} + CEPH_ARGS='' rbd --cluster ${cluster} pool init ${PARENT_POOL} + + if [ -n "${RBD_MIRROR_CONFIG_KEY}" ]; then + PEER_CLUSTER_SUFFIX=-DNE + fi + + CEPH_ARGS='' rbd --cluster ${cluster} mirror pool enable \ + --site-name ${cluster}${PEER_CLUSTER_SUFFIX} ${POOL} ${MIRROR_POOL_MODE} + rbd --cluster ${cluster} mirror pool enable ${PARENT_POOL} image + + rbd --cluster ${cluster} namespace create ${POOL}/${NS1} + rbd --cluster ${cluster} namespace create ${POOL}/${NS2} + + rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS1} ${MIRROR_POOL_MODE} + rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS2} image + + if [ -z ${RBD_MIRROR_MANUAL_PEERS} ]; then + if [ -z ${RBD_MIRROR_CONFIG_KEY} ]; then + peer_add ${cluster} ${POOL} ${remote_cluster} + peer_add ${cluster} ${PARENT_POOL} ${remote_cluster} + else + mon_map_file=${TEMPDIR}/${remote_cluster}.monmap + CEPH_ARGS='' ceph --cluster ${remote_cluster} mon getmap > ${mon_map_file} + mon_addr=$(monmaptool --print ${mon_map_file} | grep -E 'mon\.' | + head -n 1 | sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' | sed -E 's/\/[0-9]+//g') + + admin_key_file=${TEMPDIR}/${remote_cluster}.client.${CEPH_ID}.key + CEPH_ARGS='' ceph --cluster ${remote_cluster} auth get-key client.${CEPH_ID} > ${admin_key_file} + + CEPH_ARGS='' peer_add ${cluster} ${POOL} \ + client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} '' \ + --remote-mon-host "${mon_addr}" --remote-key-file ${admin_key_file} + + peer_add ${cluster} ${PARENT_POOL} client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} uuid + CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr} + CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file} + fi + fi +} + +setup_tempdir() +{ + if [ -n "${RBD_MIRROR_TEMDIR}" ]; then + test -d "${RBD_MIRROR_TEMDIR}" || + mkdir "${RBD_MIRROR_TEMDIR}" + TEMPDIR="${RBD_MIRROR_TEMDIR}" + cd ${TEMPDIR} + else + TEMPDIR=`mktemp -d` + fi +} + +setup() +{ + local c + trap 'cleanup $?' INT TERM EXIT + + setup_tempdir + if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then + setup_cluster "${CLUSTER1}" + setup_cluster "${CLUSTER2}" + fi + + setup_pools "${CLUSTER1}" "${CLUSTER2}" + setup_pools "${CLUSTER2}" "${CLUSTER1}" + + if [ -n "${RBD_MIRROR_MIN_COMPAT_CLIENT}" ]; then + CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd \ + set-require-min-compat-client ${RBD_MIRROR_MIN_COMPAT_CLIENT} + CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd \ + set-require-min-compat-client ${RBD_MIRROR_MIN_COMPAT_CLIENT} + fi +} + +cleanup() +{ + local error_code=$1 + + set +e + + if [ "${error_code}" -ne 0 ]; then + status + fi + + if [ -z "${RBD_MIRROR_NOCLEANUP}" ]; then + local cluster instance + + CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it + CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it + CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it + CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it + + for cluster in "${CLUSTER1}" "${CLUSTER2}"; do + stop_mirrors "${cluster}" + done + + if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then + cd ${CEPH_ROOT} + CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER1} + CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER2} + fi + test "${RBD_MIRROR_TEMDIR}" = "${TEMPDIR}" || rm -Rf ${TEMPDIR} + fi + + if [ "${error_code}" -eq 0 ]; then + echo "OK" + else + echo "FAIL" + fi + + exit ${error_code} +} + +start_mirror() +{ + local cluster=$1 + local instance + + set_cluster_instance "${cluster}" cluster instance + + test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return + + rbd-mirror \ + --cluster ${cluster} \ + --id ${MIRROR_USER_ID_PREFIX}${instance} \ + --rbd-mirror-delete-retry-interval=5 \ + --rbd-mirror-image-state-check-interval=5 \ + --rbd-mirror-journal-poll-age=1 \ + --rbd-mirror-pool-replayers-refresh-interval=5 \ + --debug-rbd=30 --debug-journaler=30 \ + --debug-rbd_mirror=30 \ + --daemonize=true \ + ${RBD_MIRROR_ARGS} +} + +start_mirrors() +{ + local cluster=$1 + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + start_mirror "${cluster}:${instance}" + done +} + +stop_mirror() +{ + local cluster=$1 + local sig=$2 + + test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return + + local pid + pid=$(cat $(daemon_pid_file "${cluster}") 2>/dev/null) || : + if [ -n "${pid}" ] + then + kill ${sig} ${pid} + for s in 1 2 4 8 16 32; do + sleep $s + ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break + done + ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' + fi + rm -f $(daemon_asok_file "${cluster}" "${CLUSTER1}") + rm -f $(daemon_asok_file "${cluster}" "${CLUSTER2}") + rm -f $(daemon_pid_file "${cluster}") +} + +stop_mirrors() +{ + local cluster=$1 + local sig=$2 + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + stop_mirror "${cluster}:${instance}" "${sig}" + done +} + +admin_daemon() +{ + local cluster=$1 ; shift + local instance + + set_cluster_instance "${cluster}" cluster instance + + local asok_file=$(daemon_asok_file "${cluster}:${instance}" "${cluster}") + test -S "${asok_file}" + + ceph --admin-daemon ${asok_file} $@ +} + +admin_daemons() +{ + local cluster_instance=$1 ; shift + local cluster="${cluster_instance%:*}" + local instance="${cluster_instance##*:}" + local loop_instance + + for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16; do + sleep ${s} + if [ "${instance}" != "${cluster_instance}" ]; then + admin_daemon "${cluster}:${instance}" $@ && return 0 + else + for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + admin_daemon "${cluster}:${loop_instance}" $@ && return 0 + done + fi + done + return 1 +} + +all_admin_daemons() +{ + local cluster=$1 ; shift + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + admin_daemon "${cluster}:${instance}" $@ + done +} + +status() +{ + local cluster daemon image_pool image_ns image + + for cluster in ${CLUSTER1} ${CLUSTER2} + do + echo "${cluster} status" + CEPH_ARGS='' ceph --cluster ${cluster} -s + CEPH_ARGS='' ceph --cluster ${cluster} service dump + CEPH_ARGS='' ceph --cluster ${cluster} service status + echo + + for image_pool in ${POOL} ${PARENT_POOL} + do + for image_ns in "" "${NS1}" "${NS2}" + do + echo "${cluster} ${image_pool} ${image_ns} images" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls -l + echo + + echo "${cluster} ${image_pool}${image_ns} mirror pool info" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool info + echo + + echo "${cluster} ${image_pool}${image_ns} mirror pool status" + CEPH_ARGS='' rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool status --verbose + echo + + for image in `rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls 2>/dev/null` + do + echo "image ${image} info" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" info ${image} + echo + echo "image ${image} journal status" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" journal status --image ${image} + echo + echo "image ${image} snapshots" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" snap ls --all ${image} + echo + done + + echo "${cluster} ${image_pool} ${image_ns} rbd_mirroring omap vals" + rados --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" listomapvals rbd_mirroring + echo "${cluster} ${image_pool} ${image_ns} rbd_mirror_leader omap vals" + rados --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" listomapvals rbd_mirror_leader + echo + done + done + done + + local ret + + for cluster in "${CLUSTER1}" "${CLUSTER2}" + do + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + local pid_file=$(daemon_pid_file ${cluster}:${instance}) + if [ ! -e ${pid_file} ] + then + echo "${cluster} rbd-mirror not running or unknown" \ + "(${pid_file} not exist)" + continue + fi + + local pid + pid=$(cat ${pid_file} 2>/dev/null) || : + if [ -z "${pid}" ] + then + echo "${cluster} rbd-mirror not running or unknown" \ + "(can't find pid using ${pid_file})" + ret=1 + continue + fi + + echo "${daemon} rbd-mirror process in ps output:" + if ps auxww | + awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}' + then + echo + echo "${cluster} rbd-mirror not running" \ + "(can't find pid $pid in ps output)" + ret=1 + continue + fi + echo + + local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster}) + if [ ! -S "${asok_file}" ] + then + echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)" + ret=1 + continue + fi + + echo "${cluster} rbd-mirror status" + ceph --admin-daemon ${asok_file} rbd mirror status + echo + done + done + + return ${ret} +} + +flush() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local cmd="rbd mirror flush" + + if [ -n "${image}" ] + then + cmd="${cmd} ${pool}/${image}" + fi + + admin_daemons "${cluster}" ${cmd} +} + +test_image_replay_state() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local test_state=$4 + local status_result + local current_state=stopped + + status_result=$(admin_daemons "${cluster}" rbd mirror status ${pool}/${image} | grep -i 'state') || return 1 + echo "${status_result}" | grep -i 'Replaying' && current_state=started + test "${test_state}" = "${current_state}" +} + +wait_for_image_replay_state() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state=$4 + local s + + # TODO: add a way to force rbd-mirror to update replayers + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do + sleep ${s} + test_image_replay_state "${cluster}" "${pool}" "${image}" "${state}" && return 0 + done + return 1 +} + +wait_for_image_replay_started() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + wait_for_image_replay_state "${cluster}" "${pool}" "${image}" started +} + +wait_for_image_replay_stopped() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + wait_for_image_replay_state "${cluster}" "${pool}" "${image}" stopped +} + +get_journal_position() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local id_regexp=$4 + + # Parse line like below, looking for the first position + # [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=9], [object_number=0, tag_tid=3, entry_tid=8], [object_number=3, tag_tid=3, entry_tid=7], [object_number=2, tag_tid=3, entry_tid=6]]]] + + local status_log=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.status) + rbd --cluster ${cluster} journal status --image ${pool}/${image} | + tee ${status_log} >&2 + sed -nEe 's/^.*\[id='"${id_regexp}"',.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' \ + ${status_log} +} + +get_master_journal_position() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + get_journal_position "${cluster}" "${pool}" "${image}" '' +} + +get_mirror_journal_position() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + get_journal_position "${cluster}" "${pool}" "${image}" '..*' +} + +wait_for_journal_replay_complete() +{ + local local_cluster=$1 + local cluster=$2 + local pool=$3 + local image=$4 + local s master_pos mirror_pos last_mirror_pos + local master_tag master_entry mirror_tag mirror_entry + + while true; do + for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do + sleep ${s} + flush "${local_cluster}" "${pool}" "${image}" + master_pos=$(get_master_journal_position "${cluster}" "${pool}" "${image}") + mirror_pos=$(get_mirror_journal_position "${cluster}" "${pool}" "${image}") + test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 + test "${mirror_pos}" != "${last_mirror_pos}" && break + done + + test "${mirror_pos}" = "${last_mirror_pos}" && return 1 + last_mirror_pos="${mirror_pos}" + + # handle the case where the mirror is ahead of the master + master_tag=$(echo "${master_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) + mirror_tag=$(echo "${mirror_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) + master_entry=$(echo "${master_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) + mirror_entry=$(echo "${mirror_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) + test "${master_tag}" = "${mirror_tag}" -a ${master_entry} -le ${mirror_entry} && return 0 + done + return 1 +} + +mirror_image_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster "${cluster}" mirror image snapshot "${pool}/${image}" +} + +get_newest_mirror_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local log=$4 + + rbd --cluster "${cluster}" snap list --all "${pool}/${image}" --format xml | \ + xmlstarlet sel -t -c "//snapshots/snapshot[namespace/complete='true' and position()=last()]" > \ + ${log} || true +} + +wait_for_snapshot_sync_complete() +{ + local local_cluster=$1 + local cluster=$2 + local pool=$3 + local image=$4 + + local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.status) + local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${pool}-${image}.status) + + mirror_image_snapshot "${cluster}" "${pool}" "${image}" + get_newest_mirror_snapshot "${cluster}" "${pool}" "${image}" "${status_log}" + local snapshot_id=$(xmlstarlet sel -t -v "//snapshot/id" < ${status_log}) + + while true; do + for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do + sleep ${s} + + get_newest_mirror_snapshot "${local_cluster}" "${pool}" "${image}" "${local_status_log}" + local primary_snapshot_id=$(xmlstarlet sel -t -v "//snapshot/namespace/primary_snap_id" < ${local_status_log}) + + test "${snapshot_id}" = "${primary_snapshot_id}" && return 0 + done + + return 1 + done + return 1 +} + +wait_for_replay_complete() +{ + local local_cluster=$1 + local cluster=$2 + local pool=$3 + local image=$4 + + if [ "${MIRROR_IMAGE_MODE}" = "journal" ]; then + wait_for_journal_replay_complete ${local_cluster} ${cluster} ${pool} ${image} + elif [ "${MIRROR_IMAGE_MODE}" = "snapshot" ]; then + wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${pool} ${image} + else + return 1 + fi +} + + +test_status_in_pool_dir() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state_pattern="$4" + local description_pattern="$5" + local service_pattern="$6" + + local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.mirror_status) + CEPH_ARGS='' rbd --cluster ${cluster} mirror image status ${pool}/${image} | + tee ${status_log} >&2 + grep "^ state: .*${state_pattern}" ${status_log} || return 1 + grep "^ description: .*${description_pattern}" ${status_log} || return 1 + + if [ -n "${service_pattern}" ]; then + grep "service: *${service_pattern}" ${status_log} || return 1 + elif echo ${state_pattern} | grep '^up+'; then + grep "service: *${MIRROR_USER_ID_PREFIX}.* on " ${status_log} || return 1 + else + grep "service: " ${status_log} && return 1 + fi + + # recheck using `mirror pool status` command to stress test it. + + local last_update="$(sed -nEe 's/^ last_update: *(.*) *$/\1/p' ${status_log})" + test_mirror_pool_status_verbose \ + ${cluster} ${pool} ${image} "${state_pattern}" "${last_update}" && + return 0 + + echo "'mirror pool status' test failed" >&2 + exit 1 +} + +test_mirror_pool_status_verbose() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state_pattern="$4" + local prev_last_update="$5" + + local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}.mirror_status) + + rbd --cluster ${cluster} mirror pool status ${pool} --verbose --format xml \ + > ${status_log} + + local last_update state + last_update=$($XMLSTARLET sel -t -v \ + "//images/image[name='${image}']/last_update" < ${status_log}) + state=$($XMLSTARLET sel -t -v \ + "//images/image[name='${image}']/state" < ${status_log}) + + echo "${state}" | grep "${state_pattern}" || + test "${last_update}" '>' "${prev_last_update}" +} + +wait_for_status_in_pool_dir() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state_pattern="$4" + local description_pattern="$5" + local service_pattern="$6" + + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do + sleep ${s} + test_status_in_pool_dir ${cluster} ${pool} ${image} "${state_pattern}" \ + "${description_pattern}" "${service_pattern}" && + return 0 + done + return 1 +} + +create_image() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local image=$1 ; shift + local size=128 + + if [ -n "$1" ]; then + size=$1 + shift + fi + + rbd --cluster ${cluster} create --size ${size} \ + --image-feature "${RBD_IMAGE_FEATURES}" $@ ${pool}/${image} +} + +create_image_and_enable_mirror() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local image=$1 ; shift + local mode=${1:-${MIRROR_IMAGE_MODE}} + if [ -n "$1" ]; then + shift + fi + + create_image ${cluster} ${pool} ${image} $@ + if [ "${MIRROR_POOL_MODE}" = "image" ] || [ "$pool" = "${PARENT_POOL}" ]; then + enable_mirror ${cluster} ${pool} ${image} ${mode} + fi +} + +enable_journaling() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} feature enable ${pool}/${image} journaling +} + +set_image_meta() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local key=$4 + local val=$5 + + rbd --cluster ${cluster} image-meta set ${pool}/${image} $key $val +} + +compare_image_meta() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local key=$4 + local value=$5 + + test `rbd --cluster ${cluster} image-meta get ${pool}/${image} ${key}` = "${value}" +} + +rename_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local new_name=$4 + + rbd --cluster=${cluster} rename ${pool}/${image} ${pool}/${new_name} +} + +remove_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} snap purge ${pool}/${image} + rbd --cluster=${cluster} rm ${pool}/${image} +} + +remove_image_retry() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + for s in 0 1 2 4 8 16 32; do + sleep ${s} + remove_image ${cluster} ${pool} ${image} && return 0 + done + return 1 +} + +trash_move() { + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} trash move ${pool}/${image} +} + +trash_restore() { + local cluster=$1 + local pool=$2 + local image_id=$3 + + rbd --cluster=${cluster} trash restore ${pool}/${image_id} +} + +clone_image() +{ + local cluster=$1 + local parent_pool=$2 + local parent_image=$3 + local parent_snap=$4 + local clone_pool=$5 + local clone_image=$6 + + shift 6 + + rbd --cluster ${cluster} clone \ + ${parent_pool}/${parent_image}@${parent_snap} \ + ${clone_pool}/${clone_image} --image-feature "${RBD_IMAGE_FEATURES}" $@ +} + +clone_image_and_enable_mirror() +{ + local cluster=$1 + local parent_pool=$2 + local parent_image=$3 + local parent_snap=$4 + local clone_pool=$5 + local clone_image=$6 + shift 6 + + local mode=${1:-${MIRROR_IMAGE_MODE}} + if [ -n "$1" ]; then + shift + fi + + clone_image ${cluster} ${parent_pool} ${parent_image} ${parent_snap} ${clone_pool} ${clone_image} $@ + enable_mirror ${cluster} ${clone_pool} ${clone_image} ${mode} +} + +disconnect_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} journal client disconnect \ + --image ${pool}/${image} +} + +create_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + rbd --cluster ${cluster} snap create ${pool}/${image}@${snap} +} + +remove_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + rbd --cluster ${cluster} snap rm ${pool}/${image}@${snap} +} + +rename_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + local new_snap=$5 + + rbd --cluster ${cluster} snap rename ${pool}/${image}@${snap} \ + ${pool}/${image}@${new_snap} +} + +purge_snapshots() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} snap purge ${pool}/${image} +} + +protect_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + rbd --cluster ${cluster} snap protect ${pool}/${image}@${snap} +} + +unprotect_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + rbd --cluster ${cluster} snap unprotect ${pool}/${image}@${snap} +} + +unprotect_snapshot_retry() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + for s in 0 1 2 4 8 16 32; do + sleep ${s} + unprotect_snapshot ${cluster} ${pool} ${image} ${snap} && return 0 + done + return 1 +} + +wait_for_snap_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap_name=$4 + local s + + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do + sleep ${s} + rbd --cluster ${cluster} info ${pool}/${image}@${snap_name} || continue + return 0 + done + return 1 +} + +test_snap_moved_to_trash() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap_name=$4 + + rbd --cluster ${cluster} snap ls ${pool}/${image} --all | + grep -F " trash (${snap_name})" +} + +wait_for_snap_moved_to_trash() +{ + local s + + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do + sleep ${s} + test_snap_moved_to_trash $@ || continue + return 0 + done + return 1 +} + +test_snap_removed_from_trash() +{ + test_snap_moved_to_trash $@ && return 1 + return 0 +} + +wait_for_snap_removed_from_trash() +{ + local s + + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do + sleep ${s} + test_snap_removed_from_trash $@ || continue + return 0 + done + return 1 +} + +count_mirror_snaps() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} snap ls ${pool}/${image} --all | + grep -c -F " mirror (" +} + +write_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local count=$4 + local size=$5 + + test -n "${size}" || size=4096 + + rbd --cluster ${cluster} bench ${pool}/${image} --io-type write \ + --io-size ${size} --io-threads 1 --io-total $((size * count)) \ + --io-pattern rand +} + +stress_write_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local duration=$(awk 'BEGIN {srand(); print int(10 * rand()) + 5}') + + set +e + timeout ${duration}s ceph_test_rbd_mirror_random_write \ + --cluster ${cluster} ${pool} ${image} \ + --debug-rbd=20 --debug-journaler=20 \ + 2> ${TEMPDIR}/rbd-mirror-random-write.log + error_code=$? + set -e + + if [ $error_code -eq 124 ]; then + return 0 + fi + return 1 +} + +show_diff() +{ + local file1=$1 + local file2=$2 + + xxd ${file1} > ${file1}.xxd + xxd ${file2} > ${file2}.xxd + sdiff -s ${file1}.xxd ${file2}.xxd | head -n 64 + rm -f ${file1}.xxd ${file2}.xxd +} + +compare_images() +{ + local pool=$1 + local image=$2 + local ret=0 + + local rmt_export=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.export) + local loc_export=${TEMPDIR}/$(mkfname ${CLUSTER1}-${pool}-${image}.export) + + rm -f ${rmt_export} ${loc_export} + rbd --cluster ${CLUSTER2} export ${pool}/${image} ${rmt_export} + rbd --cluster ${CLUSTER1} export ${pool}/${image} ${loc_export} + if ! cmp ${rmt_export} ${loc_export} + then + show_diff ${rmt_export} ${loc_export} + ret=1 + fi + rm -f ${rmt_export} ${loc_export} + return ${ret} +} + +compare_image_snapshots() +{ + local pool=$1 + local image=$2 + local ret=0 + + local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export + local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export + + for snap_name in $(rbd --cluster ${CLUSTER1} --format xml \ + snap list ${pool}/${image} | \ + $XMLSTARLET sel -t -v "//snapshot/name" | \ + grep -E -v "^\.rbd-mirror\."); do + rm -f ${rmt_export} ${loc_export} + rbd --cluster ${CLUSTER2} export ${pool}/${image}@${snap_name} ${rmt_export} + rbd --cluster ${CLUSTER1} export ${pool}/${image}@${snap_name} ${loc_export} + if ! cmp ${rmt_export} ${loc_export} + then + show_diff ${rmt_export} ${loc_export} + ret=1 + fi + done + rm -f ${rmt_export} ${loc_export} + return ${ret} +} + +demote_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} mirror image demote ${pool}/${image} +} + +promote_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local force=$4 + + rbd --cluster=${cluster} mirror image promote ${pool}/${image} ${force} +} + +set_pool_mirror_mode() +{ + local cluster=$1 + local pool=$2 + local mode=${3:-${MIRROR_POOL_MODE}} + + rbd --cluster=${cluster} mirror pool enable ${pool} ${mode} +} + +disable_mirror() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} mirror image disable ${pool}/${image} +} + +enable_mirror() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local mode=${4:-${MIRROR_IMAGE_MODE}} + + rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode} + # Display image info including the global image id for debugging purpose + rbd --cluster=${cluster} info ${pool}/${image} +} + +test_image_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local test_state=$4 + local image_id=$5 + local current_state=deleted + local current_image_id + + current_image_id=$(get_image_id ${cluster} ${pool} ${image}) + test -n "${current_image_id}" && + test -z "${image_id}" -o "${image_id}" = "${current_image_id}" && + current_state=present + + test "${test_state}" = "${current_state}" +} + +wait_for_image_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state=$4 + local image_id=$5 + local s + + test -n "${image_id}" || + image_id=$(get_image_id ${cluster} ${pool} ${image}) + + # TODO: add a way to force rbd-mirror to update replayers + for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do + sleep ${s} + test_image_present \ + "${cluster}" "${pool}" "${image}" "${state}" "${image_id}" && + return 0 + done + return 1 +} + +get_image_id() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} info ${pool}/${image} | + sed -ne 's/^.*block_name_prefix: rbd_data\.//p' +} + +request_resync_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local image_id_var_name=$4 + + eval "${image_id_var_name}='$(get_image_id ${cluster} ${pool} ${image})'" + eval 'test -n "$'${image_id_var_name}'"' + + rbd --cluster=${cluster} mirror image resync ${pool}/${image} +} + +get_image_data_pool() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} info ${pool}/${image} | + awk '$1 == "data_pool:" {print $2}' +} + +get_clone_format() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} info ${pool}/${image} | + awk 'BEGIN { + format = 1 + } + $1 == "parent:" { + parent = $2 + } + /op_features: .*clone-child/ { + format = 2 + } + END { + if (!parent) exit 1 + print format + }' +} + +list_omap_keys() +{ + local cluster=$1 + local pool=$2 + local obj_name=$3 + + rados --cluster ${cluster} -p ${pool} listomapkeys ${obj_name} +} + +count_omap_keys_with_filter() +{ + local cluster=$1 + local pool=$2 + local obj_name=$3 + local filter=$4 + + list_omap_keys ${cluster} ${pool} ${obj_name} | grep -c ${filter} +} + +wait_for_omap_keys() +{ + local cluster=$1 + local pool=$2 + local obj_name=$3 + local filter=$4 + + for s in 0 1 2 2 4 4 8 8 8 16 16 32; do + sleep $s + + set +e + test "$(count_omap_keys_with_filter ${cluster} ${pool} ${obj_name} ${filter})" = 0 + error_code=$? + set -e + + if [ $error_code -eq 0 ]; then + return 0 + fi + done + + return 1 +} + +wait_for_image_in_omap() +{ + local cluster=$1 + local pool=$2 + + wait_for_omap_keys ${cluster} ${pool} rbd_mirroring status_global + wait_for_omap_keys ${cluster} ${pool} rbd_mirroring image_ + wait_for_omap_keys ${cluster} ${pool} rbd_mirror_leader image_map +} + +# +# Main +# + +if [ "$#" -gt 0 ] +then + if [ -z "${RBD_MIRROR_TEMDIR}" ] + then + echo "RBD_MIRROR_TEMDIR is not set" >&2 + exit 1 + fi + + TEMPDIR="${RBD_MIRROR_TEMDIR}" + cd ${TEMPDIR} + $@ + exit $? +fi |