diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
| commit | e6918187568dbd01842d8d1d2c808ce16a894239 (patch) | |
| tree | 64f88b554b444a49f656b6c656111a145cbbaa28 /qa/workunits | |
| parent | Initial commit. (diff) | |
| download | ceph-e6918187568dbd01842d8d1d2c808ce16a894239.tar.xz ceph-e6918187568dbd01842d8d1d2c808ce16a894239.zip | |
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'qa/workunits')
275 files changed, 43789 insertions, 0 deletions
diff --git a/qa/workunits/Makefile b/qa/workunits/Makefile new file mode 100644 index 000000000..f75f5dfd4 --- /dev/null +++ b/qa/workunits/Makefile @@ -0,0 +1,4 @@ +DIRS = direct_io fs + +all: + for d in $(DIRS) ; do ( cd $$d ; $(MAKE) all ) ; done diff --git a/qa/workunits/caps/mon_commands.sh b/qa/workunits/caps/mon_commands.sh new file mode 100755 index 000000000..5b5bce62e --- /dev/null +++ b/qa/workunits/caps/mon_commands.sh @@ -0,0 +1,25 @@ +#!/bin/sh -ex + +ceph-authtool --create-keyring k --gen-key -p --name client.xx +ceph auth add -i k client.xx mon "allow command foo; allow command bar *; allow command baz ...; allow command foo add * mon allow\\ rwx osd allow\\ *" + +( ceph -k k -n client.xx foo || true ) | grep 'unrecog' +( ceph -k k -n client.xx foo ooo || true ) | grep 'Access denied' +( ceph -k k -n client.xx fo || true ) | grep 'Access denied' +( ceph -k k -n client.xx fooo || true ) | grep 'Access denied' + +( ceph -k k -n client.xx bar || true ) | grep 'Access denied' +( ceph -k k -n client.xx bar a || true ) | grep 'unrecog' +( ceph -k k -n client.xx bar a b c || true ) | grep 'Access denied' +( ceph -k k -n client.xx ba || true ) | grep 'Access denied' +( ceph -k k -n client.xx barr || true ) | grep 'Access denied' + +( ceph -k k -n client.xx baz || true ) | grep -v 'Access denied' +( ceph -k k -n client.xx baz a || true ) | grep -v 'Access denied' +( ceph -k k -n client.xx baz a b || true ) | grep -v 'Access denied' + +( ceph -k k -n client.xx foo add osd.1 -i k mon 'allow rwx' osd 'allow *' || true ) | grep 'unrecog' +( ceph -k k -n client.xx foo add osd a b c -i k mon 'allow rwx' osd 'allow *' || true ) | grep 'Access denied' +( ceph -k k -n client.xx foo add osd a b c -i k mon 'allow *' || true ) | grep 'Access denied' + +echo OK
\ No newline at end of file diff --git a/qa/workunits/ceph-helpers-root.sh b/qa/workunits/ceph-helpers-root.sh new file mode 100755 index 000000000..5b5d2b409 --- /dev/null +++ b/qa/workunits/ceph-helpers-root.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2015 Red Hat <contact@redhat.com> +# +# Author: Loic Dachary <loic@dachary.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +####################################################################### + +function distro_id() { + source /etc/os-release + echo $ID +} + +function distro_version() { + source /etc/os-release + echo $VERSION +} + +function install() { + if [ $(distro_id) = "ubuntu" ]; then + sudo apt-get purge -y gcc + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + fi + for package in "$@" ; do + install_one $package + done + if [ $(distro_id) = "ubuntu" ]; then + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 11 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 11 + sudo update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 11 + sudo update-alternatives --set cc /usr/bin/gcc + sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 11 + sudo update-alternatives --set c++ /usr/bin/g++ + fi +} + +function install_one() { + case $(distro_id) in + ubuntu|debian|devuan|softiron) + sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y "$@" + ;; + centos|fedora|rhel) + sudo yum install -y "$@" + ;; + opensuse*|suse|sles) + sudo zypper --non-interactive install "$@" + ;; + *) + echo "$(distro_id) is unknown, $@ will have to be installed manually." + ;; + esac +} + +function install_pkg_on_ubuntu { + local project=$1 + shift + local sha1=$1 + shift + local codename=$1 + shift + local force=$1 + shift + local pkgs=$@ + local missing_pkgs + if [ $force = "force" ]; then + missing_pkgs="$@" + else + for pkg in $pkgs; do + if ! dpkg -s $pkg &> /dev/null; then + missing_pkgs+=" $pkg" + fi + done + fi + if test -n "$missing_pkgs"; then + local shaman_url="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo" + sudo curl --silent --location $shaman_url --output /etc/apt/sources.list.d/$project.list + sudo env DEBIAN_FRONTEND=noninteractive apt-get update -y -o Acquire::Languages=none -o Acquire::Translation=none || true + sudo env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y $missing_pkgs + fi +} + +####################################################################### + +function control_osd() { + local action=$1 + local id=$2 + + sudo systemctl $action ceph-osd@$id + + return 0 +} + +####################################################################### + +function pool_read_write() { + local size=${1:-1} + local dir=/tmp + local timeout=360 + local test_pool=test_pool + + ceph osd pool delete $test_pool $test_pool --yes-i-really-really-mean-it || return 1 + ceph osd pool create $test_pool 4 || return 1 + ceph osd pool set $test_pool size $size --yes-i-really-mean-it || return 1 + ceph osd pool set $test_pool min_size $size || return 1 + ceph osd pool application enable $test_pool rados + + echo FOO > $dir/BAR + timeout $timeout rados --pool $test_pool put BAR $dir/BAR || return 1 + timeout $timeout rados --pool $test_pool get BAR $dir/BAR.copy || return 1 + diff $dir/BAR $dir/BAR.copy || return 1 + ceph osd pool delete $test_pool $test_pool --yes-i-really-really-mean-it || return 1 +} + +####################################################################### + +set -x + +"$@" diff --git a/qa/workunits/ceph-tests/ceph-admin-commands.sh b/qa/workunits/ceph-tests/ceph-admin-commands.sh new file mode 100755 index 000000000..4a9f0a66f --- /dev/null +++ b/qa/workunits/ceph-tests/ceph-admin-commands.sh @@ -0,0 +1,10 @@ +#!/bin/sh -ex + +ceph -s +rados lspools +rbd ls +# check that the monitors work +ceph osd set nodown +ceph osd unset nodown + +exit 0 diff --git a/qa/workunits/cephadm/create_iscsi_disks.sh b/qa/workunits/cephadm/create_iscsi_disks.sh new file mode 100755 index 000000000..45319e3a1 --- /dev/null +++ b/qa/workunits/cephadm/create_iscsi_disks.sh @@ -0,0 +1,36 @@ +#!/bin/bash -ex +# Create some file-backed iSCSI targets and attach them locally. + +# Exit if it's not CentOS +if ! grep -q rhel /etc/*-release; then + echo "The script only supports CentOS." + exit 1 +fi + +[ -z "$SUDO" ] && SUDO=sudo + +# 15 GB +DISK_FILE_SIZE="16106127360" + +$SUDO yum install -y targetcli iscsi-initiator-utils + +TARGET_NAME="iqn.2003-01.org.linux-iscsi.$(hostname).x8664:sn.foobar" +$SUDO targetcli /iscsi create ${TARGET_NAME} +$SUDO targetcli /iscsi/${TARGET_NAME}/tpg1/portals delete 0.0.0.0 3260 +$SUDO targetcli /iscsi/${TARGET_NAME}/tpg1/portals create 127.0.0.1 3260 +$SUDO targetcli /iscsi/${TARGET_NAME}/tpg1 set attribute generate_node_acls=1 +$SUDO targetcli /iscsi/${TARGET_NAME}/tpg1 set attribute demo_mode_write_protect=0 + +for i in $(seq 3); do + # Create truncated files, and add them as luns + DISK_FILE="/tmp/disk${i}" + $SUDO truncate --size ${DISK_FILE_SIZE} ${DISK_FILE} + + $SUDO targetcli /backstores/fileio create "lun${i}" ${DISK_FILE} + # Workaround for https://tracker.ceph.com/issues/47758 + $SUDO targetcli "/backstores/fileio/lun${i}" set attribute optimal_sectors=0 + $SUDO targetcli /iscsi/${TARGET_NAME}/tpg1/luns create "/backstores/fileio/lun${i}" +done + +$SUDO iscsiadm -m discovery -t sendtargets -p 127.0.0.1 +$SUDO iscsiadm -m node -p 127.0.0.1 -T ${TARGET_NAME} -l diff --git a/qa/workunits/cephadm/test_adoption.sh b/qa/workunits/cephadm/test_adoption.sh new file mode 100755 index 000000000..68580eb62 --- /dev/null +++ b/qa/workunits/cephadm/test_adoption.sh @@ -0,0 +1,60 @@ +#!/bin/bash -ex + +SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm +CORPUS_COMMIT=9cd9ad020d93b0b420924fec55da307aff8bd422 + +[ -z "$SUDO" ] && SUDO=sudo + +[ -d "$TMPDIR" ] || TMPDIR=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX) +trap "$SUDO rm -rf $TMPDIR" EXIT + +if [ -z "$CEPHADM" ]; then + CEPHADM=`mktemp -p $TMPDIR tmp.cephadm.XXXXXX` + ${CEPHADM_SRC_DIR}/build.sh "$CEPHADM" +fi + +# at this point, we need $CEPHADM set +if ! [ -x "$CEPHADM" ]; then + echo "cephadm not found. Please set \$CEPHADM" + exit 1 +fi + +# combine into a single var +CEPHADM_BIN="$CEPHADM" +CEPHADM="$SUDO $CEPHADM_BIN" + +## adopt +CORPUS_GIT_SUBMOD="cephadm-adoption-corpus" +GIT_CLONE_DIR=${TMPDIR}/${CORPUS_GIT_SUBMOD} +git clone https://github.com/ceph/$CORPUS_GIT_SUBMOD $GIT_CLONE_DIR + +git -C $GIT_CLONE_DIR checkout $CORPUS_COMMIT +CORPUS_DIR=${GIT_CLONE_DIR}/archive + +for subdir in `ls ${CORPUS_DIR}`; do + for tarfile in `ls ${CORPUS_DIR}/${subdir} | grep .tgz`; do + tarball=${CORPUS_DIR}/${subdir}/${tarfile} + FSID_LEGACY=`echo "$tarfile" | cut -c 1-36` + TMP_TAR_DIR=`mktemp -d -p $TMPDIR` + $SUDO tar xzvf $tarball -C $TMP_TAR_DIR + NAMES=$($CEPHADM ls --legacy-dir $TMP_TAR_DIR | jq -r '.[].name') + for name in $NAMES; do + $CEPHADM adopt \ + --style legacy \ + --legacy-dir $TMP_TAR_DIR \ + --name $name + # validate after adopt + out=$($CEPHADM ls | jq '.[]' \ + | jq 'select(.name == "'$name'")') + echo $out | jq -r '.style' | grep 'cephadm' + echo $out | jq -r '.fsid' | grep $FSID_LEGACY + done + # clean-up before next iter + $CEPHADM rm-cluster --fsid $FSID_LEGACY --force + $SUDO rm -rf $TMP_TAR_DIR + done +done + +echo "OK" diff --git a/qa/workunits/cephadm/test_cephadm.sh b/qa/workunits/cephadm/test_cephadm.sh new file mode 100755 index 000000000..7d06a3326 --- /dev/null +++ b/qa/workunits/cephadm/test_cephadm.sh @@ -0,0 +1,474 @@ +#!/bin/bash -ex + +SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# cleanup during exit +[ -z "$CLEANUP" ] && CLEANUP=true + +FSID='00000000-0000-0000-0000-0000deadbeef' + +# images that are used +IMAGE_MAIN=${IMAGE_MAIN:-'quay.ceph.io/ceph-ci/ceph:main'} +IMAGE_PACIFIC=${IMAGE_PACIFIC:-'quay.ceph.io/ceph-ci/ceph:pacific'} +#IMAGE_OCTOPUS=${IMAGE_OCTOPUS:-'quay.ceph.io/ceph-ci/ceph:octopus'} +IMAGE_DEFAULT=${IMAGE_MAIN} + +OSD_IMAGE_NAME="${SCRIPT_NAME%.*}_osd.img" +OSD_IMAGE_SIZE='6G' +OSD_TO_CREATE=2 +OSD_VG_NAME=${SCRIPT_NAME%.*} +OSD_LV_NAME=${SCRIPT_NAME%.*} + +# TMPDIR for test data +[ -d "$TMPDIR" ] || TMPDIR=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX) +[ -d "$TMPDIR_TEST_MULTIPLE_MOUNTS" ] || TMPDIR_TEST_MULTIPLE_MOUNTS=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX) + +CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm +CEPHADM_SAMPLES_DIR=${CEPHADM_SRC_DIR}/samples + +[ -z "$SUDO" ] && SUDO=sudo + +# If cephadm is already installed on the system, use that one, avoid building +# # one if we can. +if [ -z "$CEPHADM" ] && command -v cephadm >/dev/null ; then + CEPHADM="$(command -v cephadm)" +fi + +if [ -z "$CEPHADM" ]; then + CEPHADM=`mktemp -p $TMPDIR tmp.cephadm.XXXXXX` + ${CEPHADM_SRC_DIR}/build.sh "$CEPHADM" + NO_BUILD_INFO=1 +fi + +# at this point, we need $CEPHADM set +if ! [ -x "$CEPHADM" ]; then + echo "cephadm not found. Please set \$CEPHADM" + exit 1 +fi + +# add image to args +CEPHADM_ARGS="$CEPHADM_ARGS --image $IMAGE_DEFAULT" + +# combine into a single var +CEPHADM_BIN="$CEPHADM" +CEPHADM="$SUDO $CEPHADM_BIN $CEPHADM_ARGS" + +# clean up previous run(s)? +$CEPHADM rm-cluster --fsid $FSID --force +$SUDO vgchange -an $OSD_VG_NAME || true +loopdev=$($SUDO losetup -a | grep $(basename $OSD_IMAGE_NAME) | awk -F : '{print $1}') +if ! [ "$loopdev" = "" ]; then + $SUDO losetup -d $loopdev +fi + +function cleanup() +{ + if [ $CLEANUP = false ]; then + # preserve the TMPDIR state + echo "========================" + echo "!!! CLEANUP=$CLEANUP !!!" + echo + echo "TMPDIR=$TMPDIR" + echo "========================" + return + fi + + dump_all_logs $FSID + rm -rf $TMPDIR +} +trap cleanup EXIT + +function expect_false() +{ + set -x + if eval "$@"; then return 1; else return 0; fi +} + +# expect_return_code $expected_code $command ... +function expect_return_code() +{ + set -x + local expected_code="$1" + shift + local command="$@" + + set +e + eval "$command" + local return_code="$?" + set -e + + if [ ! "$return_code" -eq "$expected_code" ]; then return 1; else return 0; fi +} + +function is_available() +{ + local name="$1" + local condition="$2" + local tries="$3" + + local num=0 + while ! eval "$condition"; do + num=$(($num + 1)) + if [ "$num" -ge $tries ]; then + echo "$name is not available" + false + fi + sleep 5 + done + + echo "$name is available" + true +} + +function dump_log() +{ + local fsid="$1" + local name="$2" + local num_lines="$3" + + if [ -z $num_lines ]; then + num_lines=100 + fi + + echo '-------------------------' + echo 'dump daemon log:' $name + echo '-------------------------' + + $CEPHADM logs --fsid $fsid --name $name -- --no-pager -n $num_lines +} + +function dump_all_logs() +{ + local fsid="$1" + local names=$($CEPHADM ls | jq -r '.[] | select(.fsid == "'$fsid'").name') + + echo 'dumping logs for daemons: ' $names + for name in $names; do + dump_log $fsid $name + done +} + +function nfs_stop() +{ + # stop the running nfs server + local units="nfs-server nfs-kernel-server" + for unit in $units; do + if systemctl --no-pager status $unit > /dev/null; then + $SUDO systemctl stop $unit + fi + done + + # ensure the NFS port is no longer in use + expect_false "$SUDO ss -tlnp '( sport = :nfs )' | grep LISTEN" +} + +## prepare + check host +$SUDO $CEPHADM check-host + +## run a gather-facts (output to stdout) +$SUDO $CEPHADM gather-facts + +## NOTE: cephadm version is, as of around May 2023, no longer basing the +## output for `cephadm version` on the version of the containers. The version +## reported is that of the "binary" and is determined during the ceph build. +## `cephadm version` should NOT require sudo/root. +$CEPHADM_BIN version +$CEPHADM_BIN version | grep 'cephadm version' +# Typically cmake should be running the cephadm build script with CLI arguments +# that embed version info into the "binary". If not using a cephadm build via +# cmake you can set `NO_BUILD_INFO` to skip this check. +if [ -z "$NO_BUILD_INFO" ]; then + $CEPHADM_BIN version | grep -v 'UNSET' + $CEPHADM_BIN version | grep -v 'UNKNOWN' +fi + + +## test shell before bootstrap, when crash dir isn't (yet) present on this host +$CEPHADM shell --fsid $FSID -- ceph -v | grep 'ceph version' +$CEPHADM shell --fsid $FSID -e FOO=BAR -- printenv | grep FOO=BAR + +# test stdin +echo foo | $CEPHADM shell -- cat | grep -q foo + +# the shell commands a bit above this seems to cause the +# /var/lib/ceph/<fsid> directory to be made. Since we now +# check in bootstrap that there are no clusters with the same +# fsid based on the directory existing, we need to make sure +# this directory is gone before bootstrapping. We can +# accomplish this with another rm-cluster +$CEPHADM rm-cluster --fsid $FSID --force + +## bootstrap +ORIG_CONFIG=`mktemp -p $TMPDIR` +CONFIG=`mktemp -p $TMPDIR` +MONCONFIG=`mktemp -p $TMPDIR` +KEYRING=`mktemp -p $TMPDIR` +IP=127.0.0.1 +cat <<EOF > $ORIG_CONFIG +[global] + log to file = true + osd crush chooseleaf type = 0 +EOF +$CEPHADM bootstrap \ + --mon-id a \ + --mgr-id x \ + --mon-ip $IP \ + --fsid $FSID \ + --config $ORIG_CONFIG \ + --output-config $CONFIG \ + --output-keyring $KEYRING \ + --output-pub-ssh-key $TMPDIR/ceph.pub \ + --allow-overwrite \ + --skip-mon-network \ + --skip-monitoring-stack +test -e $CONFIG +test -e $KEYRING +rm -f $ORIG_CONFIG + +$SUDO test -e /var/log/ceph/$FSID/ceph-mon.a.log +$SUDO test -e /var/log/ceph/$FSID/ceph-mgr.x.log + +for u in ceph.target \ + ceph-$FSID.target \ + ceph-$FSID@mon.a \ + ceph-$FSID@mgr.x; do + systemctl is-enabled $u + systemctl is-active $u +done +systemctl | grep system-ceph | grep -q .slice # naming is escaped and annoying + +# check ceph -s works (via shell w/ passed config/keyring) +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph -s | grep $FSID + +for t in mon mgr node-exporter prometheus grafana; do + $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph orch apply $t --unmanaged +done + +## ls +$CEPHADM ls | jq '.[]' | jq 'select(.name == "mon.a").fsid' \ + | grep $FSID +$CEPHADM ls | jq '.[]' | jq 'select(.name == "mgr.x").fsid' \ + | grep $FSID + +# make sure the version is returned correctly +$CEPHADM ls | jq '.[]' | jq 'select(.name == "mon.a").version' | grep -q \\. + +## deploy +# add mon.b +cp $CONFIG $MONCONFIG +echo "public addrv = [v2:$IP:3301,v1:$IP:6790]" >> $MONCONFIG +jq --null-input \ + --arg fsid $FSID \ + --arg name mon.b \ + --arg keyring /var/lib/ceph/$FSID/mon.a/keyring \ + --arg config "$MONCONFIG" \ + '{"fsid": $fsid, "name": $name, "params":{"keyring": $keyring, "config": $config}}' | \ + $CEPHADM _orch deploy +for u in ceph-$FSID@mon.b; do + systemctl is-enabled $u + systemctl is-active $u +done +cond="$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph mon stat | grep '2 mons'" +is_available "mon.b" "$cond" 30 + +# add mgr.y +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph auth get-or-create mgr.y \ + mon 'allow profile mgr' \ + osd 'allow *' \ + mds 'allow *' > $TMPDIR/keyring.mgr.y +jq --null-input \ + --arg fsid $FSID \ + --arg name mgr.y \ + --arg keyring $TMPDIR/keyring.mgr.y \ + --arg config "$CONFIG" \ + '{"fsid": $fsid, "name": $name, "params":{"keyring": $keyring, "config": $config}}' | \ + $CEPHADM _orch deploy +for u in ceph-$FSID@mgr.y; do + systemctl is-enabled $u + systemctl is-active $u +done + +for f in `seq 1 30`; do + if $CEPHADM shell --fsid $FSID \ + --config $CONFIG --keyring $KEYRING -- \ + ceph -s -f json-pretty \ + | jq '.mgrmap.num_standbys' | grep -q 1 ; then break; fi + sleep 1 +done +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph -s -f json-pretty \ + | jq '.mgrmap.num_standbys' | grep -q 1 + +# add osd.{1,2,..} +dd if=/dev/zero of=$TMPDIR/$OSD_IMAGE_NAME bs=1 count=0 seek=$OSD_IMAGE_SIZE +loop_dev=$($SUDO losetup -f) +$SUDO vgremove -f $OSD_VG_NAME || true +$SUDO losetup $loop_dev $TMPDIR/$OSD_IMAGE_NAME +$SUDO pvcreate $loop_dev && $SUDO vgcreate $OSD_VG_NAME $loop_dev + +# osd bootstrap keyring +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph auth get client.bootstrap-osd > $TMPDIR/keyring.bootstrap.osd + +# create lvs first so ceph-volume doesn't overlap with lv creation +for id in `seq 0 $((--OSD_TO_CREATE))`; do + $SUDO lvcreate -l $((100/$OSD_TO_CREATE))%VG -n $OSD_LV_NAME.$id $OSD_VG_NAME +done + +for id in `seq 0 $((--OSD_TO_CREATE))`; do + device_name=/dev/$OSD_VG_NAME/$OSD_LV_NAME.$id + CEPH_VOLUME="$CEPHADM ceph-volume \ + --fsid $FSID \ + --config $CONFIG \ + --keyring $TMPDIR/keyring.bootstrap.osd --" + + # prepare the osd + $CEPH_VOLUME lvm prepare --bluestore --data $device_name --no-systemd + $CEPH_VOLUME lvm batch --no-auto $device_name --yes --no-systemd + + # osd id and osd fsid + $CEPH_VOLUME lvm list --format json $device_name > $TMPDIR/osd.map + osd_id=$($SUDO cat $TMPDIR/osd.map | jq -cr '.. | ."ceph.osd_id"? | select(.)') + osd_fsid=$($SUDO cat $TMPDIR/osd.map | jq -cr '.. | ."ceph.osd_fsid"? | select(.)') + + # deploy the osd + jq --null-input \ + --arg fsid $FSID \ + --arg name osd.$osd_id \ + --arg keyring $TMPDIR/keyring.bootstrap.osd \ + --arg config "$CONFIG" \ + --arg osd_fsid $osd_fsid \ + '{"fsid": $fsid, "name": $name, "params":{"keyring": $keyring, "config": $config, "osd_fsid": $osd_fsid}}' | \ + $CEPHADM _orch deploy +done + +# add node-exporter +jq --null-input \ + --arg fsid $FSID \ + --arg name node-exporter.a \ + '{"fsid": $fsid, "name": $name}' | \ + ${CEPHADM//--image $IMAGE_DEFAULT/} _orch deploy +cond="curl 'http://localhost:9100' | grep -q 'Node Exporter'" +is_available "node-exporter" "$cond" 10 + +# add prometheus +jq --null-input \ + --arg fsid $FSID \ + --arg name prometheus.a \ + --argjson config_blobs "$(cat ${CEPHADM_SAMPLES_DIR}/prometheus.json)" \ + '{"fsid": $fsid, "name": $name, "config_blobs": $config_blobs}' | \ + ${CEPHADM//--image $IMAGE_DEFAULT/} _orch deploy +cond="curl 'localhost:9095/api/v1/query?query=up'" +is_available "prometheus" "$cond" 10 + +# add grafana +jq --null-input \ + --arg fsid $FSID \ + --arg name grafana.a \ + --argjson config_blobs "$(cat ${CEPHADM_SAMPLES_DIR}/grafana.json)" \ + '{"fsid": $fsid, "name": $name, "config_blobs": $config_blobs}' | \ + ${CEPHADM//--image $IMAGE_DEFAULT/} _orch deploy +cond="curl --insecure 'https://localhost:3000' | grep -q 'grafana'" +is_available "grafana" "$cond" 50 + +# add nfs-ganesha +nfs_stop +nfs_rados_pool=$(cat ${CEPHADM_SAMPLES_DIR}/nfs.json | jq -r '.["pool"]') +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph osd pool create $nfs_rados_pool 64 +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + rados --pool nfs-ganesha --namespace nfs-ns create conf-nfs.a +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph orch pause +jq --null-input \ + --arg fsid $FSID \ + --arg name nfs.a \ + --arg keyring "$KEYRING" \ + --arg config "$CONFIG" \ + --argjson config_blobs "$(cat ${CEPHADM_SAMPLES_DIR}/nfs.json)" \ + '{"fsid": $fsid, "name": $name, "params": {"keyring": $keyring, "config": $config}, "config_blobs": $config_blobs}' | \ + ${CEPHADM} _orch deploy +cond="$SUDO ss -tlnp '( sport = :nfs )' | grep 'ganesha.nfsd'" +is_available "nfs" "$cond" 10 +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph orch resume + +# add alertmanager via custom container +alertmanager_image=$(cat ${CEPHADM_SAMPLES_DIR}/custom_container.json | jq -r '.image') +tcp_ports=$(jq .ports ${CEPHADM_SAMPLES_DIR}/custom_container.json) +jq --null-input \ + --arg fsid $FSID \ + --arg name container.alertmanager.a \ + --arg keyring $TMPDIR/keyring.bootstrap.osd \ + --arg config "$CONFIG" \ + --arg image "$alertmanager_image" \ + --argjson tcp_ports "${tcp_ports}" \ + --argjson config_blobs "$(cat ${CEPHADM_SAMPLES_DIR}/custom_container.json)" \ + '{"fsid": $fsid, "name": $name, "image": $image, "params": {"keyring": $keyring, "config": $config, "tcp_ports": $tcp_ports}, "config_blobs": $config_blobs}' | \ + ${CEPHADM//--image $IMAGE_DEFAULT/} _orch deploy +cond="$CEPHADM enter --fsid $FSID --name container.alertmanager.a -- test -f \ + /etc/alertmanager/alertmanager.yml" +is_available "alertmanager.yml" "$cond" 10 +cond="curl 'http://localhost:9093' | grep -q 'Alertmanager'" +is_available "alertmanager" "$cond" 10 + +## run +# WRITE ME + +## unit +$CEPHADM unit --fsid $FSID --name mon.a -- is-enabled +$CEPHADM unit --fsid $FSID --name mon.a -- is-active +expect_false $CEPHADM unit --fsid $FSID --name mon.xyz -- is-active +$CEPHADM unit --fsid $FSID --name mon.a -- disable +expect_false $CEPHADM unit --fsid $FSID --name mon.a -- is-enabled +$CEPHADM unit --fsid $FSID --name mon.a -- enable +$CEPHADM unit --fsid $FSID --name mon.a -- is-enabled +$CEPHADM unit --fsid $FSID --name mon.a -- status +$CEPHADM unit --fsid $FSID --name mon.a -- stop +expect_return_code 3 $CEPHADM unit --fsid $FSID --name mon.a -- status +$CEPHADM unit --fsid $FSID --name mon.a -- start + +## shell +$CEPHADM shell --fsid $FSID -- true +$CEPHADM shell --fsid $FSID -- test -d /var/log/ceph +expect_false $CEPHADM --timeout 10 shell --fsid $FSID -- sleep 60 +$CEPHADM --timeout 60 shell --fsid $FSID -- sleep 10 +$CEPHADM shell --fsid $FSID --mount $TMPDIR $TMPDIR_TEST_MULTIPLE_MOUNTS -- stat /mnt/$(basename $TMPDIR) + +## enter +expect_false $CEPHADM enter +$CEPHADM enter --fsid $FSID --name mon.a -- test -d /var/lib/ceph/mon/ceph-a +$CEPHADM enter --fsid $FSID --name mgr.x -- test -d /var/lib/ceph/mgr/ceph-x +$CEPHADM enter --fsid $FSID --name mon.a -- pidof ceph-mon +expect_false $CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mon +$CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mgr +# this triggers a bug in older versions of podman, including 18.04's 1.6.2 +#expect_false $CEPHADM --timeout 5 enter --fsid $FSID --name mon.a -- sleep 30 +$CEPHADM --timeout 60 enter --fsid $FSID --name mon.a -- sleep 10 + +## ceph-volume +$CEPHADM ceph-volume --fsid $FSID -- inventory --format=json \ + | jq '.[]' + +## preserve test state +[ $CLEANUP = false ] && exit 0 + +## rm-daemon +# mon and osd require --force +expect_false $CEPHADM rm-daemon --fsid $FSID --name mon.a +# mgr does not +$CEPHADM rm-daemon --fsid $FSID --name mgr.x + +expect_false $CEPHADM zap-osds --fsid $FSID +$CEPHADM zap-osds --fsid $FSID --force + +## rm-cluster +expect_false $CEPHADM rm-cluster --fsid $FSID --zap-osds +$CEPHADM rm-cluster --fsid $FSID --force --zap-osds + +echo PASS diff --git a/qa/workunits/cephadm/test_dashboard_e2e.sh b/qa/workunits/cephadm/test_dashboard_e2e.sh new file mode 100755 index 000000000..32e0bcc77 --- /dev/null +++ b/qa/workunits/cephadm/test_dashboard_e2e.sh @@ -0,0 +1,107 @@ +#!/bin/bash -ex + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DASHBOARD_FRONTEND_DIR=${SCRIPT_DIR}/../../../src/pybind/mgr/dashboard/frontend + +[ -z "$SUDO" ] && SUDO=sudo + +install_common () { + NODEJS_VERSION="16" + if grep -q debian /etc/*-release; then + $SUDO apt-get update + # https://github.com/nodesource/distributions#manual-installation + $SUDO apt-get install curl gpg + KEYRING=/usr/share/keyrings/nodesource.gpg + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor | $SUDO tee "$KEYRING" >/dev/null + DISTRO="$(source /etc/lsb-release; echo $DISTRIB_CODENAME)" + VERSION="node_$NODEJS_VERSION.x" + echo "deb [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | $SUDO tee /etc/apt/sources.list.d/nodesource.list + echo "deb-src [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | $SUDO tee -a /etc/apt/sources.list.d/nodesource.list + $SUDO apt-get update + $SUDO apt-get install nodejs + elif grep -q rhel /etc/*-release; then + $SUDO yum module -y enable nodejs:$NODEJS_VERSION + $SUDO yum install -y jq npm + else + echo "Unsupported distribution." + exit 1 + fi +} + +install_chrome () { + if grep -q debian /etc/*-release; then + $SUDO bash -c 'echo "deb [arch=amd64] https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list' + curl -fsSL https://dl.google.com/linux/linux_signing_key.pub | $SUDO apt-key add - + $SUDO apt-get update + $SUDO apt-get install -y google-chrome-stable + $SUDO apt-get install -y xvfb + $SUDO rm /etc/apt/sources.list.d/google-chrome.list + elif grep -q rhel /etc/*-release; then + $SUDO dd of=/etc/yum.repos.d/google-chrome.repo status=none <<EOF +[google-chrome] +name=google-chrome +baseurl=https://dl.google.com/linux/chrome/rpm/stable/\$basearch +enabled=1 +gpgcheck=1 +gpgkey=https://dl-ssl.google.com/linux/linux_signing_key.pub +EOF + $SUDO yum install -y google-chrome-stable + $SUDO rm /etc/yum.repos.d/google-chrome.repo + # Cypress dependencies + $SUDO yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss.x86_64 libXScrnSaver alsa-lib + else + echo "Unsupported distribution." + exit 1 + fi +} + +cypress_run () { + local specs="$1" + local timeout="$2" + local override_config="excludeSpecPattern=*.po.ts,retries=0,specPattern=${specs}" + + if [ x"$timeout" != "x" ]; then + override_config="${override_config},defaultCommandTimeout=${timeout}" + fi + npx cypress run --browser chrome --headless --config "$override_config" +} + +install_common +install_chrome + +CYPRESS_BASE_URL=$(ceph mgr services | jq -r .dashboard) +export CYPRESS_BASE_URL + +cd $DASHBOARD_FRONTEND_DIR + +# This is required for Cypress to understand typescript +npm ci --unsafe-perm +npx cypress verify +npx cypress info + +# Take `orch device ls` and `orch ps` as ground truth. +ceph orch device ls --refresh +ceph orch ps --refresh +sleep 10 # the previous call is asynchronous +ceph orch device ls --format=json | tee cypress/fixtures/orchestrator/inventory.json +ceph orch ps --format=json | tee cypress/fixtures/orchestrator/services.json + +DASHBOARD_ADMIN_SECRET_FILE="/tmp/dashboard-admin-secret.txt" +printf 'admin' > "${DASHBOARD_ADMIN_SECRET_FILE}" +ceph dashboard ac-user-set-password admin -i "${DASHBOARD_ADMIN_SECRET_FILE}" --force-password + +# Run Dashboard e2e tests. +# These tests are designed with execution order in mind, since orchestrator operations +# are likely to change cluster state, we can't just run tests in arbitrarily order. +# See /ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/ folder. +find cypress # List all specs + +cypress_run "cypress/e2e/orchestrator/01-hosts.e2e-spec.ts" + +# Hosts are removed and added in the previous step. Do a refresh again. +ceph orch device ls --refresh +sleep 10 +ceph orch device ls --format=json | tee cypress/fixtures/orchestrator/inventory.json + +cypress_run "cypress/e2e/orchestrator/03-inventory.e2e-spec.ts" +cypress_run "cypress/e2e/orchestrator/04-osds.e2e-spec.ts" 300000 diff --git a/qa/workunits/cephadm/test_iscsi_etc_hosts.sh b/qa/workunits/cephadm/test_iscsi_etc_hosts.sh new file mode 100755 index 000000000..adbc34a92 --- /dev/null +++ b/qa/workunits/cephadm/test_iscsi_etc_hosts.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# checks if the container and host's /etc/hosts files match +# Necessary to avoid potential bugs caused by podman making +# edits to /etc/hosts file in the container +# exits with code 1 if host and iscsi container /etc/hosts do no match + +set -ex + +ISCSI_DAEMON=$(sudo /home/ubuntu/cephtest/cephadm ls | jq -r '.[] | select(.service_name == "iscsi.foo") | .name') +sudo /home/ubuntu/cephtest/cephadm enter --name $ISCSI_DAEMON -- cat /etc/hosts > iscsi_daemon_etc_hosts.txt +if cmp --silent /etc/hosts iscsi_daemon_etc_hosts.txt; then + echo "Daemon and host /etc/hosts files successfully matched" +else + echo "ERROR: /etc/hosts on host did not match /etc/hosts in the iscsi container!" + echo "Host /etc/hosts:" + cat /etc/hosts + echo "Iscsi container /etc/hosts:" + cat iscsi_daemon_etc_hosts.txt + exit 1 +fi diff --git a/qa/workunits/cephadm/test_iscsi_pids_limit.sh b/qa/workunits/cephadm/test_iscsi_pids_limit.sh new file mode 100755 index 000000000..bed4cc9e2 --- /dev/null +++ b/qa/workunits/cephadm/test_iscsi_pids_limit.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# checks if the containers default pids-limit (4096) is removed and Iscsi +# containers continue to run +# exits 1 if fails + +set -ex + +ISCSI_CONT_IDS=$(sudo podman ps -qa --filter='name=iscsi') +CONT_COUNT=$(echo ${ISCSI_CONT_IDS} | wc -w) +test ${CONT_COUNT} -eq 2 + +for i in ${ISCSI_CONT_IDS} +do + test $(sudo podman exec ${i} cat /sys/fs/cgroup/pids/pids.max) == max +done + +for i in ${ISCSI_CONT_IDS} +do + sudo podman exec ${i} /bin/sh -c 'for j in {0..20000}; do sleep 300 & done' +done + +for i in ${ISCSI_CONT_IDS} +do + SLEEP_COUNT=$(sudo podman exec ${i} /bin/sh -c 'ps -ef | grep -c sleep') + test ${SLEEP_COUNT} -gt 20000 +done + +echo OK diff --git a/qa/workunits/cephadm/test_repos.sh b/qa/workunits/cephadm/test_repos.sh new file mode 100755 index 000000000..221585fd0 --- /dev/null +++ b/qa/workunits/cephadm/test_repos.sh @@ -0,0 +1,45 @@ +#!/bin/bash -ex + +SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm + +[ -d "$TMPDIR" ] || TMPDIR=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX) +trap "$SUDO rm -rf $TMPDIR" EXIT + +if [ -z "$CEPHADM" ]; then + CEPHADM=`mktemp -p $TMPDIR tmp.cephadm.XXXXXX` + ${CEPHADM_SRC_DIR}/build.sh "$CEPHADM" +fi + +# this is a pretty weak test, unfortunately, since the +# package may also be in the base OS. +function test_install_uninstall() { + ( sudo apt update && \ + sudo apt -y install cephadm && \ + sudo $CEPHADM install && \ + sudo apt -y remove cephadm ) || \ + ( sudo yum -y install cephadm && \ + sudo $CEPHADM install && \ + sudo yum -y remove cephadm ) || \ + ( sudo dnf -y install cephadm && \ + sudo $CEPHADM install && \ + sudo dnf -y remove cephadm ) || \ + ( sudo zypper -n install cephadm && \ + sudo $CEPHADM install && \ + sudo zypper -n remove cephadm ) +} + +sudo $CEPHADM -v add-repo --release octopus +test_install_uninstall +sudo $CEPHADM -v rm-repo + +sudo $CEPHADM -v add-repo --dev main +test_install_uninstall +sudo $CEPHADM -v rm-repo + +sudo $CEPHADM -v add-repo --release 15.2.7 +test_install_uninstall +sudo $CEPHADM -v rm-repo + +echo OK. diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh new file mode 100755 index 000000000..aecfd56a9 --- /dev/null +++ b/qa/workunits/cephtool/test.sh @@ -0,0 +1,2991 @@ +#!/usr/bin/env bash +# -*- mode:shell-script; tab-width:8; sh-basic-offset:2; indent-tabs-mode:t -*- +# vim: ts=8 sw=8 ft=bash smarttab +set -x + +source $(dirname $0)/../../standalone/ceph-helpers.sh + +set -e +set -o functrace +PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' +SUDO=${SUDO:-sudo} +export CEPH_DEV=1 + +function check_no_osd_down() +{ + ! ceph osd dump | grep ' down ' +} + +function wait_no_osd_down() +{ + max_run=300 + for i in $(seq 1 $max_run) ; do + if ! check_no_osd_down ; then + echo "waiting for osd(s) to come back up ($i/$max_run)" + sleep 1 + else + break + fi + done + check_no_osd_down +} + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +function expect_true() +{ + set -x + if ! "$@"; then return 1; else return 0; fi +} + +TEMP_DIR=$(mktemp -d ${TMPDIR-/tmp}/cephtool.XXX) +trap "rm -fr $TEMP_DIR" 0 + +TMPFILE=$(mktemp $TEMP_DIR/test_invalid.XXX) + +# +# retry_eagain max cmd args ... +# +# retry cmd args ... if it exits on error and its output contains the +# string EAGAIN, at most $max times +# +function retry_eagain() +{ + local max=$1 + shift + local status + local tmpfile=$TEMP_DIR/retry_eagain.$$ + local count + for count in $(seq 1 $max) ; do + status=0 + "$@" > $tmpfile 2>&1 || status=$? + if test $status = 0 || + ! grep --quiet EAGAIN $tmpfile ; then + break + fi + sleep 1 + done + if test $count = $max ; then + echo retried with non zero exit status, $max times: "$@" >&2 + fi + cat $tmpfile + rm $tmpfile + return $status +} + +# +# map_enxio_to_eagain cmd arg ... +# +# add EAGAIN to the output of cmd arg ... if the output contains +# ENXIO. +# +function map_enxio_to_eagain() +{ + local status=0 + local tmpfile=$TEMP_DIR/map_enxio_to_eagain.$$ + + "$@" > $tmpfile 2>&1 || status=$? + if test $status != 0 && + grep --quiet ENXIO $tmpfile ; then + echo "EAGAIN added by $0::map_enxio_to_eagain" >> $tmpfile + fi + cat $tmpfile + rm $tmpfile + return $status +} + +function check_response() +{ + expected_string=$1 + retcode=$2 + expected_retcode=$3 + if [ "$expected_retcode" -a $retcode != $expected_retcode ] ; then + echo "return code invalid: got $retcode, expected $expected_retcode" >&2 + exit 1 + fi + + if ! grep --quiet -- "$expected_string" $TMPFILE ; then + echo "Didn't find $expected_string in output" >&2 + cat $TMPFILE >&2 + exit 1 + fi +} + +function get_config_value_or_die() +{ + local target config_opt raw val + + target=$1 + config_opt=$2 + + raw="`$SUDO ceph daemon $target config get $config_opt 2>/dev/null`" + if [[ $? -ne 0 ]]; then + echo "error obtaining config opt '$config_opt' from '$target': $raw" + exit 1 + fi + + raw=`echo $raw | sed -e 's/[{} "]//g'` + val=`echo $raw | cut -f2 -d:` + + echo "$val" + return 0 +} + +function expect_config_value() +{ + local target config_opt expected_val val + target=$1 + config_opt=$2 + expected_val=$3 + + val=$(get_config_value_or_die $target $config_opt) + + if [[ "$val" != "$expected_val" ]]; then + echo "expected '$expected_val', got '$val'" + exit 1 + fi +} + +function ceph_watch_start() +{ + local whatch_opt=--watch + + if [ -n "$1" ]; then + whatch_opt=--watch-$1 + if [ -n "$2" ]; then + whatch_opt+=" --watch-channel $2" + fi + fi + + CEPH_WATCH_FILE=${TEMP_DIR}/CEPH_WATCH_$$ + ceph $whatch_opt > $CEPH_WATCH_FILE & + CEPH_WATCH_PID=$! + + # wait until the "ceph" client is connected and receiving + # log messages from monitor + for i in `seq 3`; do + grep -q "cluster" $CEPH_WATCH_FILE && break + sleep 1 + done +} + +function ceph_watch_wait() +{ + local regexp=$1 + local timeout=30 + + if [ -n "$2" ]; then + timeout=$2 + fi + + for i in `seq ${timeout}`; do + grep -q "$regexp" $CEPH_WATCH_FILE && break + sleep 1 + done + + kill $CEPH_WATCH_PID + + if ! grep "$regexp" $CEPH_WATCH_FILE; then + echo "pattern ${regexp} not found in watch file. Full watch file content:" >&2 + cat $CEPH_WATCH_FILE >&2 + return 1 + fi +} + +function test_mon_injectargs() +{ + ceph tell osd.0 injectargs --no-osd_enable_op_tracker + ceph tell osd.0 config get osd_enable_op_tracker | grep false + ceph tell osd.0 injectargs '--osd_enable_op_tracker --osd_op_history_duration 500' + ceph tell osd.0 config get osd_enable_op_tracker | grep true + ceph tell osd.0 config get osd_op_history_duration | grep 500 + ceph tell osd.0 injectargs --no-osd_enable_op_tracker + ceph tell osd.0 config get osd_enable_op_tracker | grep false + ceph tell osd.0 injectargs -- --osd_enable_op_tracker + ceph tell osd.0 config get osd_enable_op_tracker | grep true + ceph tell osd.0 injectargs -- '--osd_enable_op_tracker --osd_op_history_duration 600' + ceph tell osd.0 config get osd_enable_op_tracker | grep true + ceph tell osd.0 config get osd_op_history_duration | grep 600 + + ceph tell osd.0 injectargs -- '--osd_deep_scrub_interval 2419200' + ceph tell osd.0 config get osd_deep_scrub_interval | grep 2419200 + + ceph tell osd.0 injectargs -- '--mon_probe_timeout 2' + ceph tell osd.0 config get mon_probe_timeout | grep 2 + + ceph tell osd.0 injectargs -- '--mon-lease 6' + ceph tell osd.0 config get mon_lease | grep 6 + + # osd-scrub-auto-repair-num-errors is an OPT_U32, so -1 is not a valid setting + expect_false ceph tell osd.0 injectargs --osd-scrub-auto-repair-num-errors -1 2> $TMPFILE || return 1 + check_response "Error EINVAL: Parse error setting osd_scrub_auto_repair_num_errors to '-1' using injectargs" + + expect_failure $TEMP_DIR "Option --osd_op_history_duration requires an argument" \ + ceph tell osd.0 injectargs -- '--osd_op_history_duration' + +} + +function test_mon_injectargs_SI() +{ + # Test SI units during injectargs and 'config set' + # We only aim at testing the units are parsed accordingly + # and don't intend to test whether the options being set + # actually expect SI units to be passed. + # Keep in mind that all integer based options that are not based on bytes + # (i.e., INT, LONG, U32, U64) will accept SI unit modifiers and be parsed to + # base 10. + initial_value=$(get_config_value_or_die "mon.a" "mon_pg_warn_min_objects") + $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10 + expect_config_value "mon.a" "mon_pg_warn_min_objects" 10 + $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10K + expect_config_value "mon.a" "mon_pg_warn_min_objects" 10000 + $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 1G + expect_config_value "mon.a" "mon_pg_warn_min_objects" 1000000000 + $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10F > $TMPFILE || true + check_response "(22) Invalid argument" + # now test with injectargs + ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10' + expect_config_value "mon.a" "mon_pg_warn_min_objects" 10 + ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10K' + expect_config_value "mon.a" "mon_pg_warn_min_objects" 10000 + ceph tell mon.a injectargs '--mon_pg_warn_min_objects 1G' + expect_config_value "mon.a" "mon_pg_warn_min_objects" 1000000000 + expect_false ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10F' + expect_false ceph tell mon.a injectargs '--mon_globalid_prealloc -1' + $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects $initial_value +} + +function test_mon_injectargs_IEC() +{ + # Test IEC units during injectargs and 'config set' + # We only aim at testing the units are parsed accordingly + # and don't intend to test whether the options being set + # actually expect IEC units to be passed. + # Keep in mind that all integer based options that are based on bytes + # (i.e., INT, LONG, U32, U64) will accept IEC unit modifiers, as well as SI + # unit modifiers (for backwards compatibility and convenience) and be parsed + # to base 2. + initial_value=$(get_config_value_or_die "mon.a" "mon_data_size_warn") + $SUDO ceph daemon mon.a config set mon_data_size_warn 15000000000 + expect_config_value "mon.a" "mon_data_size_warn" 15000000000 + $SUDO ceph daemon mon.a config set mon_data_size_warn 15G + expect_config_value "mon.a" "mon_data_size_warn" 16106127360 + $SUDO ceph daemon mon.a config set mon_data_size_warn 16Gi + expect_config_value "mon.a" "mon_data_size_warn" 17179869184 + $SUDO ceph daemon mon.a config set mon_data_size_warn 10F > $TMPFILE || true + check_response "(22) Invalid argument" + # now test with injectargs + ceph tell mon.a injectargs '--mon_data_size_warn 15000000000' + expect_config_value "mon.a" "mon_data_size_warn" 15000000000 + ceph tell mon.a injectargs '--mon_data_size_warn 15G' + expect_config_value "mon.a" "mon_data_size_warn" 16106127360 + ceph tell mon.a injectargs '--mon_data_size_warn 16Gi' + expect_config_value "mon.a" "mon_data_size_warn" 17179869184 + expect_false ceph tell mon.a injectargs '--mon_data_size_warn 10F' + $SUDO ceph daemon mon.a config set mon_data_size_warn $initial_value +} + +function test_tiering_agent() +{ + local slow=slow_eviction + local fast=fast_eviction + ceph osd pool create $slow 1 1 + ceph osd pool application enable $slow rados + ceph osd pool create $fast 1 1 + ceph osd tier add $slow $fast + ceph osd tier cache-mode $fast writeback + ceph osd tier set-overlay $slow $fast + ceph osd pool set $fast hit_set_type bloom + rados -p $slow put obj1 /etc/group + ceph osd pool set $fast target_max_objects 1 + ceph osd pool set $fast hit_set_count 1 + ceph osd pool set $fast hit_set_period 5 + # wait for the object to be evicted from the cache + local evicted + evicted=false + for i in `seq 1 300` ; do + if ! rados -p $fast ls | grep obj1 ; then + evicted=true + break + fi + sleep 1 + done + $evicted # assert + # the object is proxy read and promoted to the cache + rados -p $slow get obj1 - >/dev/null + # wait for the promoted object to be evicted again + evicted=false + for i in `seq 1 300` ; do + if ! rados -p $fast ls | grep obj1 ; then + evicted=true + break + fi + sleep 1 + done + $evicted # assert + ceph osd tier remove-overlay $slow + ceph osd tier remove $slow $fast + ceph osd pool delete $fast $fast --yes-i-really-really-mean-it + ceph osd pool delete $slow $slow --yes-i-really-really-mean-it +} + +function test_tiering_1() +{ + # tiering + ceph osd pool create slow 2 + ceph osd pool application enable slow rados + ceph osd pool create slow2 2 + ceph osd pool application enable slow2 rados + ceph osd pool create cache 2 + ceph osd pool create cache2 2 + ceph osd tier add slow cache + ceph osd tier add slow cache2 + expect_false ceph osd tier add slow2 cache + # application metadata should propagate to the tiers + ceph osd pool ls detail -f json | jq '.[] | select(.pool_name == "slow") | .application_metadata["rados"]' | grep '{}' + ceph osd pool ls detail -f json | jq '.[] | select(.pool_name == "slow2") | .application_metadata["rados"]' | grep '{}' + ceph osd pool ls detail -f json | jq '.[] | select(.pool_name == "cache") | .application_metadata["rados"]' | grep '{}' + ceph osd pool ls detail -f json | jq '.[] | select(.pool_name == "cache2") | .application_metadata["rados"]' | grep '{}' + # forward is removed/deprecated + expect_false ceph osd tier cache-mode cache forward + expect_false ceph osd tier cache-mode cache forward --yes-i-really-mean-it + # test some state transitions + ceph osd tier cache-mode cache writeback + expect_false ceph osd tier cache-mode cache readonly + expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it + ceph osd tier cache-mode cache proxy + ceph osd tier cache-mode cache readproxy + ceph osd tier cache-mode cache none + ceph osd tier cache-mode cache readonly --yes-i-really-mean-it + ceph osd tier cache-mode cache none + ceph osd tier cache-mode cache writeback + ceph osd tier cache-mode cache proxy + ceph osd tier cache-mode cache writeback + expect_false ceph osd tier cache-mode cache none + expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it + # test with dirty objects in the tier pool + # tier pool currently set to 'writeback' + rados -p cache put /etc/passwd /etc/passwd + flush_pg_stats + # 1 dirty object in pool 'cache' + ceph osd tier cache-mode cache proxy + expect_false ceph osd tier cache-mode cache none + expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it + ceph osd tier cache-mode cache writeback + # remove object from tier pool + rados -p cache rm /etc/passwd + rados -p cache cache-flush-evict-all + flush_pg_stats + # no dirty objects in pool 'cache' + ceph osd tier cache-mode cache proxy + ceph osd tier cache-mode cache none + ceph osd tier cache-mode cache readonly --yes-i-really-mean-it + TRIES=0 + while ! ceph osd pool set cache pg_num 3 --yes-i-really-mean-it 2>$TMPFILE + do + grep 'currently creating pgs' $TMPFILE + TRIES=$(( $TRIES + 1 )) + test $TRIES -ne 60 + sleep 3 + done + expect_false ceph osd pool set cache pg_num 4 + ceph osd tier cache-mode cache none + ceph osd tier set-overlay slow cache + expect_false ceph osd tier set-overlay slow cache2 + expect_false ceph osd tier remove slow cache + ceph osd tier remove-overlay slow + ceph osd tier set-overlay slow cache2 + ceph osd tier remove-overlay slow + ceph osd tier remove slow cache + ceph osd tier add slow2 cache + expect_false ceph osd tier set-overlay slow cache + ceph osd tier set-overlay slow2 cache + ceph osd tier remove-overlay slow2 + ceph osd tier remove slow2 cache + ceph osd tier remove slow cache2 + + # make sure a non-empty pool fails + rados -p cache2 put /etc/passwd /etc/passwd + while ! ceph df | grep cache2 | grep ' 1 ' ; do + echo waiting for pg stats to flush + sleep 2 + done + expect_false ceph osd tier add slow cache2 + ceph osd tier add slow cache2 --force-nonempty + ceph osd tier remove slow cache2 + + ceph osd pool ls | grep cache2 + ceph osd pool ls -f json-pretty | grep cache2 + ceph osd pool ls detail | grep cache2 + ceph osd pool ls detail -f json-pretty | grep cache2 + + ceph osd pool delete slow slow --yes-i-really-really-mean-it + ceph osd pool delete slow2 slow2 --yes-i-really-really-mean-it + ceph osd pool delete cache cache --yes-i-really-really-mean-it + ceph osd pool delete cache2 cache2 --yes-i-really-really-mean-it +} + +function test_tiering_2() +{ + # make sure we can't clobber snapshot state + ceph osd pool create snap_base 2 + ceph osd pool application enable snap_base rados + ceph osd pool create snap_cache 2 + ceph osd pool mksnap snap_cache snapname + expect_false ceph osd tier add snap_base snap_cache + ceph osd pool delete snap_base snap_base --yes-i-really-really-mean-it + ceph osd pool delete snap_cache snap_cache --yes-i-really-really-mean-it +} + +function test_tiering_3() +{ + # make sure we can't create snapshot on tier + ceph osd pool create basex 2 + ceph osd pool application enable basex rados + ceph osd pool create cachex 2 + ceph osd tier add basex cachex + expect_false ceph osd pool mksnap cache snapname + ceph osd tier remove basex cachex + ceph osd pool delete basex basex --yes-i-really-really-mean-it + ceph osd pool delete cachex cachex --yes-i-really-really-mean-it +} + +function test_tiering_4() +{ + # make sure we can't create an ec pool tier + ceph osd pool create eccache 2 2 erasure + expect_false ceph osd set-require-min-compat-client bobtail + ceph osd pool create repbase 2 + ceph osd pool application enable repbase rados + expect_false ceph osd tier add repbase eccache + ceph osd pool delete repbase repbase --yes-i-really-really-mean-it + ceph osd pool delete eccache eccache --yes-i-really-really-mean-it +} + +function test_tiering_5() +{ + # convenient add-cache command + ceph osd pool create slow 2 + ceph osd pool application enable slow rados + ceph osd pool create cache3 2 + ceph osd tier add-cache slow cache3 1024000 + ceph osd dump | grep cache3 | grep bloom | grep 'false_positive_probability: 0.05' | grep 'target_bytes 1024000' | grep '1200s x4' + ceph osd tier remove slow cache3 2> $TMPFILE || true + check_response "EBUSY: tier pool 'cache3' is the overlay for 'slow'; please remove-overlay first" + ceph osd tier remove-overlay slow + ceph osd tier remove slow cache3 + ceph osd pool ls | grep cache3 + ceph osd pool delete cache3 cache3 --yes-i-really-really-mean-it + ! ceph osd pool ls | grep cache3 || exit 1 + ceph osd pool delete slow slow --yes-i-really-really-mean-it +} + +function test_tiering_6() +{ + # check add-cache whether work + ceph osd pool create datapool 2 + ceph osd pool application enable datapool rados + ceph osd pool create cachepool 2 + ceph osd tier add-cache datapool cachepool 1024000 + ceph osd tier cache-mode cachepool writeback + rados -p datapool put object /etc/passwd + rados -p cachepool stat object + rados -p cachepool cache-flush object + rados -p datapool stat object + ceph osd tier remove-overlay datapool + ceph osd tier remove datapool cachepool + ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it + ceph osd pool delete datapool datapool --yes-i-really-really-mean-it +} + +function test_tiering_7() +{ + # protection against pool removal when used as tiers + ceph osd pool create datapool 2 + ceph osd pool application enable datapool rados + ceph osd pool create cachepool 2 + ceph osd tier add-cache datapool cachepool 1024000 + ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it 2> $TMPFILE || true + check_response "EBUSY: pool 'cachepool' is a tier of 'datapool'" + ceph osd pool delete datapool datapool --yes-i-really-really-mean-it 2> $TMPFILE || true + check_response "EBUSY: pool 'datapool' has tiers cachepool" + ceph osd tier remove-overlay datapool + ceph osd tier remove datapool cachepool + ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it + ceph osd pool delete datapool datapool --yes-i-really-really-mean-it +} + +function test_tiering_8() +{ + ## check health check + ceph osd set notieragent + ceph osd pool create datapool 2 + ceph osd pool application enable datapool rados + ceph osd pool create cache4 2 + ceph osd tier add-cache datapool cache4 1024000 + ceph osd tier cache-mode cache4 writeback + tmpfile=$(mktemp|grep tmp) + dd if=/dev/zero of=$tmpfile bs=4K count=1 + ceph osd pool set cache4 target_max_objects 200 + ceph osd pool set cache4 target_max_bytes 1000000 + rados -p cache4 put foo1 $tmpfile + rados -p cache4 put foo2 $tmpfile + rm -f $tmpfile + flush_pg_stats + ceph df | grep datapool | grep ' 2 ' + ceph osd tier remove-overlay datapool + ceph osd tier remove datapool cache4 + ceph osd pool delete cache4 cache4 --yes-i-really-really-mean-it + ceph osd pool delete datapool datapool --yes-i-really-really-mean-it + ceph osd unset notieragent +} + +function test_tiering_9() +{ + # make sure 'tier remove' behaves as we expect + # i.e., removing a tier from a pool that's not its base pool only + # results in a 'pool foo is now (or already was) not a tier of bar' + # + ceph osd pool create basepoolA 2 + ceph osd pool application enable basepoolA rados + ceph osd pool create basepoolB 2 + ceph osd pool application enable basepoolB rados + poolA_id=$(ceph osd dump | grep 'pool.*basepoolA' | awk '{print $2;}') + poolB_id=$(ceph osd dump | grep 'pool.*basepoolB' | awk '{print $2;}') + + ceph osd pool create cache5 2 + ceph osd pool create cache6 2 + ceph osd tier add basepoolA cache5 + ceph osd tier add basepoolB cache6 + ceph osd tier remove basepoolB cache5 2>&1 | grep 'not a tier of' + ceph osd dump | grep "pool.*'cache5'" 2>&1 | grep "tier_of[ \t]\+$poolA_id" + ceph osd tier remove basepoolA cache6 2>&1 | grep 'not a tier of' + ceph osd dump | grep "pool.*'cache6'" 2>&1 | grep "tier_of[ \t]\+$poolB_id" + + ceph osd tier remove basepoolA cache5 2>&1 | grep 'not a tier of' + ! ceph osd dump | grep "pool.*'cache5'" 2>&1 | grep "tier_of" || exit 1 + ceph osd tier remove basepoolB cache6 2>&1 | grep 'not a tier of' + ! ceph osd dump | grep "pool.*'cache6'" 2>&1 | grep "tier_of" || exit 1 + + ! ceph osd dump | grep "pool.*'basepoolA'" 2>&1 | grep "tiers" || exit 1 + ! ceph osd dump | grep "pool.*'basepoolB'" 2>&1 | grep "tiers" || exit 1 + + ceph osd pool delete cache6 cache6 --yes-i-really-really-mean-it + ceph osd pool delete cache5 cache5 --yes-i-really-really-mean-it + ceph osd pool delete basepoolB basepoolB --yes-i-really-really-mean-it + ceph osd pool delete basepoolA basepoolA --yes-i-really-really-mean-it +} + +function test_auth() +{ + expect_false ceph auth add client.xx mon 'invalid' osd "allow *" + expect_false ceph auth add client.xx mon 'allow *' osd "allow *" invalid "allow *" + ceph auth add client.xx mon 'allow *' osd "allow *" + ceph auth export client.xx >client.xx.keyring + ceph auth add client.xx -i client.xx.keyring + rm -f client.xx.keyring + ceph auth list | grep client.xx + ceph auth ls | grep client.xx + ceph auth get client.xx | grep caps | grep mon + ceph auth get client.xx | grep caps | grep osd + ceph auth get-key client.xx + ceph auth print-key client.xx + ceph auth print_key client.xx + ceph auth caps client.xx osd "allow rw" + expect_false sh <<< "ceph auth get client.xx | grep caps | grep mon" + ceph auth get client.xx | grep osd | grep "allow rw" + ceph auth caps client.xx mon 'allow command "osd tree"' + ceph auth export | grep client.xx + ceph auth export -o authfile + ceph auth import -i authfile + + ceph auth export -o authfile2 + diff authfile authfile2 + rm authfile authfile2 + ceph auth del client.xx + expect_false ceph auth get client.xx + + # (almost) interactive mode + echo -e 'auth add client.xx mon "allow *" osd "allow *"\n' | ceph + ceph auth get client.xx + # script mode + echo 'auth del client.xx' | ceph + expect_false ceph auth get client.xx +} + +function test_auth_profiles() +{ + ceph auth add client.xx-profile-ro mon 'allow profile read-only' \ + mgr 'allow profile read-only' + ceph auth add client.xx-profile-rw mon 'allow profile read-write' \ + mgr 'allow profile read-write' + ceph auth add client.xx-profile-rd mon 'allow profile role-definer' + + ceph auth export > client.xx.keyring + + # read-only is allowed all read-only commands (auth excluded) + ceph -n client.xx-profile-ro -k client.xx.keyring status + ceph -n client.xx-profile-ro -k client.xx.keyring osd dump + ceph -n client.xx-profile-ro -k client.xx.keyring pg dump + ceph -n client.xx-profile-ro -k client.xx.keyring mon dump + # read-only gets access denied for rw commands or auth commands + ceph -n client.xx-profile-ro -k client.xx.keyring log foo >& $TMPFILE || true + check_response "EACCES: access denied" + ceph -n client.xx-profile-ro -k client.xx.keyring osd set noout >& $TMPFILE || true + check_response "EACCES: access denied" + ceph -n client.xx-profile-ro -k client.xx.keyring auth ls >& $TMPFILE || true + check_response "EACCES: access denied" + + # read-write is allowed for all read-write commands (except auth) + ceph -n client.xx-profile-rw -k client.xx.keyring status + ceph -n client.xx-profile-rw -k client.xx.keyring osd dump + ceph -n client.xx-profile-rw -k client.xx.keyring pg dump + ceph -n client.xx-profile-rw -k client.xx.keyring mon dump + ceph -n client.xx-profile-rw -k client.xx.keyring fs dump + ceph -n client.xx-profile-rw -k client.xx.keyring log foo + ceph -n client.xx-profile-rw -k client.xx.keyring osd set noout + ceph -n client.xx-profile-rw -k client.xx.keyring osd unset noout + # read-write gets access denied for auth commands + ceph -n client.xx-profile-rw -k client.xx.keyring auth ls >& $TMPFILE || true + check_response "EACCES: access denied" + + # role-definer is allowed RWX 'auth' commands and read-only 'mon' commands + ceph -n client.xx-profile-rd -k client.xx.keyring auth ls + ceph -n client.xx-profile-rd -k client.xx.keyring auth export + ceph -n client.xx-profile-rd -k client.xx.keyring auth add client.xx-profile-foo + ceph -n client.xx-profile-rd -k client.xx.keyring status + ceph -n client.xx-profile-rd -k client.xx.keyring osd dump >& $TMPFILE || true + check_response "EACCES: access denied" + ceph -n client.xx-profile-rd -k client.xx.keyring pg dump >& $TMPFILE || true + check_response "EACCES: access denied" + # read-only 'mon' subsystem commands are allowed + ceph -n client.xx-profile-rd -k client.xx.keyring mon dump + # but read-write 'mon' commands are not + ceph -n client.xx-profile-rd -k client.xx.keyring mon add foo 1.1.1.1 >& $TMPFILE || true + check_response "EACCES: access denied" + ceph -n client.xx-profile-rd -k client.xx.keyring fs dump >& $TMPFILE || true + check_response "EACCES: access denied" + ceph -n client.xx-profile-rd -k client.xx.keyring log foo >& $TMPFILE || true + check_response "EACCES: access denied" + ceph -n client.xx-profile-rd -k client.xx.keyring osd set noout >& $TMPFILE || true + check_response "EACCES: access denied" + + ceph -n client.xx-profile-rd -k client.xx.keyring auth del client.xx-profile-ro + ceph -n client.xx-profile-rd -k client.xx.keyring auth del client.xx-profile-rw + + # add a new role-definer with the existing role-definer + ceph -n client.xx-profile-rd -k client.xx.keyring \ + auth add client.xx-profile-rd2 mon 'allow profile role-definer' + ceph -n client.xx-profile-rd -k client.xx.keyring \ + auth export > client.xx.keyring.2 + # remove old role-definer using the new role-definer + ceph -n client.xx-profile-rd2 -k client.xx.keyring.2 \ + auth del client.xx-profile-rd + # remove the remaining role-definer with admin + ceph auth del client.xx-profile-rd2 + rm -f client.xx.keyring client.xx.keyring.2 +} + +function test_mon_caps() +{ + ceph-authtool --create-keyring $TEMP_DIR/ceph.client.bug.keyring + chmod +r $TEMP_DIR/ceph.client.bug.keyring + ceph-authtool $TEMP_DIR/ceph.client.bug.keyring -n client.bug --gen-key + ceph auth add client.bug -i $TEMP_DIR/ceph.client.bug.keyring + + # pass --no-mon-config since we are looking for the permission denied error + rados lspools --no-mon-config --keyring $TEMP_DIR/ceph.client.bug.keyring -n client.bug >& $TMPFILE || true + cat $TMPFILE + check_response "Permission denied" + + rm -rf $TEMP_DIR/ceph.client.bug.keyring + ceph auth del client.bug + ceph-authtool --create-keyring $TEMP_DIR/ceph.client.bug.keyring + chmod +r $TEMP_DIR/ceph.client.bug.keyring + ceph-authtool $TEMP_DIR/ceph.client.bug.keyring -n client.bug --gen-key + ceph-authtool -n client.bug --cap mon '' $TEMP_DIR/ceph.client.bug.keyring + ceph auth add client.bug -i $TEMP_DIR/ceph.client.bug.keyring + rados lspools --no-mon-config --keyring $TEMP_DIR/ceph.client.bug.keyring -n client.bug >& $TMPFILE || true + check_response "Permission denied" +} + +function test_mon_misc() +{ + # with and without verbosity + ceph osd dump | grep '^epoch' + ceph --concise osd dump | grep '^epoch' + + ceph osd df | grep 'MIN/MAX VAR' + + # df + ceph df > $TMPFILE + grep RAW $TMPFILE + grep -v DIRTY $TMPFILE + ceph df detail > $TMPFILE + grep DIRTY $TMPFILE + ceph df --format json > $TMPFILE + grep 'total_bytes' $TMPFILE + grep -v 'dirty' $TMPFILE + ceph df detail --format json > $TMPFILE + grep 'rd_bytes' $TMPFILE + grep 'dirty' $TMPFILE + ceph df --format xml | grep '<total_bytes>' + ceph df detail --format xml | grep '<rd_bytes>' + + ceph fsid + ceph health + ceph health detail + ceph health --format json-pretty + ceph health detail --format xml-pretty + + ceph time-sync-status + + ceph node ls + for t in mon osd mds mgr ; do + ceph node ls $t + done + + ceph_watch_start + mymsg="this is a test log message $$.$(date)" + ceph log "$mymsg" + ceph log last | grep "$mymsg" + ceph log last 100 | grep "$mymsg" + ceph_watch_wait "$mymsg" + + ceph mgr stat + ceph mgr dump + ceph mgr dump | jq -e '.active_clients[0].name' + ceph mgr module ls + ceph mgr module enable restful + expect_false ceph mgr module enable foodne + ceph mgr module enable foodne --force + ceph mgr module disable foodne + ceph mgr module disable foodnebizbangbash + + ceph mon metadata a + ceph mon metadata + ceph mon count-metadata ceph_version + ceph mon versions + + ceph mgr metadata + ceph mgr versions + ceph mgr count-metadata ceph_version + + ceph versions + + ceph node ls +} + +function check_mds_active() +{ + fs_name=$1 + ceph fs get $fs_name | grep active +} + +function wait_mds_active() +{ + fs_name=$1 + max_run=300 + for i in $(seq 1 $max_run) ; do + if ! check_mds_active $fs_name ; then + echo "waiting for an active MDS daemon ($i/$max_run)" + sleep 5 + else + break + fi + done + check_mds_active $fs_name +} + +function get_mds_gids() +{ + fs_name=$1 + ceph fs get $fs_name --format=json | python3 -c "import json; import sys; print(' '.join([m['gid'].__str__() for m in json.load(sys.stdin)['mdsmap']['info'].values()]))" +} + +function fail_all_mds() +{ + fs_name=$1 + ceph fs set $fs_name cluster_down true + mds_gids=$(get_mds_gids $fs_name) + for mds_gid in $mds_gids ; do + ceph mds fail $mds_gid + done + if check_mds_active $fs_name ; then + echo "An active MDS remains, something went wrong" + ceph fs get $fs_name + exit -1 + fi + +} + +function remove_all_fs() +{ + existing_fs=$(ceph fs ls --format=json | python3 -c "import json; import sys; print(' '.join([fs['name'] for fs in json.load(sys.stdin)]))") + for fs_name in $existing_fs ; do + echo "Removing fs ${fs_name}..." + fail_all_mds $fs_name + echo "Removing existing filesystem '${fs_name}'..." + ceph fs rm $fs_name --yes-i-really-mean-it + echo "Removed '${fs_name}'." + done +} + +# So that tests requiring MDS can skip if one is not configured +# in the cluster at all +function mds_exists() +{ + ceph auth ls | grep "^mds" +} + +# some of the commands are just not idempotent. +function without_test_dup_command() +{ + if [ -z ${CEPH_CLI_TEST_DUP_COMMAND+x} ]; then + $@ + else + local saved=${CEPH_CLI_TEST_DUP_COMMAND} + unset CEPH_CLI_TEST_DUP_COMMAND + $@ + CEPH_CLI_TEST_DUP_COMMAND=saved + fi +} + +function test_mds_tell() +{ + local FS_NAME=cephfs + if ! mds_exists ; then + echo "Skipping test, no MDS found" + return + fi + + remove_all_fs + ceph osd pool create fs_data 16 + ceph osd pool create fs_metadata 16 + ceph fs new $FS_NAME fs_metadata fs_data + wait_mds_active $FS_NAME + + # Test injectargs by GID + old_mds_gids=$(get_mds_gids $FS_NAME) + echo Old GIDs: $old_mds_gids + + for mds_gid in $old_mds_gids ; do + ceph tell mds.$mds_gid injectargs "--debug-mds 20" + done + expect_false ceph tell mds.a injectargs mds_max_file_recover -1 + + # Test respawn by rank + without_test_dup_command ceph tell mds.0 respawn + new_mds_gids=$old_mds_gids + while [ $new_mds_gids -eq $old_mds_gids ] ; do + sleep 5 + new_mds_gids=$(get_mds_gids $FS_NAME) + done + echo New GIDs: $new_mds_gids + + # Test respawn by ID + without_test_dup_command ceph tell mds.a respawn + new_mds_gids=$old_mds_gids + while [ $new_mds_gids -eq $old_mds_gids ] ; do + sleep 5 + new_mds_gids=$(get_mds_gids $FS_NAME) + done + echo New GIDs: $new_mds_gids + + remove_all_fs + ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it + ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it +} + +function test_mon_mds() +{ + local FS_NAME=cephfs + remove_all_fs + + ceph osd pool create fs_data 16 + ceph osd pool create fs_metadata 16 + ceph fs new $FS_NAME fs_metadata fs_data + + ceph fs set $FS_NAME cluster_down true + ceph fs set $FS_NAME cluster_down false + + ceph mds compat rm_incompat 4 + ceph mds compat rm_incompat 4 + + # We don't want any MDSs to be up, their activity can interfere with + # the "current_epoch + 1" checking below if they're generating updates + fail_all_mds $FS_NAME + + ceph mds compat show + ceph fs dump + ceph fs get $FS_NAME + for mds_gid in $(get_mds_gids $FS_NAME) ; do + ceph mds metadata $mds_id + done + ceph mds metadata + ceph mds versions + ceph mds count-metadata os + + # XXX mds fail, but how do you undo it? + mdsmapfile=$TEMP_DIR/mdsmap.$$ + current_epoch=$(ceph fs dump -o $mdsmapfile --no-log-to-stderr 2>&1 | grep epoch | sed 's/.*epoch //') + [ -s $mdsmapfile ] + rm $mdsmapfile + + ceph osd pool create data2 16 + ceph osd pool create data3 16 + data2_pool=$(ceph osd dump | grep "pool.*'data2'" | awk '{print $2;}') + data3_pool=$(ceph osd dump | grep "pool.*'data3'" | awk '{print $2;}') + ceph fs add_data_pool cephfs $data2_pool + ceph fs add_data_pool cephfs $data3_pool + ceph fs add_data_pool cephfs 100 >& $TMPFILE || true + check_response "Error ENOENT" + ceph fs add_data_pool cephfs foobarbaz >& $TMPFILE || true + check_response "Error ENOENT" + ceph fs rm_data_pool cephfs $data2_pool + ceph fs rm_data_pool cephfs $data3_pool + ceph osd pool delete data2 data2 --yes-i-really-really-mean-it + ceph osd pool delete data3 data3 --yes-i-really-really-mean-it + ceph fs set cephfs max_mds 4 + ceph fs set cephfs max_mds 3 + ceph fs set cephfs max_mds 256 + expect_false ceph fs set cephfs max_mds 257 + ceph fs set cephfs max_mds 4 + ceph fs set cephfs max_mds 256 + expect_false ceph fs set cephfs max_mds 257 + expect_false ceph fs set cephfs max_mds asdf + expect_false ceph fs set cephfs inline_data true + ceph fs set cephfs inline_data true --yes-i-really-really-mean-it + ceph fs set cephfs inline_data yes --yes-i-really-really-mean-it + ceph fs set cephfs inline_data 1 --yes-i-really-really-mean-it + expect_false ceph fs set cephfs inline_data --yes-i-really-really-mean-it + ceph fs set cephfs inline_data false + ceph fs set cephfs inline_data no + ceph fs set cephfs inline_data 0 + expect_false ceph fs set cephfs inline_data asdf + ceph fs set cephfs max_file_size 1048576 + expect_false ceph fs set cephfs max_file_size 123asdf + + expect_false ceph fs set cephfs allow_new_snaps + ceph fs set cephfs allow_new_snaps true + ceph fs set cephfs allow_new_snaps 0 + ceph fs set cephfs allow_new_snaps false + ceph fs set cephfs allow_new_snaps no + expect_false ceph fs set cephfs allow_new_snaps taco + + # we should never be able to add EC pools as data or metadata pools + # create an ec-pool... + ceph osd pool create mds-ec-pool 16 16 erasure + set +e + ceph fs add_data_pool cephfs mds-ec-pool 2>$TMPFILE + check_response 'erasure-code' $? 22 + set -e + ec_poolnum=$(ceph osd dump | grep "pool.* 'mds-ec-pool" | awk '{print $2;}') + data_poolnum=$(ceph osd dump | grep "pool.* 'fs_data" | awk '{print $2;}') + metadata_poolnum=$(ceph osd dump | grep "pool.* 'fs_metadata" | awk '{print $2;}') + + fail_all_mds $FS_NAME + + set +e + # Check that rmfailed requires confirmation + expect_false ceph mds rmfailed 0 + ceph mds rmfailed 0 --yes-i-really-mean-it + set -e + + # Check that `fs new` is no longer permitted + expect_false ceph fs new cephfs $metadata_poolnum $data_poolnum --yes-i-really-mean-it 2>$TMPFILE + + # Check that 'fs reset' runs + ceph fs reset $FS_NAME --yes-i-really-mean-it + + # Check that creating a second FS fails by default + ceph osd pool create fs_metadata2 16 + ceph osd pool create fs_data2 16 + set +e + expect_false ceph fs new cephfs2 fs_metadata2 fs_data2 + set -e + + # Check that setting enable_multiple enables creation of second fs + ceph fs flag set enable_multiple true --yes-i-really-mean-it + ceph fs new cephfs2 fs_metadata2 fs_data2 + + # Clean up multi-fs stuff + fail_all_mds cephfs2 + ceph fs rm cephfs2 --yes-i-really-mean-it + ceph osd pool delete fs_metadata2 fs_metadata2 --yes-i-really-really-mean-it + ceph osd pool delete fs_data2 fs_data2 --yes-i-really-really-mean-it + + fail_all_mds $FS_NAME + + # Clean up to enable subsequent fs new tests + ceph fs rm $FS_NAME --yes-i-really-mean-it + + set +e + ceph fs new $FS_NAME fs_metadata mds-ec-pool --force 2>$TMPFILE + check_response 'erasure-code' $? 22 + ceph fs new $FS_NAME mds-ec-pool fs_data 2>$TMPFILE + check_response 'already used by filesystem' $? 22 + ceph fs new $FS_NAME mds-ec-pool fs_data --force 2>$TMPFILE + check_response 'erasure-code' $? 22 + ceph fs new $FS_NAME mds-ec-pool mds-ec-pool 2>$TMPFILE + check_response 'erasure-code' $? 22 + set -e + + # ... new create a cache tier in front of the EC pool... + ceph osd pool create mds-tier 2 + ceph osd tier add mds-ec-pool mds-tier + ceph osd tier set-overlay mds-ec-pool mds-tier + tier_poolnum=$(ceph osd dump | grep "pool.* 'mds-tier" | awk '{print $2;}') + + # Use of a readonly tier should be forbidden + ceph osd tier cache-mode mds-tier readonly --yes-i-really-mean-it + set +e + ceph fs new $FS_NAME fs_metadata mds-ec-pool --force 2>$TMPFILE + check_response 'has a write tier (mds-tier) that is configured to forward' $? 22 + set -e + + # Use of a writeback tier should enable FS creation + ceph osd tier cache-mode mds-tier writeback + ceph fs new $FS_NAME fs_metadata mds-ec-pool --force + + # While a FS exists using the tiered pools, I should not be allowed + # to remove the tier + set +e + ceph osd tier remove-overlay mds-ec-pool 2>$TMPFILE + check_response 'in use by CephFS' $? 16 + ceph osd tier remove mds-ec-pool mds-tier 2>$TMPFILE + check_response 'in use by CephFS' $? 16 + set -e + + fail_all_mds $FS_NAME + ceph fs rm $FS_NAME --yes-i-really-mean-it + + # ... but we should be forbidden from using the cache pool in the FS directly. + set +e + ceph fs new $FS_NAME fs_metadata mds-tier --force 2>$TMPFILE + check_response 'in use as a cache tier' $? 22 + ceph fs new $FS_NAME mds-tier fs_data 2>$TMPFILE + check_response 'already used by filesystem' $? 22 + ceph fs new $FS_NAME mds-tier fs_data --force 2>$TMPFILE + check_response 'in use as a cache tier' $? 22 + ceph fs new $FS_NAME mds-tier mds-tier 2>$TMPFILE + check_response 'already used by filesystem' $? 22 + ceph fs new $FS_NAME mds-tier mds-tier --force 2>$TMPFILE + check_response 'in use as a cache tier' $? 22 + set -e + + # Clean up tier + EC pools + ceph osd tier remove-overlay mds-ec-pool + ceph osd tier remove mds-ec-pool mds-tier + + # Create a FS using the 'cache' pool now that it's no longer a tier + ceph fs new $FS_NAME fs_metadata mds-tier --force + + # We should be forbidden from using this pool as a tier now that + # it's in use for CephFS + set +e + ceph osd tier add mds-ec-pool mds-tier 2>$TMPFILE + check_response 'in use by CephFS' $? 16 + set -e + + fail_all_mds $FS_NAME + ceph fs rm $FS_NAME --yes-i-really-mean-it + + # We should be permitted to use an EC pool with overwrites enabled + # as the data pool... + ceph osd pool set mds-ec-pool allow_ec_overwrites true + ceph fs new $FS_NAME fs_metadata mds-ec-pool --force 2>$TMPFILE + fail_all_mds $FS_NAME + ceph fs rm $FS_NAME --yes-i-really-mean-it + + # ...but not as the metadata pool + set +e + ceph fs new $FS_NAME mds-ec-pool fs_data 2>$TMPFILE + check_response 'already used by filesystem' $? 22 + ceph fs new $FS_NAME mds-ec-pool fs_data --force 2>$TMPFILE + check_response 'erasure-code' $? 22 + set -e + + ceph osd pool delete mds-ec-pool mds-ec-pool --yes-i-really-really-mean-it + + # Create a FS and check that we can subsequently add a cache tier to it + ceph fs new $FS_NAME fs_metadata fs_data --force + + # Adding overlay to FS pool should be permitted, RADOS clients handle this. + ceph osd tier add fs_metadata mds-tier + ceph osd tier cache-mode mds-tier writeback + ceph osd tier set-overlay fs_metadata mds-tier + + # Removing tier should be permitted because the underlying pool is + # replicated (#11504 case) + ceph osd tier cache-mode mds-tier proxy + ceph osd tier remove-overlay fs_metadata + ceph osd tier remove fs_metadata mds-tier + ceph osd pool delete mds-tier mds-tier --yes-i-really-really-mean-it + + # Clean up FS + fail_all_mds $FS_NAME + ceph fs rm $FS_NAME --yes-i-really-mean-it + + + + ceph mds stat + # ceph mds tell mds.a getmap + # ceph mds rm + # ceph mds rmfailed + # ceph mds set_state + + ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it + ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it +} + +function test_mon_mds_metadata() +{ + local nmons=$(ceph tell 'mon.*' version | grep -c 'version') + test "$nmons" -gt 0 + + ceph fs dump | + sed -nEe "s/^([0-9]+):.*'([a-z])' mds\\.([0-9]+)\\..*/\\1 \\2 \\3/p" | + while read gid id rank; do + ceph mds metadata ${gid} | grep '"hostname":' + ceph mds metadata ${id} | grep '"hostname":' + ceph mds metadata ${rank} | grep '"hostname":' + + local n=$(ceph tell 'mon.*' mds metadata ${id} | grep -c '"hostname":') + test "$n" -eq "$nmons" + done + + expect_false ceph mds metadata UNKNOWN +} + +function test_mon_mon() +{ + # print help message + ceph --help mon + # -h works even when some arguments are passed + ceph osd dump -h | grep 'osd dump' + ceph osd dump 123 -h | grep 'osd dump' + # no mon add/remove + ceph mon dump + ceph mon getmap -o $TEMP_DIR/monmap.$$ + [ -s $TEMP_DIR/monmap.$$ ] + + # ceph mon tell + first=$(ceph mon dump -f json | jq -r '.mons[0].name') + ceph tell mon.$first mon_status + + # test mon features + ceph mon feature ls + ceph mon feature set kraken --yes-i-really-mean-it + expect_false ceph mon feature set abcd + expect_false ceph mon feature set abcd --yes-i-really-mean-it + + # test elector + expect_failure $TEMP_DIR ceph mon add disallowed_leader $first + ceph mon set election_strategy disallow + ceph mon add disallowed_leader $first + ceph mon set election_strategy connectivity + ceph mon rm disallowed_leader $first + ceph mon set election_strategy classic + expect_failure $TEMP_DIR ceph mon rm disallowed_leader $first + + # test mon stat + # don't check output, just ensure it does not fail. + ceph mon stat + ceph mon stat -f json | jq '.' +} + +function test_mon_priority_and_weight() +{ + for i in 0 1 65535; do + ceph mon set-weight a $i + w=$(ceph mon dump --format=json-pretty 2>/dev/null | jq '.mons[0].weight') + [[ "$w" == "$i" ]] + done + + for i in -1 65536; do + expect_false ceph mon set-weight a $i + done +} + +function gen_secrets_file() +{ + # lets assume we can have the following types + # all - generates both cephx and lockbox, with mock dm-crypt key + # cephx - only cephx + # no_cephx - lockbox and dm-crypt, no cephx + # no_lockbox - dm-crypt and cephx, no lockbox + # empty - empty file + # empty_json - correct json, empty map + # bad_json - bad json :) + # + local t=$1 + if [[ -z "$t" ]]; then + t="all" + fi + + fn=$(mktemp $TEMP_DIR/secret.XXXXXX) + echo $fn + if [[ "$t" == "empty" ]]; then + return 0 + fi + + echo "{" > $fn + if [[ "$t" == "bad_json" ]]; then + echo "asd: ; }" >> $fn + return 0 + elif [[ "$t" == "empty_json" ]]; then + echo "}" >> $fn + return 0 + fi + + cephx_secret="\"cephx_secret\": \"$(ceph-authtool --gen-print-key)\"" + lb_secret="\"cephx_lockbox_secret\": \"$(ceph-authtool --gen-print-key)\"" + dmcrypt_key="\"dmcrypt_key\": \"$(ceph-authtool --gen-print-key)\"" + + if [[ "$t" == "all" ]]; then + echo "$cephx_secret,$lb_secret,$dmcrypt_key" >> $fn + elif [[ "$t" == "cephx" ]]; then + echo "$cephx_secret" >> $fn + elif [[ "$t" == "no_cephx" ]]; then + echo "$lb_secret,$dmcrypt_key" >> $fn + elif [[ "$t" == "no_lockbox" ]]; then + echo "$cephx_secret,$dmcrypt_key" >> $fn + else + echo "unknown gen_secrets_file() type \'$fn\'" + return 1 + fi + echo "}" >> $fn + return 0 +} + +function test_mon_osd_create_destroy() +{ + ceph osd new 2>&1 | grep 'EINVAL' + ceph osd new '' -1 2>&1 | grep 'EINVAL' + ceph osd new '' 10 2>&1 | grep 'EINVAL' + + old_maxosd=$(ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//') + + old_osds=$(ceph osd ls) + num_osds=$(ceph osd ls | wc -l) + + uuid=$(uuidgen) + id=$(ceph osd new $uuid 2>/dev/null) + + for i in $old_osds; do + [[ "$i" != "$id" ]] + done + + ceph osd find $id + + id2=`ceph osd new $uuid 2>/dev/null` + + [[ $id2 == $id ]] + + ceph osd new $uuid $id + + id3=$(ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//') + ceph osd new $uuid $((id3+1)) 2>&1 | grep EEXIST + + uuid2=$(uuidgen) + id2=$(ceph osd new $uuid2) + ceph osd find $id2 + [[ "$id2" != "$id" ]] + + ceph osd new $uuid $id2 2>&1 | grep EEXIST + ceph osd new $uuid2 $id2 + + # test with secrets + empty_secrets=$(gen_secrets_file "empty") + empty_json=$(gen_secrets_file "empty_json") + all_secrets=$(gen_secrets_file "all") + cephx_only=$(gen_secrets_file "cephx") + no_cephx=$(gen_secrets_file "no_cephx") + no_lockbox=$(gen_secrets_file "no_lockbox") + bad_json=$(gen_secrets_file "bad_json") + + # empty secrets should be idempotent + new_id=$(ceph osd new $uuid $id -i $empty_secrets) + [[ "$new_id" == "$id" ]] + + # empty json, thus empty secrets + new_id=$(ceph osd new $uuid $id -i $empty_json) + [[ "$new_id" == "$id" ]] + + ceph osd new $uuid $id -i $all_secrets 2>&1 | grep 'EEXIST' + + ceph osd rm $id + ceph osd rm $id2 + ceph osd setmaxosd $old_maxosd + + ceph osd new $uuid -i $no_cephx 2>&1 | grep 'EINVAL' + ceph osd new $uuid -i $no_lockbox 2>&1 | grep 'EINVAL' + + osds=$(ceph osd ls) + id=$(ceph osd new $uuid -i $all_secrets) + for i in $osds; do + [[ "$i" != "$id" ]] + done + + ceph osd find $id + + # validate secrets and dm-crypt are set + k=$(ceph auth get-key osd.$id --format=json-pretty 2>/dev/null | jq '.key') + s=$(cat $all_secrets | jq '.cephx_secret') + [[ $k == $s ]] + k=$(ceph auth get-key client.osd-lockbox.$uuid --format=json-pretty 2>/dev/null | \ + jq '.key') + s=$(cat $all_secrets | jq '.cephx_lockbox_secret') + [[ $k == $s ]] + ceph config-key exists dm-crypt/osd/$uuid/luks + + osds=$(ceph osd ls) + id2=$(ceph osd new $uuid2 -i $cephx_only) + for i in $osds; do + [[ "$i" != "$id2" ]] + done + + ceph osd find $id2 + k=$(ceph auth get-key osd.$id --format=json-pretty 2>/dev/null | jq '.key') + s=$(cat $all_secrets | jq '.cephx_secret') + [[ $k == $s ]] + expect_false ceph auth get-key client.osd-lockbox.$uuid2 + expect_false ceph config-key exists dm-crypt/osd/$uuid2/luks + + ceph osd destroy osd.$id2 --yes-i-really-mean-it + ceph osd destroy $id2 --yes-i-really-mean-it + ceph osd find $id2 + expect_false ceph auth get-key osd.$id2 + ceph osd dump | grep osd.$id2 | grep destroyed + + id3=$id2 + uuid3=$(uuidgen) + ceph osd new $uuid3 $id3 -i $all_secrets + ceph osd dump | grep osd.$id3 | expect_false grep destroyed + ceph auth get-key client.osd-lockbox.$uuid3 + ceph auth get-key osd.$id3 + ceph config-key exists dm-crypt/osd/$uuid3/luks + + ceph osd purge-new osd.$id3 --yes-i-really-mean-it + expect_false ceph osd find $id2 + expect_false ceph auth get-key osd.$id2 + expect_false ceph auth get-key client.osd-lockbox.$uuid3 + expect_false ceph config-key exists dm-crypt/osd/$uuid3/luks + ceph osd purge osd.$id3 --yes-i-really-mean-it + ceph osd purge-new osd.$id3 --yes-i-really-mean-it # idempotent + + ceph osd purge osd.$id --yes-i-really-mean-it + ceph osd purge 123456 --yes-i-really-mean-it + expect_false ceph osd find $id + expect_false ceph auth get-key osd.$id + expect_false ceph auth get-key client.osd-lockbox.$uuid + expect_false ceph config-key exists dm-crypt/osd/$uuid/luks + + rm $empty_secrets $empty_json $all_secrets $cephx_only \ + $no_cephx $no_lockbox $bad_json + + for i in $(ceph osd ls); do + [[ "$i" != "$id" ]] + [[ "$i" != "$id2" ]] + [[ "$i" != "$id3" ]] + done + + [[ "$(ceph osd ls | wc -l)" == "$num_osds" ]] + ceph osd setmaxosd $old_maxosd + +} + +function test_mon_config_key() +{ + key=asdfasdfqwerqwreasdfuniquesa123df + ceph config-key list | grep -c $key | grep 0 + ceph config-key get $key | grep -c bar | grep 0 + ceph config-key set $key bar + ceph config-key get $key | grep bar + ceph config-key list | grep -c $key | grep 1 + ceph config-key dump | grep $key | grep bar + ceph config-key rm $key + expect_false ceph config-key get $key + ceph config-key list | grep -c $key | grep 0 + ceph config-key dump | grep -c $key | grep 0 +} + +function test_mon_osd() +{ + # + # osd blocklist + # + bl=192.168.0.1:0/1000 + ceph osd blocklist add $bl + ceph osd blocklist ls | grep $bl + ceph osd blocklist ls --format=json-pretty | sed 's/\\\//\//' | grep $bl + ceph osd dump --format=json-pretty | grep $bl + ceph osd dump | grep $bl + ceph osd blocklist rm $bl + ceph osd blocklist ls | expect_false grep $bl + + bl=192.168.0.1 + # test without nonce, invalid nonce + ceph osd blocklist add $bl + ceph osd blocklist ls | grep $bl + ceph osd blocklist rm $bl + ceph osd blocklist ls | expect_false grep $bl + expect_false "ceph osd blocklist add $bl/-1" + expect_false "ceph osd blocklist add $bl/foo" + + # test with invalid address + expect_false "ceph osd blocklist add 1234.56.78.90/100" + + # test range blocklisting + bl=192.168.0.1:0/24 + ceph osd blocklist range add $bl + ceph osd blocklist ls | grep $bl + ceph osd blocklist range rm $bl + ceph osd blocklist ls | expect_false grep $bl + bad_bl=192.168.0.1/33 + expect_false ceph osd blocklist range add $bad_bl + + # Test `clear` + ceph osd blocklist add $bl + ceph osd blocklist ls | grep $bl + ceph osd blocklist clear + ceph osd blocklist ls | expect_false grep $bl + + # deprecated syntax? + ceph osd blacklist ls + + # + # osd crush + # + ceph osd crush reweight-all + ceph osd crush tunables legacy + ceph osd crush show-tunables | grep argonaut + ceph osd crush tunables bobtail + ceph osd crush show-tunables | grep bobtail + ceph osd crush tunables firefly + ceph osd crush show-tunables | grep firefly + + ceph osd crush set-tunable straw_calc_version 0 + ceph osd crush get-tunable straw_calc_version | grep 0 + ceph osd crush set-tunable straw_calc_version 1 + ceph osd crush get-tunable straw_calc_version | grep 1 + + # + # require-min-compat-client + expect_false ceph osd set-require-min-compat-client dumpling # firefly tunables + ceph osd get-require-min-compat-client | grep luminous + ceph osd dump | grep 'require_min_compat_client luminous' + + # + # osd scrub + # + + # blocking + ceph osd scrub 0 --block + ceph osd deep-scrub 0 --block + + # how do I tell when these are done? + ceph osd scrub 0 + ceph osd deep-scrub 0 + ceph osd repair 0 + + # pool scrub, force-recovery/backfill + pool_names=`rados lspools` + for pool_name in $pool_names + do + ceph osd pool scrub $pool_name + ceph osd pool deep-scrub $pool_name + ceph osd pool repair $pool_name + ceph osd pool force-recovery $pool_name + ceph osd pool cancel-force-recovery $pool_name + ceph osd pool force-backfill $pool_name + ceph osd pool cancel-force-backfill $pool_name + done + + for f in noup nodown noin noout noscrub nodeep-scrub nobackfill \ + norebalance norecover notieragent noautoscale + do + ceph osd set $f + ceph osd unset $f + done + expect_false ceph osd set bogus + expect_false ceph osd unset bogus + for f in sortbitwise recover_deletes require_jewel_osds \ + require_kraken_osds + do + expect_false ceph osd set $f + expect_false ceph osd unset $f + done + ceph osd require-osd-release reef + # can't lower + expect_false ceph osd require-osd-release quincy + expect_false ceph osd require-osd-release pacific + # these are no-ops but should succeed. + + ceph osd set noup + ceph osd down 0 + ceph osd dump | grep 'osd.0 down' + ceph osd unset noup + max_run=1000 + for ((i=0; i < $max_run; i++)); do + if ! ceph osd dump | grep 'osd.0 up'; then + echo "waiting for osd.0 to come back up ($i/$max_run)" + sleep 1 + else + break + fi + done + ceph osd dump | grep 'osd.0 up' + + ceph osd dump | grep 'osd.0 up' + # ceph osd find expects the OsdName, so both ints and osd.n should work. + ceph osd find 1 + ceph osd find osd.1 + expect_false ceph osd find osd.xyz + expect_false ceph osd find xyz + expect_false ceph osd find 0.1 + ceph --format plain osd find 1 # falls back to json-pretty + if [ `uname` == Linux ]; then + ceph osd metadata 1 | grep 'distro' + ceph --format plain osd metadata 1 | grep 'distro' # falls back to json-pretty + fi + ceph osd out 0 + ceph osd dump | grep 'osd.0.*out' + ceph osd in 0 + ceph osd dump | grep 'osd.0.*in' + ceph osd find 0 + + ceph osd info 0 + ceph osd info osd.0 + expect_false ceph osd info osd.xyz + expect_false ceph osd info xyz + expect_false ceph osd info 42 + expect_false ceph osd info osd.42 + + ceph osd info + info_json=$(ceph osd info --format=json | jq -cM '.') + dump_json=$(ceph osd dump --format=json | jq -cM '.osds') + if [[ "${info_json}" != "${dump_json}" ]]; then + echo "waiting for OSDs to settle" + sleep 10 + info_json=$(ceph osd info --format=json | jq -cM '.') + dump_json=$(ceph osd dump --format=json | jq -cM '.osds') + [[ "${info_json}" == "${dump_json}" ]] + fi + + info_json=$(ceph osd info 0 --format=json | jq -cM '.') + dump_json=$(ceph osd dump --format=json | \ + jq -cM '.osds[] | select(.osd == 0)') + [[ "${info_json}" == "${dump_json}" ]] + + info_plain="$(ceph osd info)" + dump_plain="$(ceph osd dump | grep '^osd')" + [[ "${info_plain}" == "${dump_plain}" ]] + + info_plain="$(ceph osd info 0)" + dump_plain="$(ceph osd dump | grep '^osd.0')" + [[ "${info_plain}" == "${dump_plain}" ]] + + ceph osd add-nodown 0 1 + ceph health detail | grep 'NODOWN' + ceph osd rm-nodown 0 1 + ! ceph health detail | grep 'NODOWN' + + ceph osd out 0 # so we can mark it as noin later + ceph osd add-noin 0 + ceph health detail | grep 'NOIN' + ceph osd rm-noin 0 + ! ceph health detail | grep 'NOIN' + ceph osd in 0 + + ceph osd add-noout 0 + ceph health detail | grep 'NOOUT' + ceph osd rm-noout 0 + ! ceph health detail | grep 'NOOUT' + + # test osd id parse + expect_false ceph osd add-noup 797er + expect_false ceph osd add-nodown u9uwer + expect_false ceph osd add-noin 78~15 + + expect_false ceph osd rm-noup 1234567 + expect_false ceph osd rm-nodown fsadf7 + expect_false ceph osd rm-noout 790-fd + + ids=`ceph osd ls-tree default` + for osd in $ids + do + ceph osd add-nodown $osd + ceph osd add-noout $osd + done + ceph -s | grep 'NODOWN' + ceph -s | grep 'NOOUT' + ceph osd rm-nodown any + ceph osd rm-noout all + ! ceph -s | grep 'NODOWN' + ! ceph -s | grep 'NOOUT' + + # test crush node flags + ceph osd add-noup osd.0 + ceph osd add-nodown osd.0 + ceph osd add-noin osd.0 + ceph osd add-noout osd.0 + ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep "osd.0" + ceph osd rm-noup osd.0 + ceph osd rm-nodown osd.0 + ceph osd rm-noin osd.0 + ceph osd rm-noout osd.0 + ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep "osd.0" + + ceph osd crush add-bucket foo host root=default + ceph osd add-noup foo + ceph osd add-nodown foo + ceph osd add-noin foo + ceph osd add-noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags" | grep foo + ceph osd rm-noup foo + ceph osd rm-nodown foo + ceph osd rm-noin foo + ceph osd rm-noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep foo + ceph osd add-noup foo + ceph osd dump -f json-pretty | jq ".crush_node_flags" | grep foo + ceph osd crush rm foo + ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep foo + + ceph osd set-group noup osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd set-group noup,nodown osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd set-group noup,nodown,noin osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd set-group noup,nodown,noin,noout osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd unset-group noup osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd unset-group noup,nodown osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd unset-group noup,nodown,noin osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown\|noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd unset-group noup,nodown,noin,noout osd.0 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown\|noin\|noout' + + ceph osd set-group noup,nodown,noin,noout osd.0 osd.1 + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout' + ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'noup' + ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'noin' + ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'noout' + ceph osd unset-group noup,nodown,noin,noout osd.0 osd.1 + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown\|noin\|noout' + ceph osd dump -f json-pretty | jq ".osds[1].state" | expect_false grep 'noup\|nodown\|noin\|noout' + + ceph osd set-group noup all + ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup' + ceph osd unset-group noup all + ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup' + + # crush node flags + ceph osd crush add-bucket foo host root=default + ceph osd set-group noup foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd set-group noup,nodown foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd set-group noup,nodown,noin foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd set-group noup,nodown,noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + + ceph osd unset-group noup foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd unset-group noup,nodown foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup\|nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd unset-group noup,nodown,noin foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup\|nodown\|noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd unset-group noup,nodown,noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup\|nodown\|noin\|noout' + + ceph osd set-group noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd unset-group noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep 'foo' + + ceph osd set-group noup,nodown,noin,noout foo + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin' + ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout' + ceph osd crush rm foo + ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep 'foo' + + # test device class flags + osd_0_device_class=$(ceph osd crush get-device-class osd.0) + ceph osd set-group noup $osd_0_device_class + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noup' + ceph osd set-group noup,nodown $osd_0_device_class + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noup' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'nodown' + ceph osd set-group noup,nodown,noin $osd_0_device_class + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noup' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin' + ceph osd set-group noup,nodown,noin,noout $osd_0_device_class + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noup' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout' + + ceph osd unset-group noup $osd_0_device_class + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | expect_false grep 'noup' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'nodown' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout' + ceph osd unset-group noup,nodown $osd_0_device_class + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | expect_false grep 'noup\|nodown' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout' + ceph osd unset-group noup,nodown,noin $osd_0_device_class + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | expect_false grep 'noup\|nodown\|noin' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout' + ceph osd unset-group noup,nodown,noin,noout $osd_0_device_class + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | expect_false grep 'noup\|nodown\|noin\|noout' + + ceph osd set-group noin,noout $osd_0_device_class + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin' + ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout' + ceph osd unset-group noin,noout $osd_0_device_class + ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep $osd_0_device_class + + # make sure mark out preserves weight + ceph osd reweight osd.0 .5 + ceph osd dump | grep ^osd.0 | grep 'weight 0.5' + ceph osd out 0 + ceph osd in 0 + ceph osd dump | grep ^osd.0 | grep 'weight 0.5' + + ceph osd getmap -o $f + [ -s $f ] + rm $f + save=$(ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//') + [ "$save" -gt 0 ] + ceph osd setmaxosd $((save - 1)) 2>&1 | grep 'EBUSY' + ceph osd setmaxosd 10 + ceph osd getmaxosd | grep 'max_osd = 10' + ceph osd setmaxosd $save + ceph osd getmaxosd | grep "max_osd = $save" + + for id in `ceph osd ls` ; do + retry_eagain 5 map_enxio_to_eagain ceph tell osd.$id version + done + + ceph osd rm 0 2>&1 | grep 'EBUSY' + + local old_osds=$(echo $(ceph osd ls)) + id=`ceph osd create` + ceph osd find $id + ceph osd lost $id --yes-i-really-mean-it + expect_false ceph osd setmaxosd $id + local new_osds=$(echo $(ceph osd ls)) + for id in $(echo $new_osds | sed -e "s/$old_osds//") ; do + ceph osd rm $id + done + + uuid=`uuidgen` + id=`ceph osd create $uuid` + id2=`ceph osd create $uuid` + [ "$id" = "$id2" ] + ceph osd rm $id + + ceph --help osd + + # reset max_osd. + ceph osd setmaxosd $id + ceph osd getmaxosd | grep "max_osd = $save" + local max_osd=$save + + ceph osd create $uuid 0 2>&1 | grep 'EINVAL' + ceph osd create $uuid $((max_osd - 1)) 2>&1 | grep 'EINVAL' + + id=`ceph osd create $uuid $max_osd` + [ "$id" = "$max_osd" ] + ceph osd find $id + max_osd=$((max_osd + 1)) + ceph osd getmaxosd | grep "max_osd = $max_osd" + + ceph osd create $uuid $((id - 1)) 2>&1 | grep 'EEXIST' + ceph osd create $uuid $((id + 1)) 2>&1 | grep 'EEXIST' + id2=`ceph osd create $uuid` + [ "$id" = "$id2" ] + id2=`ceph osd create $uuid $id` + [ "$id" = "$id2" ] + + uuid=`uuidgen` + local gap_start=$max_osd + id=`ceph osd create $uuid $((gap_start + 100))` + [ "$id" = "$((gap_start + 100))" ] + max_osd=$((id + 1)) + ceph osd getmaxosd | grep "max_osd = $max_osd" + + ceph osd create $uuid $gap_start 2>&1 | grep 'EEXIST' + + # + # When CEPH_CLI_TEST_DUP_COMMAND is set, osd create + # is repeated and consumes two osd id, not just one. + # + local next_osd=$gap_start + id=`ceph osd create $(uuidgen)` + [ "$id" = "$next_osd" ] + + next_osd=$((id + 1)) + id=`ceph osd create $(uuidgen) $next_osd` + [ "$id" = "$next_osd" ] + + local new_osds=$(echo $(ceph osd ls)) + for id in $(echo $new_osds | sed -e "s/$old_osds//") ; do + [ $id -ge $save ] + ceph osd rm $id + done + ceph osd setmaxosd $save + + ceph osd ls + ceph osd pool create data 16 + ceph osd pool application enable data rados + ceph osd lspools | grep data + ceph osd map data foo | grep 'pool.*data.*object.*foo.*pg.*up.*acting' + ceph osd map data foo namespace| grep 'pool.*data.*object.*namespace/foo.*pg.*up.*acting' + ceph osd pool delete data data --yes-i-really-really-mean-it + + ceph osd pause + ceph osd dump | grep 'flags.*pauserd,pausewr' + ceph osd unpause + + ceph osd tree + ceph osd tree up + ceph osd tree down + ceph osd tree in + ceph osd tree out + ceph osd tree destroyed + ceph osd tree up in + ceph osd tree up out + ceph osd tree down in + ceph osd tree down out + ceph osd tree out down + expect_false ceph osd tree up down + expect_false ceph osd tree up destroyed + expect_false ceph osd tree down destroyed + expect_false ceph osd tree up down destroyed + expect_false ceph osd tree in out + expect_false ceph osd tree up foo + + ceph osd metadata + ceph osd count-metadata os + ceph osd versions + + ceph osd perf + ceph osd blocked-by + + ceph osd stat | grep up +} + +function test_mon_crush() +{ + f=$TEMP_DIR/map.$$ + epoch=$(ceph osd getcrushmap -o $f 2>&1 | tail -n1) + [ -s $f ] + [ "$epoch" -gt 1 ] + nextepoch=$(( $epoch + 1 )) + echo epoch $epoch nextepoch $nextepoch + rm -f $f.epoch + expect_false ceph osd setcrushmap $nextepoch -i $f + gotepoch=$(ceph osd setcrushmap $epoch -i $f 2>&1 | tail -n1) + echo gotepoch $gotepoch + [ "$gotepoch" -eq "$nextepoch" ] + # should be idempotent + gotepoch=$(ceph osd setcrushmap $epoch -i $f 2>&1 | tail -n1) + echo epoch $gotepoch + [ "$gotepoch" -eq "$nextepoch" ] + rm $f +} + +function test_mon_osd_pool() +{ + # + # osd pool + # + ceph osd pool create data 16 + ceph osd pool application enable data rados + ceph osd pool mksnap data datasnap + rados -p data lssnap | grep datasnap + ceph osd pool rmsnap data datasnap + expect_false ceph osd pool rmsnap pool_fake snapshot + ceph osd pool delete data data --yes-i-really-really-mean-it + + ceph osd pool create data2 16 + ceph osd pool application enable data2 rados + ceph osd pool rename data2 data3 + ceph osd lspools | grep data3 + ceph osd pool delete data3 data3 --yes-i-really-really-mean-it + + ceph osd pool create replicated 16 16 replicated + ceph osd pool create replicated 1 16 replicated + ceph osd pool create replicated 16 16 # default is replicated + ceph osd pool create replicated 16 # default is replicated, pgp_num = pg_num + ceph osd pool application enable replicated rados + # should fail because the type is not the same + expect_false ceph osd pool create replicated 16 16 erasure + ceph osd lspools | grep replicated + ceph osd pool create ec_test 1 1 erasure + ceph osd pool application enable ec_test rados + set +e + ceph osd count-metadata osd_objectstore | grep 'bluestore' + if [ $? -eq 1 ]; then # enable ec_overwrites on non-bluestore pools should fail + ceph osd pool set ec_test allow_ec_overwrites true >& $TMPFILE + check_response "pool must only be stored on bluestore for scrubbing to work" $? 22 + else + ceph osd pool set ec_test allow_ec_overwrites true || return 1 + expect_false ceph osd pool set ec_test allow_ec_overwrites false + fi + set -e + ceph osd pool delete replicated replicated --yes-i-really-really-mean-it + ceph osd pool delete ec_test ec_test --yes-i-really-really-mean-it + + # test create pool with rule + ceph osd erasure-code-profile set foo foo + ceph osd erasure-code-profile ls | grep foo + ceph osd crush rule create-erasure foo foo + ceph osd pool create erasure 16 16 erasure foo + expect_false ceph osd erasure-code-profile rm foo + ceph osd pool delete erasure erasure --yes-i-really-really-mean-it + ceph osd crush rule rm foo + ceph osd erasure-code-profile rm foo + + # autoscale mode + ceph osd pool create modeon --autoscale-mode=on + ceph osd dump | grep modeon | grep 'autoscale_mode on' + ceph osd pool create modewarn --autoscale-mode=warn + ceph osd dump | grep modewarn | grep 'autoscale_mode warn' + ceph osd pool create modeoff --autoscale-mode=off + ceph osd dump | grep modeoff | grep 'autoscale_mode off' + ceph osd pool delete modeon modeon --yes-i-really-really-mean-it + ceph osd pool delete modewarn modewarn --yes-i-really-really-mean-it + ceph osd pool delete modeoff modeoff --yes-i-really-really-mean-it +} + +function test_mon_osd_pool_quota() +{ + # + # test osd pool set/get quota + # + + # create tmp pool + ceph osd pool create tmp-quota-pool 32 + ceph osd pool application enable tmp-quota-pool rados + # + # set erroneous quotas + # + expect_false ceph osd pool set-quota tmp-quota-pool max_fooness 10 + expect_false ceph osd pool set-quota tmp-quota-pool max_bytes -1 + expect_false ceph osd pool set-quota tmp-quota-pool max_objects aaa + # + # set valid quotas + # + ceph osd pool set-quota tmp-quota-pool max_bytes 10 + ceph osd pool set-quota tmp-quota-pool max_objects 10M + # + # get quotas in json-pretty format + # + ceph osd pool get-quota tmp-quota-pool --format=json-pretty | \ + grep '"quota_max_objects":.*10000000' + ceph osd pool get-quota tmp-quota-pool --format=json-pretty | \ + grep '"quota_max_bytes":.*10' + # + # get quotas + # + ceph osd pool get-quota tmp-quota-pool | grep 'max bytes.*10 B' + ceph osd pool get-quota tmp-quota-pool | grep 'max objects.*10.*M objects' + # + # set valid quotas with unit prefix + # + ceph osd pool set-quota tmp-quota-pool max_bytes 10K + # + # get quotas + # + ceph osd pool get-quota tmp-quota-pool | grep 'max bytes.*10 Ki' + # + # set valid quotas with unit prefix + # + ceph osd pool set-quota tmp-quota-pool max_bytes 10Ki + # + # get quotas + # + ceph osd pool get-quota tmp-quota-pool | grep 'max bytes.*10 Ki' + # + # + # reset pool quotas + # + ceph osd pool set-quota tmp-quota-pool max_bytes 0 + ceph osd pool set-quota tmp-quota-pool max_objects 0 + # + # test N/A quotas + # + ceph osd pool get-quota tmp-quota-pool | grep 'max bytes.*N/A' + ceph osd pool get-quota tmp-quota-pool | grep 'max objects.*N/A' + # + # cleanup tmp pool + ceph osd pool delete tmp-quota-pool tmp-quota-pool --yes-i-really-really-mean-it +} + +function test_mon_pg() +{ + # Make sure we start healthy. + wait_for_health_ok + + ceph pg debug unfound_objects_exist + ceph pg debug degraded_pgs_exist + ceph pg deep-scrub 1.0 + ceph pg dump + ceph pg dump pgs_brief --format=json + ceph pg dump pgs --format=json + ceph pg dump pools --format=json + ceph pg dump osds --format=json + ceph pg dump sum --format=json + ceph pg dump all --format=json + ceph pg dump pgs_brief osds --format=json + ceph pg dump pools osds pgs_brief --format=json + ceph pg dump_json + ceph pg dump_pools_json + ceph pg dump_stuck inactive + ceph pg dump_stuck unclean + ceph pg dump_stuck stale + ceph pg dump_stuck undersized + ceph pg dump_stuck degraded + ceph pg ls + ceph pg ls 1 + ceph pg ls stale + expect_false ceph pg ls scrubq + ceph pg ls active stale repair recovering + ceph pg ls 1 active + ceph pg ls 1 active stale + ceph pg ls-by-primary osd.0 + ceph pg ls-by-primary osd.0 1 + ceph pg ls-by-primary osd.0 active + ceph pg ls-by-primary osd.0 active stale + ceph pg ls-by-primary osd.0 1 active stale + ceph pg ls-by-osd osd.0 + ceph pg ls-by-osd osd.0 1 + ceph pg ls-by-osd osd.0 active + ceph pg ls-by-osd osd.0 active stale + ceph pg ls-by-osd osd.0 1 active stale + ceph pg ls-by-pool rbd + ceph pg ls-by-pool rbd active stale + # can't test this... + # ceph pg force_create_pg + ceph pg getmap -o $TEMP_DIR/map.$$ + [ -s $TEMP_DIR/map.$$ ] + ceph pg map 1.0 | grep acting + ceph pg repair 1.0 + ceph pg scrub 1.0 + + ceph osd set-full-ratio .962 + ceph osd dump | grep '^full_ratio 0.962' + ceph osd set-backfillfull-ratio .912 + ceph osd dump | grep '^backfillfull_ratio 0.912' + ceph osd set-nearfull-ratio .892 + ceph osd dump | grep '^nearfull_ratio 0.892' + + # Check health status + ceph osd set-nearfull-ratio .913 + ceph health -f json | grep OSD_OUT_OF_ORDER_FULL + ceph health detail | grep OSD_OUT_OF_ORDER_FULL + ceph osd set-nearfull-ratio .892 + ceph osd set-backfillfull-ratio .963 + ceph health -f json | grep OSD_OUT_OF_ORDER_FULL + ceph health detail | grep OSD_OUT_OF_ORDER_FULL + ceph osd set-backfillfull-ratio .912 + + # Check injected full results + $SUDO ceph tell osd.0 injectfull nearfull + wait_for_health "OSD_NEARFULL" + ceph health detail | grep "osd.0 is near full" + $SUDO ceph tell osd.0 injectfull none + wait_for_health_ok + + $SUDO ceph tell osd.1 injectfull backfillfull + wait_for_health "OSD_BACKFILLFULL" + ceph health detail | grep "osd.1 is backfill full" + $SUDO ceph tell osd.1 injectfull none + wait_for_health_ok + + $SUDO ceph tell osd.2 injectfull failsafe + # failsafe and full are the same as far as the monitor is concerned + wait_for_health "OSD_FULL" + ceph health detail | grep "osd.2 is full" + $SUDO ceph tell osd.2 injectfull none + wait_for_health_ok + + $SUDO ceph tell osd.0 injectfull full + wait_for_health "OSD_FULL" + ceph health detail | grep "osd.0 is full" + $SUDO ceph tell osd.0 injectfull none + wait_for_health_ok + + ceph pg stat | grep 'pgs:' + ceph pg 1.0 query + ceph tell 1.0 query + first=$(ceph mon dump -f json | jq -r '.mons[0].name') + ceph tell mon.$first quorum enter + ceph quorum_status + ceph report | grep osd_stats + ceph status + ceph -s + + # + # tell osd version + # + ceph tell osd.0 version + expect_false ceph tell osd.9999 version + expect_false ceph tell osd.foo version + + # back to pg stuff + + ceph tell osd.0 dump_pg_recovery_stats | grep Started + + ceph osd reweight 0 0.9 + expect_false ceph osd reweight 0 -1 + ceph osd reweight osd.0 1 + + ceph osd primary-affinity osd.0 .9 + expect_false ceph osd primary-affinity osd.0 -2 + expect_false ceph osd primary-affinity osd.9999 .5 + ceph osd primary-affinity osd.0 1 + + ceph osd pool set rbd size 2 + ceph osd pg-temp 1.0 0 1 + ceph osd pg-temp 1.0 osd.1 osd.0 + expect_false ceph osd pg-temp 1.0 0 1 2 + expect_false ceph osd pg-temp asdf qwer + expect_false ceph osd pg-temp 1.0 asdf + ceph osd pg-temp 1.0 # cleanup pg-temp + + ceph pg repeer 1.0 + expect_false ceph pg repeer 0.0 # pool 0 shouldn't exist anymore + + # don't test ceph osd primary-temp for now +} + +function test_mon_osd_pool_set() +{ + TEST_POOL_GETSET=pool_getset + expect_false ceph osd pool create $TEST_POOL_GETSET 1 --target_size_ratio -0.3 + expect_true ceph osd pool create $TEST_POOL_GETSET 1 --target_size_ratio 1 + ceph osd pool application enable $TEST_POOL_GETSET rados + ceph osd pool set $TEST_POOL_GETSET pg_autoscale_mode off + wait_for_clean + ceph osd pool get $TEST_POOL_GETSET all + + for s in pg_num pgp_num size min_size crush_rule target_size_ratio; do + ceph osd pool get $TEST_POOL_GETSET $s + done + + old_size=$(ceph osd pool get $TEST_POOL_GETSET size | sed -e 's/size: //') + (( new_size = old_size + 1 )) + ceph osd pool set $TEST_POOL_GETSET size $new_size --yes-i-really-mean-it + ceph osd pool get $TEST_POOL_GETSET size | grep "size: $new_size" + ceph osd pool set $TEST_POOL_GETSET size $old_size --yes-i-really-mean-it + + ceph osd pool create pool_erasure 1 1 erasure + ceph osd pool application enable pool_erasure rados + wait_for_clean + set +e + ceph osd pool set pool_erasure size 4444 2>$TMPFILE + check_response 'not change the size' + set -e + ceph osd pool get pool_erasure erasure_code_profile + ceph osd pool rm pool_erasure pool_erasure --yes-i-really-really-mean-it + + for flag in nodelete nopgchange nosizechange write_fadvise_dontneed noscrub nodeep-scrub bulk; do + ceph osd pool set $TEST_POOL_GETSET $flag false + ceph osd pool get $TEST_POOL_GETSET $flag | grep "$flag: false" + ceph osd pool set $TEST_POOL_GETSET $flag true + ceph osd pool get $TEST_POOL_GETSET $flag | grep "$flag: true" + ceph osd pool set $TEST_POOL_GETSET $flag 1 + ceph osd pool get $TEST_POOL_GETSET $flag | grep "$flag: true" + ceph osd pool set $TEST_POOL_GETSET $flag 0 + ceph osd pool get $TEST_POOL_GETSET $flag | grep "$flag: false" + expect_false ceph osd pool set $TEST_POOL_GETSET $flag asdf + expect_false ceph osd pool set $TEST_POOL_GETSET $flag 2 + done + + ceph osd pool get $TEST_POOL_GETSET scrub_min_interval | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET scrub_min_interval 123456 + ceph osd pool get $TEST_POOL_GETSET scrub_min_interval | grep 'scrub_min_interval: 123456' + ceph osd pool set $TEST_POOL_GETSET scrub_min_interval 0 + ceph osd pool get $TEST_POOL_GETSET scrub_min_interval | expect_false grep '.' + + ceph osd pool get $TEST_POOL_GETSET scrub_max_interval | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET scrub_max_interval 123456 + ceph osd pool get $TEST_POOL_GETSET scrub_max_interval | grep 'scrub_max_interval: 123456' + ceph osd pool set $TEST_POOL_GETSET scrub_max_interval 0 + ceph osd pool get $TEST_POOL_GETSET scrub_max_interval | expect_false grep '.' + + ceph osd pool get $TEST_POOL_GETSET deep_scrub_interval | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET deep_scrub_interval 123456 + ceph osd pool get $TEST_POOL_GETSET deep_scrub_interval | grep 'deep_scrub_interval: 123456' + ceph osd pool set $TEST_POOL_GETSET deep_scrub_interval 0 + ceph osd pool get $TEST_POOL_GETSET deep_scrub_interval | expect_false grep '.' + + ceph osd pool get $TEST_POOL_GETSET recovery_priority | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET recovery_priority 5 + ceph osd pool get $TEST_POOL_GETSET recovery_priority | grep 'recovery_priority: 5' + ceph osd pool set $TEST_POOL_GETSET recovery_priority -5 + ceph osd pool get $TEST_POOL_GETSET recovery_priority | grep 'recovery_priority: -5' + ceph osd pool set $TEST_POOL_GETSET recovery_priority 0 + ceph osd pool get $TEST_POOL_GETSET recovery_priority | expect_false grep '.' + expect_false ceph osd pool set $TEST_POOL_GETSET recovery_priority -11 + expect_false ceph osd pool set $TEST_POOL_GETSET recovery_priority 11 + + ceph osd pool get $TEST_POOL_GETSET recovery_op_priority | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET recovery_op_priority 5 + ceph osd pool get $TEST_POOL_GETSET recovery_op_priority | grep 'recovery_op_priority: 5' + ceph osd pool set $TEST_POOL_GETSET recovery_op_priority 0 + ceph osd pool get $TEST_POOL_GETSET recovery_op_priority | expect_false grep '.' + + ceph osd pool get $TEST_POOL_GETSET scrub_priority | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET scrub_priority 5 + ceph osd pool get $TEST_POOL_GETSET scrub_priority | grep 'scrub_priority: 5' + ceph osd pool set $TEST_POOL_GETSET scrub_priority 0 + ceph osd pool get $TEST_POOL_GETSET scrub_priority | expect_false grep '.' + + expect_false ceph osd pool set $TEST_POOL_GETSET target_size_ratio -3 + expect_false ceph osd pool set $TEST_POOL_GETSET target_size_ratio abc + expect_true ceph osd pool set $TEST_POOL_GETSET target_size_ratio 0.1 + expect_true ceph osd pool set $TEST_POOL_GETSET target_size_ratio 1 + ceph osd pool get $TEST_POOL_GETSET target_size_ratio | grep 'target_size_ratio: 1' + + ceph osd pool set $TEST_POOL_GETSET nopgchange 1 + expect_false ceph osd pool set $TEST_POOL_GETSET pg_num 10 + expect_false ceph osd pool set $TEST_POOL_GETSET pgp_num 10 + ceph osd pool set $TEST_POOL_GETSET nopgchange 0 + ceph osd pool set $TEST_POOL_GETSET pg_num 10 + wait_for_clean + ceph osd pool set $TEST_POOL_GETSET pgp_num 10 + expect_false ceph osd pool set $TEST_POOL_GETSET pg_num 0 + expect_false ceph osd pool set $TEST_POOL_GETSET pgp_num 0 + + old_pgs=$(ceph osd pool get $TEST_POOL_GETSET pg_num | sed -e 's/pg_num: //') + new_pgs=$(($old_pgs + $(ceph osd stat --format json | jq '.num_osds') * 32)) + ceph osd pool set $TEST_POOL_GETSET pg_num $new_pgs + ceph osd pool set $TEST_POOL_GETSET pgp_num $new_pgs + wait_for_clean + + ceph osd pool set $TEST_POOL_GETSET nosizechange 1 + expect_false ceph osd pool set $TEST_POOL_GETSET size 2 + expect_false ceph osd pool set $TEST_POOL_GETSET min_size 2 + ceph osd pool set $TEST_POOL_GETSET nosizechange 0 + ceph osd pool set $TEST_POOL_GETSET size 2 + wait_for_clean + ceph osd pool set $TEST_POOL_GETSET min_size 2 + + expect_false ceph osd pool set $TEST_POOL_GETSET hashpspool 0 + ceph osd pool set $TEST_POOL_GETSET hashpspool 0 --yes-i-really-mean-it + + expect_false ceph osd pool set $TEST_POOL_GETSET hashpspool 1 + ceph osd pool set $TEST_POOL_GETSET hashpspool 1 --yes-i-really-mean-it + + ceph osd pool get rbd crush_rule | grep 'crush_rule: ' + + ceph osd pool get $TEST_POOL_GETSET compression_mode | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET compression_mode aggressive + ceph osd pool get $TEST_POOL_GETSET compression_mode | grep 'aggressive' + ceph osd pool set $TEST_POOL_GETSET compression_mode unset + ceph osd pool get $TEST_POOL_GETSET compression_mode | expect_false grep '.' + + ceph osd pool get $TEST_POOL_GETSET compression_algorithm | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET compression_algorithm zlib + ceph osd pool get $TEST_POOL_GETSET compression_algorithm | grep 'zlib' + ceph osd pool set $TEST_POOL_GETSET compression_algorithm unset + ceph osd pool get $TEST_POOL_GETSET compression_algorithm | expect_false grep '.' + + ceph osd pool get $TEST_POOL_GETSET compression_required_ratio | expect_false grep '.' + expect_false ceph osd pool set $TEST_POOL_GETSET compression_required_ratio 1.1 + expect_false ceph osd pool set $TEST_POOL_GETSET compression_required_ratio -.2 + ceph osd pool set $TEST_POOL_GETSET compression_required_ratio .2 + ceph osd pool get $TEST_POOL_GETSET compression_required_ratio | grep '.2' + ceph osd pool set $TEST_POOL_GETSET compression_required_ratio 0 + ceph osd pool get $TEST_POOL_GETSET compression_required_ratio | expect_false grep '.' + + ceph osd pool get $TEST_POOL_GETSET csum_type | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET csum_type crc32c + ceph osd pool get $TEST_POOL_GETSET csum_type | grep 'crc32c' + ceph osd pool set $TEST_POOL_GETSET csum_type unset + ceph osd pool get $TEST_POOL_GETSET csum_type | expect_false grep '.' + + for size in compression_max_blob_size compression_min_blob_size csum_max_block csum_min_block; do + ceph osd pool get $TEST_POOL_GETSET $size | expect_false grep '.' + ceph osd pool set $TEST_POOL_GETSET $size 100 + ceph osd pool get $TEST_POOL_GETSET $size | grep '100' + ceph osd pool set $TEST_POOL_GETSET $size 0 + ceph osd pool get $TEST_POOL_GETSET $size | expect_false grep '.' + done + + ceph osd pool set $TEST_POOL_GETSET nodelete 1 + expect_false ceph osd pool delete $TEST_POOL_GETSET $TEST_POOL_GETSET --yes-i-really-really-mean-it + ceph osd pool set $TEST_POOL_GETSET nodelete 0 + ceph osd pool delete $TEST_POOL_GETSET $TEST_POOL_GETSET --yes-i-really-really-mean-it + +} + +function test_mon_osd_tiered_pool_set() +{ + # this is really a tier pool + ceph osd pool create real-tier 2 + ceph osd tier add rbd real-tier + + # expect us to be unable to set negative values for hit_set_* + for o in hit_set_period hit_set_count hit_set_fpp; do + expect_false ceph osd pool set real_tier $o -1 + done + + # and hit_set_fpp should be in range 0..1 + expect_false ceph osd pool set real_tier hit_set_fpp 2 + + ceph osd pool set real-tier hit_set_type explicit_hash + ceph osd pool get real-tier hit_set_type | grep "hit_set_type: explicit_hash" + ceph osd pool set real-tier hit_set_type explicit_object + ceph osd pool get real-tier hit_set_type | grep "hit_set_type: explicit_object" + ceph osd pool set real-tier hit_set_type bloom + ceph osd pool get real-tier hit_set_type | grep "hit_set_type: bloom" + expect_false ceph osd pool set real-tier hit_set_type i_dont_exist + ceph osd pool set real-tier hit_set_period 123 + ceph osd pool get real-tier hit_set_period | grep "hit_set_period: 123" + ceph osd pool set real-tier hit_set_count 12 + ceph osd pool get real-tier hit_set_count | grep "hit_set_count: 12" + ceph osd pool set real-tier hit_set_fpp .01 + ceph osd pool get real-tier hit_set_fpp | grep "hit_set_fpp: 0.01" + + ceph osd pool set real-tier target_max_objects 123 + ceph osd pool get real-tier target_max_objects | \ + grep 'target_max_objects:[ \t]\+123' + ceph osd pool set real-tier target_max_bytes 123456 + ceph osd pool get real-tier target_max_bytes | \ + grep 'target_max_bytes:[ \t]\+123456' + ceph osd pool set real-tier cache_target_dirty_ratio .123 + ceph osd pool get real-tier cache_target_dirty_ratio | \ + grep 'cache_target_dirty_ratio:[ \t]\+0.123' + expect_false ceph osd pool set real-tier cache_target_dirty_ratio -.2 + expect_false ceph osd pool set real-tier cache_target_dirty_ratio 1.1 + ceph osd pool set real-tier cache_target_dirty_high_ratio .123 + ceph osd pool get real-tier cache_target_dirty_high_ratio | \ + grep 'cache_target_dirty_high_ratio:[ \t]\+0.123' + expect_false ceph osd pool set real-tier cache_target_dirty_high_ratio -.2 + expect_false ceph osd pool set real-tier cache_target_dirty_high_ratio 1.1 + ceph osd pool set real-tier cache_target_full_ratio .123 + ceph osd pool get real-tier cache_target_full_ratio | \ + grep 'cache_target_full_ratio:[ \t]\+0.123' + ceph osd dump -f json-pretty | grep '"cache_target_full_ratio_micro": 123000' + ceph osd pool set real-tier cache_target_full_ratio 1.0 + ceph osd pool set real-tier cache_target_full_ratio 0 + expect_false ceph osd pool set real-tier cache_target_full_ratio 1.1 + ceph osd pool set real-tier cache_min_flush_age 123 + ceph osd pool get real-tier cache_min_flush_age | \ + grep 'cache_min_flush_age:[ \t]\+123' + ceph osd pool set real-tier cache_min_evict_age 234 + ceph osd pool get real-tier cache_min_evict_age | \ + grep 'cache_min_evict_age:[ \t]\+234' + + # iec vs si units + ceph osd pool set real-tier target_max_objects 1K + ceph osd pool get real-tier target_max_objects | grep 1000 + for o in target_max_bytes target_size_bytes compression_max_blob_size compression_min_blob_size csum_max_block csum_min_block; do + ceph osd pool set real-tier $o 1Ki # no i suffix + val=$(ceph osd pool get real-tier $o --format=json | jq -c ".$o") + [[ $val == 1024 ]] + ceph osd pool set real-tier $o 1M # with i suffix + val=$(ceph osd pool get real-tier $o --format=json | jq -c ".$o") + [[ $val == 1048576 ]] + done + + # this is not a tier pool + ceph osd pool create fake-tier 2 + ceph osd pool application enable fake-tier rados + wait_for_clean + + expect_false ceph osd pool set fake-tier hit_set_type explicit_hash + expect_false ceph osd pool get fake-tier hit_set_type + expect_false ceph osd pool set fake-tier hit_set_type explicit_object + expect_false ceph osd pool get fake-tier hit_set_type + expect_false ceph osd pool set fake-tier hit_set_type bloom + expect_false ceph osd pool get fake-tier hit_set_type + expect_false ceph osd pool set fake-tier hit_set_type i_dont_exist + expect_false ceph osd pool set fake-tier hit_set_period 123 + expect_false ceph osd pool get fake-tier hit_set_period + expect_false ceph osd pool set fake-tier hit_set_count 12 + expect_false ceph osd pool get fake-tier hit_set_count + expect_false ceph osd pool set fake-tier hit_set_fpp .01 + expect_false ceph osd pool get fake-tier hit_set_fpp + + expect_false ceph osd pool set fake-tier target_max_objects 123 + expect_false ceph osd pool get fake-tier target_max_objects + expect_false ceph osd pool set fake-tier target_max_bytes 123456 + expect_false ceph osd pool get fake-tier target_max_bytes + expect_false ceph osd pool set fake-tier cache_target_dirty_ratio .123 + expect_false ceph osd pool get fake-tier cache_target_dirty_ratio + expect_false ceph osd pool set fake-tier cache_target_dirty_ratio -.2 + expect_false ceph osd pool set fake-tier cache_target_dirty_ratio 1.1 + expect_false ceph osd pool set fake-tier cache_target_dirty_high_ratio .123 + expect_false ceph osd pool get fake-tier cache_target_dirty_high_ratio + expect_false ceph osd pool set fake-tier cache_target_dirty_high_ratio -.2 + expect_false ceph osd pool set fake-tier cache_target_dirty_high_ratio 1.1 + expect_false ceph osd pool set fake-tier cache_target_full_ratio .123 + expect_false ceph osd pool get fake-tier cache_target_full_ratio + expect_false ceph osd pool set fake-tier cache_target_full_ratio 1.0 + expect_false ceph osd pool set fake-tier cache_target_full_ratio 0 + expect_false ceph osd pool set fake-tier cache_target_full_ratio 1.1 + expect_false ceph osd pool set fake-tier cache_min_flush_age 123 + expect_false ceph osd pool get fake-tier cache_min_flush_age + expect_false ceph osd pool set fake-tier cache_min_evict_age 234 + expect_false ceph osd pool get fake-tier cache_min_evict_age + + ceph osd tier remove rbd real-tier + ceph osd pool delete real-tier real-tier --yes-i-really-really-mean-it + ceph osd pool delete fake-tier fake-tier --yes-i-really-really-mean-it +} + +function test_mon_osd_erasure_code() +{ + + ceph osd erasure-code-profile set fooprofile a=b c=d + ceph osd erasure-code-profile set fooprofile a=b c=d + expect_false ceph osd erasure-code-profile set fooprofile a=b c=d e=f + ceph osd erasure-code-profile set fooprofile a=b c=d e=f --force + ceph osd erasure-code-profile set fooprofile a=b c=d e=f + expect_false ceph osd erasure-code-profile set fooprofile a=b c=d e=f g=h + # make sure rule-foo doesn't work anymore + expect_false ceph osd erasure-code-profile set barprofile ruleset-failure-domain=host + ceph osd erasure-code-profile set barprofile crush-failure-domain=host + # clean up + ceph osd erasure-code-profile rm fooprofile + ceph osd erasure-code-profile rm barprofile + + # try weird k and m values + expect_false ceph osd erasure-code-profile set badk k=1 m=1 + expect_false ceph osd erasure-code-profile set badk k=1 m=2 + expect_false ceph osd erasure-code-profile set badk k=0 m=2 + expect_false ceph osd erasure-code-profile set badk k=-1 m=2 + expect_false ceph osd erasure-code-profile set badm k=2 m=0 + expect_false ceph osd erasure-code-profile set badm k=2 m=-1 + ceph osd erasure-code-profile set good k=2 m=1 + ceph osd erasure-code-profile rm good +} + +function test_mon_osd_misc() +{ + set +e + + # expect error about missing 'pool' argument + ceph osd map 2>$TMPFILE; check_response 'pool' $? 22 + + # expect error about unused argument foo + ceph osd ls foo 2>$TMPFILE; check_response 'unused' $? 22 + + # expect "not in range" for invalid overload percentage + ceph osd reweight-by-utilization 80 2>$TMPFILE; check_response 'higher than 100' $? 22 + + set -e + + local old_bytes_per_osd=$(ceph config get mgr mon_reweight_min_bytes_per_osd) + local old_pgs_per_osd=$(ceph config get mgr mon_reweight_min_pgs_per_osd) + # otherwise ceph-mgr complains like: + # Error EDOM: Refusing to reweight: we only have 5372 kb used across all osds! + # Error EDOM: Refusing to reweight: we only have 20 PGs across 3 osds! + ceph config set mgr mon_reweight_min_bytes_per_osd 0 + ceph config set mgr mon_reweight_min_pgs_per_osd 0 + ceph osd reweight-by-utilization 110 + ceph osd reweight-by-utilization 110 .5 + expect_false ceph osd reweight-by-utilization 110 0 + expect_false ceph osd reweight-by-utilization 110 -0.1 + ceph osd test-reweight-by-utilization 110 .5 --no-increasing + ceph osd test-reweight-by-utilization 110 .5 4 --no-increasing + expect_false ceph osd test-reweight-by-utilization 110 .5 0 --no-increasing + expect_false ceph osd test-reweight-by-utilization 110 .5 -10 --no-increasing + ceph osd reweight-by-pg 110 + ceph osd test-reweight-by-pg 110 .5 + ceph osd reweight-by-pg 110 rbd + ceph osd reweight-by-pg 110 .5 rbd + expect_false ceph osd reweight-by-pg 110 boguspoolasdfasdfasdf + # restore the setting + ceph config set mgr mon_reweight_min_bytes_per_osd $old_bytes_per_osd + ceph config set mgr mon_reweight_min_pgs_per_osd $old_pgs_per_osd +} + +function test_admin_heap_profiler() +{ + do_test=1 + set +e + # expect 'heap' commands to be correctly parsed + ceph tell osd.0 heap stats 2>$TMPFILE + if [[ $? -eq 22 && `grep 'tcmalloc not enabled' $TMPFILE` ]]; then + echo "tcmalloc not enabled; skip heap profiler test" + do_test=0 + fi + set -e + + [[ $do_test -eq 0 ]] && return 0 + + $SUDO ceph tell osd.0 heap start_profiler + $SUDO ceph tell osd.0 heap dump + $SUDO ceph tell osd.0 heap stop_profiler + $SUDO ceph tell osd.0 heap release +} + +function test_osd_bench() +{ + # test osd bench limits + # As we should not rely on defaults (as they may change over time), + # lets inject some values and perform some simple tests + # max iops: 10 # 100 IOPS + # max throughput: 10485760 # 10MB/s + # max block size: 2097152 # 2MB + # duration: 10 # 10 seconds + + local args="\ + --osd-bench-duration 10 \ + --osd-bench-max-block-size 2097152 \ + --osd-bench-large-size-max-throughput 10485760 \ + --osd-bench-small-size-max-iops 10" + ceph tell osd.0 injectargs ${args## } + + # anything with a bs larger than 2097152 must fail + expect_false ceph tell osd.0 bench 1 2097153 + # but using 'osd_bench_max_bs' must succeed + ceph tell osd.0 bench 1 2097152 + + # we assume 1MB as a large bs; anything lower is a small bs + # for a 4096 bytes bs, for 10 seconds, we are limited by IOPS + # max count: 409600 (bytes) + + # more than max count must not be allowed + expect_false ceph tell osd.0 bench 409601 4096 + # but 409600 must be succeed + ceph tell osd.0 bench 409600 4096 + + # for a large bs, we are limited by throughput. + # for a 2MB block size for 10 seconds, assuming 10MB/s throughput, + # the max count will be (10MB * 10s) = 100MB + # max count: 104857600 (bytes) + + # more than max count must not be allowed + expect_false ceph tell osd.0 bench 104857601 2097152 + # up to max count must be allowed + ceph tell osd.0 bench 104857600 2097152 +} + +function test_osd_negative_filestore_merge_threshold() +{ + $SUDO ceph daemon osd.0 config set filestore_merge_threshold -1 + expect_config_value "osd.0" "filestore_merge_threshold" -1 +} + +function test_mon_tell() +{ + for m in mon.a mon.b; do + ceph tell $m sessions + ceph_watch_start debug audit + ceph tell mon.a sessions + ceph_watch_wait "${m} \[DBG\] from.*cmd='sessions' args=\[\]: dispatch" + done + expect_false ceph tell mon.foo version +} + +function test_mon_ping() +{ + ceph ping mon.a + ceph ping mon.b + expect_false ceph ping mon.foo + + ceph ping mon.\* +} + +function test_mon_deprecated_commands() +{ + # current DEPRECATED commands are marked with FLAG(DEPRECATED) + # + # Testing should be accomplished by setting + # 'mon_debug_deprecated_as_obsolete = true' and expecting ENOTSUP for + # each one of these commands. + + ceph tell mon.* injectargs '--mon-debug-deprecated-as-obsolete' + expect_false ceph config-key list 2> $TMPFILE + check_response "\(EOPNOTSUPP\|ENOTSUP\): command is obsolete" + + ceph tell mon.* injectargs '--no-mon-debug-deprecated-as-obsolete' +} + +function test_mon_cephdf_commands() +{ + # ceph df detail: + # pool section: + # RAW USED The near raw used per pool in raw total + + ceph osd pool create cephdf_for_test 1 1 replicated + ceph osd pool application enable cephdf_for_test rados + ceph osd pool set cephdf_for_test size 2 + + dd if=/dev/zero of=./cephdf_for_test bs=4k count=1 + rados put cephdf_for_test cephdf_for_test -p cephdf_for_test + + #wait for update + for i in `seq 1 10`; do + rados -p cephdf_for_test ls - | grep -q cephdf_for_test && break + sleep 1 + done + # "rados ls" goes straight to osd, but "ceph df" is served by mon. so we need + # to sync mon with osd + flush_pg_stats + local jq_filter='.pools | .[] | select(.name == "cephdf_for_test") | .stats' + stored=`ceph df detail --format=json | jq "$jq_filter.stored * 2"` + stored_raw=`ceph df detail --format=json | jq "$jq_filter.stored_raw"` + + ceph osd pool delete cephdf_for_test cephdf_for_test --yes-i-really-really-mean-it + rm ./cephdf_for_test + + expect_false test $stored != $stored_raw +} + +function test_mon_pool_application() +{ + ceph osd pool create app_for_test 16 + + ceph osd pool application enable app_for_test rbd + expect_false ceph osd pool application enable app_for_test rgw + ceph osd pool application enable app_for_test rgw --yes-i-really-mean-it + ceph osd pool ls detail | grep "application rbd,rgw" + ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{},"rgw":{}}' + + expect_false ceph osd pool application set app_for_test cephfs key value + ceph osd pool application set app_for_test rbd key1 value1 + ceph osd pool application set app_for_test rbd key2 value2 + ceph osd pool application set app_for_test rgw key1 value1 + ceph osd pool application get app_for_test rbd key1 | grep 'value1' + ceph osd pool application get app_for_test rbd key2 | grep 'value2' + ceph osd pool application get app_for_test rgw key1 | grep 'value1' + + ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{"key1":"value1","key2":"value2"},"rgw":{"key1":"value1"}}' + + ceph osd pool application rm app_for_test rgw key1 + ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{"key1":"value1","key2":"value2"},"rgw":{}}' + ceph osd pool application rm app_for_test rbd key2 + ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{"key1":"value1"},"rgw":{}}' + ceph osd pool application rm app_for_test rbd key1 + ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{},"rgw":{}}' + ceph osd pool application rm app_for_test rbd key1 # should be idempotent + + expect_false ceph osd pool application disable app_for_test rgw + ceph osd pool application disable app_for_test rgw --yes-i-really-mean-it + ceph osd pool application disable app_for_test rgw --yes-i-really-mean-it # should be idempotent + ceph osd pool ls detail | grep "application rbd" + ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{}}' + + ceph osd pool application disable app_for_test rgw --yes-i-really-mean-it + ceph osd pool ls detail | grep -v "application " + ceph osd pool ls detail --format=json | grep '"application_metadata":{}' + + ceph osd pool rm app_for_test app_for_test --yes-i-really-really-mean-it +} + +function test_mon_tell_help_command() +{ + ceph tell mon.a help | grep sync_force + ceph tell mon.a -h | grep sync_force + ceph tell mon.a config -h | grep 'config diff get' + + # wrong target + expect_false ceph tell mon.zzz help +} + +function test_mon_stdin_stdout() +{ + echo foo | ceph config-key set test_key -i - + ceph config-key get test_key -o - | grep -c foo | grep -q 1 +} + +function test_osd_tell_help_command() +{ + ceph tell osd.1 help + expect_false ceph tell osd.100 help +} + +function test_osd_compact() +{ + ceph tell osd.1 compact + $SUDO ceph daemon osd.1 compact +} + +function test_mds_tell_help_command() +{ + local FS_NAME=cephfs + if ! mds_exists ; then + echo "Skipping test, no MDS found" + return + fi + + remove_all_fs + ceph osd pool create fs_data 16 + ceph osd pool create fs_metadata 16 + ceph fs new $FS_NAME fs_metadata fs_data + wait_mds_active $FS_NAME + + + ceph tell mds.a help + expect_false ceph tell mds.z help + + remove_all_fs + ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it + ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it +} + +function test_mgr_tell() +{ + ceph tell mgr version +} + +function test_mgr_devices() +{ + ceph device ls + expect_false ceph device info doesnotexist + expect_false ceph device get-health-metrics doesnotexist +} + +function test_per_pool_scrub_status() +{ + ceph osd pool create noscrub_pool 16 + ceph osd pool create noscrub_pool2 16 + ceph -s | expect_false grep -q "Some pool(s) have the.*scrub.* flag(s) set" + ceph -s --format json | \ + jq .health.checks.POOL_SCRUB_FLAGS.summary.message | \ + expect_false grep -q "Some pool(s) have the.*scrub.* flag(s) set" + ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail | + expect_false grep -q "Pool .* has .*scrub.* flag" + ceph health detail | jq .health.checks.POOL_SCRUB_FLAGS.detail | \ + expect_false grep -q "Pool .* has .*scrub.* flag" + + ceph osd pool set noscrub_pool noscrub 1 + ceph -s | expect_true grep -q "Some pool(s) have the noscrub flag(s) set" + ceph -s --format json | \ + jq .health.checks.POOL_SCRUB_FLAGS.summary.message | \ + expect_true grep -q "Some pool(s) have the noscrub flag(s) set" + ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail | \ + expect_true grep -q "Pool noscrub_pool has noscrub flag" + ceph health detail | expect_true grep -q "Pool noscrub_pool has noscrub flag" + + ceph osd pool set noscrub_pool nodeep-scrub 1 + ceph osd pool set noscrub_pool2 nodeep-scrub 1 + ceph -s | expect_true grep -q "Some pool(s) have the noscrub, nodeep-scrub flag(s) set" + ceph -s --format json | \ + jq .health.checks.POOL_SCRUB_FLAGS.summary.message | \ + expect_true grep -q "Some pool(s) have the noscrub, nodeep-scrub flag(s) set" + ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail | \ + expect_true grep -q "Pool noscrub_pool has noscrub flag" + ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail | \ + expect_true grep -q "Pool noscrub_pool has nodeep-scrub flag" + ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail | \ + expect_true grep -q "Pool noscrub_pool2 has nodeep-scrub flag" + ceph health detail | expect_true grep -q "Pool noscrub_pool has noscrub flag" + ceph health detail | expect_true grep -q "Pool noscrub_pool has nodeep-scrub flag" + ceph health detail | expect_true grep -q "Pool noscrub_pool2 has nodeep-scrub flag" + + ceph osd pool rm noscrub_pool noscrub_pool --yes-i-really-really-mean-it + ceph osd pool rm noscrub_pool2 noscrub_pool2 --yes-i-really-really-mean-it +} + +# +# New tests should be added to the TESTS array below +# +# Individual tests may be run using the '-t <testname>' argument +# The user can specify '-t <testname>' as many times as she wants +# +# Tests will be run in order presented in the TESTS array, or in +# the order specified by the '-t <testname>' options. +# +# '-l' will list all the available test names +# '-h' will show usage +# +# The test maintains backward compatibility: not specifying arguments +# will run all tests following the order they appear in the TESTS array. +# + +set +x +MON_TESTS+=" mon_injectargs" +MON_TESTS+=" mon_injectargs_SI" +for i in `seq 9`; do + MON_TESTS+=" tiering_$i"; +done +MON_TESTS+=" auth" +MON_TESTS+=" auth_profiles" +MON_TESTS+=" mon_misc" +MON_TESTS+=" mon_mon" +MON_TESTS+=" mon_osd" +MON_TESTS+=" mon_config_key" +MON_TESTS+=" mon_crush" +MON_TESTS+=" mon_osd_create_destroy" +MON_TESTS+=" mon_osd_pool" +MON_TESTS+=" mon_osd_pool_quota" +MON_TESTS+=" mon_pg" +MON_TESTS+=" mon_osd_pool_set" +MON_TESTS+=" mon_osd_tiered_pool_set" +MON_TESTS+=" mon_osd_erasure_code" +MON_TESTS+=" mon_osd_misc" +MON_TESTS+=" mon_tell" +MON_TESTS+=" mon_ping" +MON_TESTS+=" mon_deprecated_commands" +MON_TESTS+=" mon_caps" +MON_TESTS+=" mon_cephdf_commands" +MON_TESTS+=" mon_tell_help_command" +MON_TESTS+=" mon_stdin_stdout" + +OSD_TESTS+=" osd_bench" +OSD_TESTS+=" osd_negative_filestore_merge_threshold" +OSD_TESTS+=" tiering_agent" +OSD_TESTS+=" admin_heap_profiler" +OSD_TESTS+=" osd_tell_help_command" +OSD_TESTS+=" osd_compact" +OSD_TESTS+=" per_pool_scrub_status" + +MDS_TESTS+=" mds_tell" +MDS_TESTS+=" mon_mds" +MDS_TESTS+=" mon_mds_metadata" +MDS_TESTS+=" mds_tell_help_command" + +MGR_TESTS+=" mgr_tell" +MGR_TESTS+=" mgr_devices" + +TESTS+=$MON_TESTS +TESTS+=$OSD_TESTS +TESTS+=$MDS_TESTS +TESTS+=$MGR_TESTS + +# +# "main" follows +# + +function list_tests() +{ + echo "AVAILABLE TESTS" + for i in $TESTS; do + echo " $i" + done +} + +function usage() +{ + echo "usage: $0 [-h|-l|-t <testname> [-t <testname>...]]" +} + +tests_to_run=() + +sanity_check=true + +while [[ $# -gt 0 ]]; do + opt=$1 + + case "$opt" in + "-l" ) + do_list=1 + ;; + "--asok-does-not-need-root" ) + SUDO="" + ;; + "--no-sanity-check" ) + sanity_check=false + ;; + "--test-mon" ) + tests_to_run+="$MON_TESTS" + ;; + "--test-osd" ) + tests_to_run+="$OSD_TESTS" + ;; + "--test-mds" ) + tests_to_run+="$MDS_TESTS" + ;; + "--test-mgr" ) + tests_to_run+="$MGR_TESTS" + ;; + "-t" ) + shift + if [[ -z "$1" ]]; then + echo "missing argument to '-t'" + usage ; + exit 1 + fi + tests_to_run+=" $1" + ;; + "-h" ) + usage ; + exit 0 + ;; + esac + shift +done + +if [[ $do_list -eq 1 ]]; then + list_tests ; + exit 0 +fi + +ceph osd pool create rbd 16 + +if test -z "$tests_to_run" ; then + tests_to_run="$TESTS" +fi + +if $sanity_check ; then + wait_no_osd_down +fi +for i in $tests_to_run; do + if $sanity_check ; then + check_no_osd_down + fi + set -x + test_${i} + set +x +done +if $sanity_check ; then + check_no_osd_down +fi + +set -x + +echo OK diff --git a/qa/workunits/cephtool/test_daemon.sh b/qa/workunits/cephtool/test_daemon.sh new file mode 100755 index 000000000..08ae937cc --- /dev/null +++ b/qa/workunits/cephtool/test_daemon.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -ex + +expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +echo note: assuming mon.a is on the current host + +# can set to 'sudo ./ceph' to execute tests from current dir for development +CEPH=${CEPH:-'sudo ceph'} + +${CEPH} daemon mon.a version | grep version + +# get debug_ms setting and strip it, painfully for reuse +old_ms=$(${CEPH} daemon mon.a config get debug_ms | \ + grep debug_ms | sed -e 's/.*: //' -e 's/["\}\\]//g') +${CEPH} daemon mon.a config set debug_ms 13 +new_ms=$(${CEPH} daemon mon.a config get debug_ms | \ + grep debug_ms | sed -e 's/.*: //' -e 's/["\}\\]//g') +[ "$new_ms" = "13/13" ] +${CEPH} daemon mon.a config set debug_ms $old_ms +new_ms=$(${CEPH} daemon mon.a config get debug_ms | \ + grep debug_ms | sed -e 's/.*: //' -e 's/["\}\\]//g') +[ "$new_ms" = "$old_ms" ] + +# unregistered/non-existent command +expect_false ${CEPH} daemon mon.a bogus_command_blah foo + +set +e +OUTPUT=$(${CEPH} -c /not/a/ceph.conf daemon mon.a help 2>&1) +# look for EINVAL +if [ $? != 22 ] ; then exit 1; fi +if ! echo "$OUTPUT" | grep -q '.*open.*/not/a/ceph.conf'; then + echo "didn't find expected error in bad conf search" + exit 1 +fi +set -e + +echo OK diff --git a/qa/workunits/cephtool/test_kvstore_tool.sh b/qa/workunits/cephtool/test_kvstore_tool.sh new file mode 100755 index 000000000..b7953dd21 --- /dev/null +++ b/qa/workunits/cephtool/test_kvstore_tool.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +set -x + +source $(dirname $0)/../../standalone/ceph-helpers.sh + +set -e +set -o functrace +PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' +SUDO=${SUDO:-sudo} +export CEPH_DEV=1 + +echo note: test ceph_kvstore_tool with bluestore + +expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +TEMP_DIR=$(mktemp -d ./cephtool.XXX) +trap "rm -fr $TEMP_DIR" 0 + +TEMP_FILE=$(mktemp $TEMP_DIR/test_invalid.XXX) + +function test_ceph_kvstore_tool() +{ + # create a data directory + ceph-objectstore-tool --data-path ${TEMP_DIR} --op mkfs --no-mon-config + + # list + origin_kv_nums=`ceph-kvstore-tool bluestore-kv ${TEMP_DIR} list 2>/dev/null | wc -l` + + # exists + prefix=`ceph-kvstore-tool bluestore-kv ${TEMP_DIR} list 2>/dev/null | head -n 1 | awk '{print $1}'` + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} exists ${prefix} + expect_false ceph-kvstore-tool bluestore-kv ${TEMP_DIR} exists ${prefix}notexist + + # list-crc + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} list-crc + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} list-crc ${prefix} + + # list with prefix + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} list ${prefix} + + # set + echo "helloworld" >> ${TEMP_FILE} + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} set TESTPREFIX TESTKEY in ${TEMP_FILE} + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} exists TESTPREFIX TESTKEY + + # get + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} get TESTPREFIX TESTKEY out ${TEMP_FILE}.bak + diff ${TEMP_FILE} ${TEMP_FILE}.bak + + # rm + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} rm TESTPREFIX TESTKEY + expect_false ceph-kvstore-tool bluestore-kv ${TEMP_DIR} exists TESTPREFIX TESTKEY + + # compact + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} compact + + # destructive-repair + ceph-kvstore-tool bluestore-kv ${TEMP_DIR} destructive-repair + + current_kv_nums=`ceph-kvstore-tool bluestore-kv ${TEMP_DIR} list 2>/dev/null | wc -l` + test ${origin_kv_nums} -eq ${current_kv_nums} +} + +test_ceph_kvstore_tool + +echo OK diff --git a/qa/workunits/client/test.sh b/qa/workunits/client/test.sh new file mode 100755 index 000000000..12abd3a5d --- /dev/null +++ b/qa/workunits/client/test.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -ex + +ceph_test_client diff --git a/qa/workunits/cls/test_cls_2pc_queue.sh b/qa/workunits/cls/test_cls_2pc_queue.sh new file mode 100755 index 000000000..b4f68800f --- /dev/null +++ b/qa/workunits/cls/test_cls_2pc_queue.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_2pc_queue + +exit 0 diff --git a/qa/workunits/cls/test_cls_cas.sh b/qa/workunits/cls/test_cls_cas.sh new file mode 100755 index 000000000..765913482 --- /dev/null +++ b/qa/workunits/cls/test_cls_cas.sh @@ -0,0 +1,6 @@ +#!/bin/sh -e + +GTEST_FILTER=${CLS_CAS_GTEST_FILTER:-*} +ceph_test_cls_cas --gtest_filter=${GTEST_FILTER} + +exit 0 diff --git a/qa/workunits/cls/test_cls_cmpomap.sh b/qa/workunits/cls/test_cls_cmpomap.sh new file mode 100755 index 000000000..af079f6e6 --- /dev/null +++ b/qa/workunits/cls/test_cls_cmpomap.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_cmpomap + +exit 0 diff --git a/qa/workunits/cls/test_cls_hello.sh b/qa/workunits/cls/test_cls_hello.sh new file mode 100755 index 000000000..0a2e09620 --- /dev/null +++ b/qa/workunits/cls/test_cls_hello.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_hello + +exit 0 diff --git a/qa/workunits/cls/test_cls_journal.sh b/qa/workunits/cls/test_cls_journal.sh new file mode 100755 index 000000000..9aa7450a9 --- /dev/null +++ b/qa/workunits/cls/test_cls_journal.sh @@ -0,0 +1,6 @@ +#!/bin/sh -e + +GTEST_FILTER=${CLS_JOURNAL_GTEST_FILTER:-*} +ceph_test_cls_journal --gtest_filter=${GTEST_FILTER} + +exit 0 diff --git a/qa/workunits/cls/test_cls_lock.sh b/qa/workunits/cls/test_cls_lock.sh new file mode 100755 index 000000000..c14527053 --- /dev/null +++ b/qa/workunits/cls/test_cls_lock.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_lock + +exit 0 diff --git a/qa/workunits/cls/test_cls_log.sh b/qa/workunits/cls/test_cls_log.sh new file mode 100755 index 000000000..523f985e7 --- /dev/null +++ b/qa/workunits/cls/test_cls_log.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_log + +exit 0 diff --git a/qa/workunits/cls/test_cls_numops.sh b/qa/workunits/cls/test_cls_numops.sh new file mode 100755 index 000000000..dcbafcab2 --- /dev/null +++ b/qa/workunits/cls/test_cls_numops.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_numops + +exit 0 diff --git a/qa/workunits/cls/test_cls_rbd.sh b/qa/workunits/cls/test_cls_rbd.sh new file mode 100755 index 000000000..fd4bec0f8 --- /dev/null +++ b/qa/workunits/cls/test_cls_rbd.sh @@ -0,0 +1,6 @@ +#!/bin/sh -e + +GTEST_FILTER=${CLS_RBD_GTEST_FILTER:-*} +ceph_test_cls_rbd --gtest_filter=${GTEST_FILTER} + +exit 0 diff --git a/qa/workunits/cls/test_cls_refcount.sh b/qa/workunits/cls/test_cls_refcount.sh new file mode 100755 index 000000000..d722f5ad9 --- /dev/null +++ b/qa/workunits/cls/test_cls_refcount.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_refcount + +exit 0 diff --git a/qa/workunits/cls/test_cls_rgw.sh b/qa/workunits/cls/test_cls_rgw.sh new file mode 100755 index 000000000..257338a05 --- /dev/null +++ b/qa/workunits/cls/test_cls_rgw.sh @@ -0,0 +1,8 @@ +#!/bin/sh -e + +ceph_test_cls_rgw +#ceph_test_cls_rgw_meta +#ceph_test_cls_rgw_log +#ceph_test_cls_rgw_opstate + +exit 0 diff --git a/qa/workunits/cls/test_cls_rgw_gc.sh b/qa/workunits/cls/test_cls_rgw_gc.sh new file mode 100755 index 000000000..0266438f8 --- /dev/null +++ b/qa/workunits/cls/test_cls_rgw_gc.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_rgw_gc + +exit 0 diff --git a/qa/workunits/cls/test_cls_rgw_stats.sh b/qa/workunits/cls/test_cls_rgw_stats.sh new file mode 100755 index 000000000..e1b5bd6b9 --- /dev/null +++ b/qa/workunits/cls/test_cls_rgw_stats.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_rgw_stats + +exit 0 diff --git a/qa/workunits/cls/test_cls_sdk.sh b/qa/workunits/cls/test_cls_sdk.sh new file mode 100755 index 000000000..f1ccdc3b4 --- /dev/null +++ b/qa/workunits/cls/test_cls_sdk.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_cls_sdk + +exit 0 diff --git a/qa/workunits/direct_io/.gitignore b/qa/workunits/direct_io/.gitignore new file mode 100644 index 000000000..80f1fd1aa --- /dev/null +++ b/qa/workunits/direct_io/.gitignore @@ -0,0 +1,3 @@ +/direct_io_test +/test_sync_io +/test_short_dio_read diff --git a/qa/workunits/direct_io/Makefile b/qa/workunits/direct_io/Makefile new file mode 100644 index 000000000..20fec0be5 --- /dev/null +++ b/qa/workunits/direct_io/Makefile @@ -0,0 +1,11 @@ +CFLAGS = -Wall -Wextra -D_GNU_SOURCE + +TARGETS = direct_io_test test_sync_io test_short_dio_read + +.c: + $(CC) $(CFLAGS) $@.c -o $@ + +all: $(TARGETS) + +clean: + rm $(TARGETS) diff --git a/qa/workunits/direct_io/big.sh b/qa/workunits/direct_io/big.sh new file mode 100755 index 000000000..43bd6d72b --- /dev/null +++ b/qa/workunits/direct_io/big.sh @@ -0,0 +1,6 @@ +#!/bin/sh -ex + +echo "test large (16MB) dio write" +dd if=/dev/zero of=foo.big bs=16M count=1 oflag=direct + +echo OK diff --git a/qa/workunits/direct_io/direct_io_test.c b/qa/workunits/direct_io/direct_io_test.c new file mode 100644 index 000000000..ccfbbb860 --- /dev/null +++ b/qa/workunits/direct_io/direct_io_test.c @@ -0,0 +1,312 @@ +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2011 New Dream Network + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include <errno.h> +#include <inttypes.h> +#include <fcntl.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <time.h> +#include <unistd.h> + +/* + * direct_io_test + * + * This test does some I/O using O_DIRECT. + * + * Semantics of O_DIRECT can be found at http://lwn.net/Articles/348739/ + * + */ + +static int g_num_pages = 100; + +static int g_duration = 10; + +struct chunk { + uint64_t offset; + uint64_t pad0; + uint64_t pad1; + uint64_t pad2; + uint64_t pad3; + uint64_t pad4; + uint64_t pad5; + uint64_t not_offset; +} __attribute__((packed)); + +static int page_size; + +static char temp_file[] = "direct_io_temp_file_XXXXXX"; + +static int safe_write(int fd, const void *buf, signed int len) +{ + const char *b = (const char*)buf; + /* Handle EINTR and short writes */ + while (1) { + int res = write(fd, b, len); + if (res < 0) { + int err = errno; + if (err != EINTR) { + return err; + } + } + len -= res; + b += res; + if (len <= 0) + return 0; + } +} + +static int do_read(int fd, char *buf, int buf_sz) +{ + /* We assume no short reads or EINTR. It's not really clear how + * those things interact with O_DIRECT. */ + int ret = read(fd, buf, buf_sz); + if (ret < 0) { + int err = errno; + printf("do_read: error: %d (%s)\n", err, strerror(err)); + return err; + } + if (ret != buf_sz) { + printf("do_read: short read\n"); + return -EIO; + } + return 0; +} + +static int setup_temp_file(void) +{ + int fd; + int64_t num_chunks, i; + + if (page_size % sizeof(struct chunk)) { + printf("setup_big_file: page_size doesn't divide evenly " + "into data blocks.\n"); + return -EINVAL; + } + + fd = mkstemp(temp_file); + if (fd < 0) { + int err = errno; + printf("setup_big_file: mkostemps failed with error %d\n", err); + return err; + } + + num_chunks = g_num_pages * (page_size / sizeof(struct chunk)); + for (i = 0; i < num_chunks; ++i) { + int ret; + struct chunk c; + memset(&c, 0, sizeof(c)); + c.offset = i * sizeof(struct chunk); + c.pad0 = 0; + c.pad1 = 1; + c.pad2 = 2; + c.pad3 = 3; + c.pad4 = 4; + c.pad5 = 5; + c.not_offset = ~c.offset; + ret = safe_write(fd, &c, sizeof(struct chunk)); + if (ret) { + printf("setup_big_file: safe_write failed with " + "error: %d\n", ret); + TEMP_FAILURE_RETRY(close(fd)); + unlink(temp_file); + return ret; + } + } + TEMP_FAILURE_RETRY(close(fd)); + return 0; +} + +static int verify_chunk(const struct chunk *c, uint64_t offset) +{ + if (c->offset != offset) { + printf("verify_chunk(%" PRId64 "): bad offset value (got: %" + PRId64 ", expected: %" PRId64 "\n", offset, c->offset, offset); + return EIO; + } + if (c->pad0 != 0) { + printf("verify_chunk(%" PRId64 "): bad pad0 value\n", offset); + return EIO; + } + if (c->pad1 != 1) { + printf("verify_chunk(%" PRId64 "): bad pad1 value\n", offset); + return EIO; + } + if (c->pad2 != 2) { + printf("verify_chunk(%" PRId64 "): bad pad2 value\n", offset); + return EIO; + } + if (c->pad3 != 3) { + printf("verify_chunk(%" PRId64 "): bad pad3 value\n", offset); + return EIO; + } + if (c->pad4 != 4) { + printf("verify_chunk(%" PRId64 "): bad pad4 value\n", offset); + return EIO; + } + if (c->pad5 != 5) { + printf("verify_chunk(%" PRId64 "): bad pad5 value\n", offset); + return EIO; + } + if (c->not_offset != ~offset) { + printf("verify_chunk(%" PRId64 "): bad not_offset value\n", + offset); + return EIO; + } + return 0; +} + +static int do_o_direct_reads(void) +{ + int fd, ret; + unsigned int i; + void *buf = 0; + time_t cur_time, end_time; + ret = posix_memalign(&buf, page_size, page_size); + if (ret) { + printf("do_o_direct_reads: posix_memalign returned %d\n", ret); + goto done; + } + + fd = open(temp_file, O_RDONLY | O_DIRECT); + if (fd < 0) { + ret = errno; + printf("do_o_direct_reads: error opening fd: %d\n", ret); + goto free_buf; + } + + // read the first chunk and see if it looks OK + ret = do_read(fd, buf, page_size); + if (ret) + goto close_fd; + ret = verify_chunk((struct chunk*)buf, 0); + if (ret) + goto close_fd; + + // read some random chunks and see how they look + cur_time = time(NULL); + end_time = cur_time + g_duration; + i = 0; + do { + time_t next_time; + uint64_t offset; + int page; + unsigned int seed; + + seed = i++; + page = rand_r(&seed) % g_num_pages; + offset = page; + offset *= page_size; + if (lseek64(fd, offset, SEEK_SET) == -1) { + int err = errno; + printf("lseek64(%" PRId64 ") failed: error %d (%s)\n", + offset, err, strerror(err)); + goto close_fd; + } + ret = do_read(fd, buf, page_size); + if (ret) + goto close_fd; + ret = verify_chunk((struct chunk*)buf, offset); + if (ret) + goto close_fd; + next_time = time(NULL); + if (next_time > cur_time) { + printf("."); + } + cur_time = next_time; + } while (time(NULL) < end_time); + + printf("\ndo_o_direct_reads: SUCCESS\n"); +close_fd: + TEMP_FAILURE_RETRY(close(fd)); +free_buf: + free(buf); +done: + return ret; +} + +static void usage(char *argv0) +{ + printf("%s: tests direct I/O\n", argv0); + printf("-d <seconds>: sets duration to <seconds>\n"); + printf("-h: this help\n"); + printf("-p <pages>: sets number of pages to allocate\n"); +} + +static void parse_args(int argc, char *argv[]) +{ + int c; + while ((c = getopt (argc, argv, "d:hp:")) != -1) { + switch (c) { + case 'd': + g_duration = atoi(optarg); + if (g_duration <= 0) { + printf("tried to set invalid value of " + "g_duration: %d\n", g_num_pages); + exit(1); + } + break; + case 'h': + usage(argv[0]); + exit(0); + break; + case 'p': + g_num_pages = atoi(optarg); + if (g_num_pages <= 0) { + printf("tried to set invalid value of " + "g_num_pages: %d\n", g_num_pages); + exit(1); + } + break; + case '?': + usage(argv[0]); + exit(1); + break; + default: + usage(argv[0]); + exit(1); + break; + } + } +} + +int main(int argc, char *argv[]) +{ + int ret; + + parse_args(argc, argv); + + setvbuf(stdout, NULL, _IONBF, 0); + + page_size = getpagesize(); + + ret = setup_temp_file(); + if (ret) { + printf("setup_temp_file failed with error %d\n", ret); + goto done; + } + + ret = do_o_direct_reads(); + if (ret) { + printf("do_o_direct_reads failed with error %d\n", ret); + goto unlink_temp_file; + } + +unlink_temp_file: + unlink(temp_file); +done: + return ret; +} diff --git a/qa/workunits/direct_io/misc.sh b/qa/workunits/direct_io/misc.sh new file mode 100755 index 000000000..6de080d2d --- /dev/null +++ b/qa/workunits/direct_io/misc.sh @@ -0,0 +1,16 @@ +#!/bin/sh -ex + +# a few test cases from henry +echo "test read from hole" +dd if=/dev/zero of=dd3 bs=1 seek=1048576 count=0 +dd if=dd3 of=/tmp/ddout1 skip=8 bs=512 count=2 iflag=direct +dd if=/dev/zero of=/tmp/dd3 bs=512 count=2 +cmp /tmp/dd3 /tmp/ddout1 + +echo "other thing" +dd if=/dev/urandom of=/tmp/dd10 bs=500 count=1 +dd if=/tmp/dd10 of=dd10 bs=512 seek=8388 count=1 +dd if=dd10 of=/tmp/dd10out bs=512 skip=8388 count=1 iflag=direct +cmp /tmp/dd10 /tmp/dd10out + +echo OK diff --git a/qa/workunits/direct_io/test_short_dio_read.c b/qa/workunits/direct_io/test_short_dio_read.c new file mode 100644 index 000000000..502485557 --- /dev/null +++ b/qa/workunits/direct_io/test_short_dio_read.c @@ -0,0 +1,57 @@ +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <stdlib.h> + +int main() +{ + char buf[409600]; + ssize_t r; + int err; + int fd = open("shortfile", O_WRONLY|O_CREAT, 0644); + + if (fd < 0) { + err = errno; + printf("error: open() failed with: %d (%s)\n", err, strerror(err)); + exit(err); + } + + printf("writing first 3 bytes of 10k file\n"); + r = write(fd, "foo", 3); + if (r == -1) { + err = errno; + printf("error: write() failed with: %d (%s)\n", err, strerror(err)); + close(fd); + exit(err); + } + r = ftruncate(fd, 10000); + if (r == -1) { + err = errno; + printf("error: ftruncate() failed with: %d (%s)\n", err, strerror(err)); + close(fd); + exit(err); + } + + fsync(fd); + close(fd); + + printf("reading O_DIRECT\n"); + fd = open("shortfile", O_RDONLY|O_DIRECT); + if (fd < 0) { + err = errno; + printf("error: open() failed with: %d (%s)\n", err, strerror(err)); + exit(err); + } + + r = read(fd, buf, sizeof(buf)); + close(fd); + + printf("got %d\n", (int)r); + if (r != 10000) + return 1; + return 0; +} diff --git a/qa/workunits/direct_io/test_sync_io.c b/qa/workunits/direct_io/test_sync_io.c new file mode 100644 index 000000000..f393fa6e8 --- /dev/null +++ b/qa/workunits/direct_io/test_sync_io.c @@ -0,0 +1,250 @@ +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <stdio.h> +#include <inttypes.h> +#include <linux/types.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <errno.h> + +//#include "../client/ioctl.h" + +#include <linux/ioctl.h> +#define CEPH_IOCTL_MAGIC 0x97 +#define CEPH_IOC_SYNCIO _IO(CEPH_IOCTL_MAGIC, 5) + +void write_pattern() +{ + printf("writing pattern\n"); + + uint64_t i; + int r; + + int fd = open("foo", O_CREAT|O_WRONLY, 0644); + if (fd < 0) { + r = errno; + printf("write_pattern: error: open() failed with: %d (%s)\n", r, strerror(r)); + exit(r); + } + for (i=0; i<1048576 * sizeof(i); i += sizeof(i)) { + r = write(fd, &i, sizeof(i)); + if (r == -1) { + r = errno; + printf("write_pattern: error: write() failed with: %d (%s)\n", r, strerror(r)); + break; + } + } + + close(fd); +} + +int verify_pattern(char *buf, size_t len, uint64_t off) +{ + size_t i; + + for (i = 0; i < len; i += sizeof(uint64_t)) { + uint64_t expected = i + off; + uint64_t actual = *(uint64_t*)(buf + i); + if (expected != actual) { + printf("error: offset %llu had %llu\n", (unsigned long long)expected, + (unsigned long long)actual); + exit(1); + } + } + return 0; +} + +void generate_pattern(void *buf, size_t len, uint64_t offset) +{ + uint64_t *v = buf; + size_t i; + + for (i=0; i<len / sizeof(v); i++) + v[i] = i * sizeof(v) + offset; + verify_pattern(buf, len, offset); +} + +int read_file(int buf_align, uint64_t offset, int len, int direct) { + + printf("read_file buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + void *rawbuf; + int r; + int flags; + int err = 0; + + if(direct) + flags = O_RDONLY|O_DIRECT; + else + flags = O_RDONLY; + + int fd = open("foo", flags); + if (fd < 0) { + err = errno; + printf("read_file: error: open() failed with: %d (%s)\n", err, strerror(err)); + exit(err); + } + + if (!direct) + ioctl(fd, CEPH_IOC_SYNCIO); + + if ((r = posix_memalign(&rawbuf, 4096, len + buf_align)) != 0) { + printf("read_file: error: posix_memalign failed with %d", r); + close(fd); + exit (r); + } + + void *buf = (char *)rawbuf + buf_align; + memset(buf, 0, len); + r = pread(fd, buf, len, offset); + if (r == -1) { + err = errno; + printf("read_file: error: pread() failed with: %d (%s)\n", err, strerror(err)); + goto out; + } + r = verify_pattern(buf, len, offset); + +out: + close(fd); + free(rawbuf); + return r; +} + +int read_direct(int buf_align, uint64_t offset, int len) +{ + printf("read_direct buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + return read_file(buf_align, offset, len, 1); +} + +int read_sync(int buf_align, uint64_t offset, int len) +{ + printf("read_sync buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + return read_file(buf_align, offset, len, 0); +} + +int write_file(int buf_align, uint64_t offset, int len, int direct) +{ + printf("write_file buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + void *rawbuf; + int r; + int err = 0; + int flags; + if (direct) + flags = O_WRONLY|O_DIRECT|O_CREAT; + else + flags = O_WRONLY|O_CREAT; + + int fd = open("foo", flags, 0644); + if (fd < 0) { + int err = errno; + printf("write_file: error: open() failed with: %d (%s)\n", err, strerror(err)); + exit(err); + } + + if ((r = posix_memalign(&rawbuf, 4096, len + buf_align)) != 0) { + printf("write_file: error: posix_memalign failed with %d", r); + err = r; + goto out_close; + } + + if (!direct) + ioctl(fd, CEPH_IOC_SYNCIO); + + void *buf = (char *)rawbuf + buf_align; + + generate_pattern(buf, len, offset); + + r = pwrite(fd, buf, len, offset); + close(fd); + + fd = open("foo", O_RDONLY); + if (fd < 0) { + err = errno; + printf("write_file: error: open() failed with: %d (%s)\n", err, strerror(err)); + free(rawbuf); + goto out_unlink; + } + void *buf2 = malloc(len); + if (!buf2) { + err = -ENOMEM; + printf("write_file: error: malloc failed\n"); + goto out_free; + } + + memset(buf2, 0, len); + r = pread(fd, buf2, len, offset); + if (r == -1) { + err = errno; + printf("write_file: error: pread() failed with: %d (%s)\n", err, strerror(err)); + goto out_free_buf; + } + r = verify_pattern(buf2, len, offset); + +out_free_buf: + free(buf2); +out_free: + free(rawbuf); +out_close: + close(fd); +out_unlink: + unlink("foo"); + if (err) + exit(err); + return r; +} + +int write_direct(int buf_align, uint64_t offset, int len) +{ + printf("write_direct buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + return write_file (buf_align, offset, len, 1); +} + +int write_sync(int buf_align, uint64_t offset, int len) +{ + printf("write_sync buf_align %d offset %llu len %d\n", buf_align, + (unsigned long long)offset, len); + return write_file (buf_align, offset, len, 0); +} + +int main(int argc, char **argv) +{ + uint64_t i, j, k; + int read = 1; + int write = 1; + + if (argc >= 2 && strcmp(argv[1], "read") == 0) + write = 0; + if (argc >= 2 && strcmp(argv[1], "write") == 0) + read = 0; + + if (read) { + write_pattern(); + + for (i = 0; i < 4096; i += 512) + for (j = 4*1024*1024 - 4096; j < 4*1024*1024 + 4096; j += 512) + for (k = 1024; k <= 16384; k *= 2) { + read_direct(i, j, k); + read_sync(i, j, k); + } + + } + unlink("foo"); + if (write) { + for (i = 0; i < 4096; i += 512) + for (j = 4*1024*1024 - 4096 + 512; j < 4*1024*1024 + 4096; j += 512) + for (k = 1024; k <= 16384; k *= 2) { + write_direct(i, j, k); + write_sync(i, j, k); + } + } + + + return 0; +} diff --git a/qa/workunits/erasure-code/.gitignore b/qa/workunits/erasure-code/.gitignore new file mode 100644 index 000000000..7e563b8b3 --- /dev/null +++ b/qa/workunits/erasure-code/.gitignore @@ -0,0 +1,2 @@ +*.log +*.trs diff --git a/qa/workunits/erasure-code/bench.html b/qa/workunits/erasure-code/bench.html new file mode 100644 index 000000000..3b4b6c74c --- /dev/null +++ b/qa/workunits/erasure-code/bench.html @@ -0,0 +1,34 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd" > +<html> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <title>Erasure Code Plugins Benchmarks</title> + <link href="examples.css" rel="stylesheet" type="text/css"> + <script language="javascript" type="text/javascript" src="jquery.js"></script> + <script language="javascript" type="text/javascript" src="jquery.flot.js"></script> + <script language="javascript" type="text/javascript" src="jquery.flot.categories.js"></script> + <script language="javascript" type="text/javascript" src="bench.js"></script> + <script language="javascript" type="text/javascript" src="plot.js"></script> + </head> + <body> + + <div id="header"> + <h2>Erasure Code Plugins Benchmarks</h2> + </div> + + <div id="content"> + + <div class="demo-container"> + <div id="encode" class="demo-placeholder"></div> + </div> + <p>encode: Y = GB/s, X = K/M</p> + + <div class="demo-container"> + <div id="decode" class="demo-placeholder"></div> + </div> + <p>decode: Y = GB/s, X = K/M/erasures</p> + + </div> + + </body> +</html> diff --git a/qa/workunits/erasure-code/bench.sh b/qa/workunits/erasure-code/bench.sh new file mode 100755 index 000000000..8e288f053 --- /dev/null +++ b/qa/workunits/erasure-code/bench.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2015 Red Hat <contact@redhat.com> +# Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com> +# +# Author: Loic Dachary <loic@dachary.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +# Test that it works from sources with: +# +# CEPH_ERASURE_CODE_BENCHMARK=src/ceph_erasure_code_benchmark \ +# PLUGIN_DIRECTORY=build/lib \ +# qa/workunits/erasure-code/bench.sh fplot jerasure | +# tee qa/workunits/erasure-code/bench.js +# +# This should start immediately and display: +# +# ... +# [ '2/1', .48035538612887358583 ], +# [ '3/2', .21648470405675016626 ], +# etc. +# +# and complete within a few seconds. The result can then be displayed with: +# +# firefox qa/workunits/erasure-code/bench.html +# +# Once it is confirmed to work, it can be run with a more significant +# volume of data so that the measures are more reliable: +# +# TOTAL_SIZE=$((4 * 1024 * 1024 * 1024)) \ +# CEPH_ERASURE_CODE_BENCHMARK=src/ceph_erasure_code_benchmark \ +# PLUGIN_DIRECTORY=build/lib \ +# qa/workunits/erasure-code/bench.sh fplot jerasure | +# tee qa/workunits/erasure-code/bench.js +# +set -e + +export PATH=/sbin:$PATH + +: ${VERBOSE:=false} +: ${CEPH_ERASURE_CODE_BENCHMARK:=ceph_erasure_code_benchmark} +: ${PLUGIN_DIRECTORY:=/usr/lib/ceph/erasure-code} +: ${PLUGINS:=isa jerasure} +: ${TECHNIQUES:=vandermonde cauchy} +: ${TOTAL_SIZE:=$((1024 * 1024))} +: ${SIZE:=4096} +: ${PARAMETERS:=--parameter jerasure-per-chunk-alignment=true} + +function bench_header() { + echo -e "seconds\tKB\tplugin\tk\tm\twork.\titer.\tsize\teras.\tcommand." +} + +function bench() { + local plugin=$1 + shift + local k=$1 + shift + local m=$1 + shift + local workload=$1 + shift + local iterations=$1 + shift + local size=$1 + shift + local erasures=$1 + shift + command=$(echo $CEPH_ERASURE_CODE_BENCHMARK \ + --plugin $plugin \ + --workload $workload \ + --iterations $iterations \ + --size $size \ + --erasures $erasures \ + --parameter k=$k \ + --parameter m=$m \ + --erasure-code-dir $PLUGIN_DIRECTORY) + result=$($command "$@") + echo -e "$result\t$plugin\t$k\t$m\t$workload\t$iterations\t$size\t$erasures\t$command ""$@" +} + +function packetsize() { + local k=$1 + local w=$2 + local vector_wordsize=$3 + local size=$4 + + local p=$(( ($size / $k / $w / $vector_wordsize ) * $vector_wordsize)) + if [ $p -gt 3100 ] ; then + p=3100 + fi + echo $p +} + +function bench_run() { + local plugin=jerasure + local w=8 + local VECTOR_WORDSIZE=16 + local ks="2 3 4 6 10" + declare -A k2ms + k2ms[2]="1" + k2ms[3]="2" + k2ms[4]="2 3" + k2ms[6]="2 3 4" + k2ms[10]="3 4" + local isa2technique_vandermonde='reed_sol_van' + local isa2technique_cauchy='cauchy' + local jerasure2technique_vandermonde='reed_sol_van' + local jerasure2technique_cauchy='cauchy_good' + for technique in ${TECHNIQUES} ; do + for plugin in ${PLUGINS} ; do + eval technique_parameter=\$${plugin}2technique_${technique} + echo "serie encode_${technique}_${plugin}" + for k in $ks ; do + for m in ${k2ms[$k]} ; do + bench $plugin $k $m encode $(($TOTAL_SIZE / $SIZE)) $SIZE 0 \ + --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE $SIZE) \ + ${PARAMETERS} \ + --parameter technique=$technique_parameter + + done + done + done + done + for technique in ${TECHNIQUES} ; do + for plugin in ${PLUGINS} ; do + eval technique_parameter=\$${plugin}2technique_${technique} + echo "serie decode_${technique}_${plugin}" + for k in $ks ; do + for m in ${k2ms[$k]} ; do + echo + for erasures in $(seq 1 $m) ; do + bench $plugin $k $m decode $(($TOTAL_SIZE / $SIZE)) $SIZE $erasures \ + --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE $SIZE) \ + ${PARAMETERS} \ + --parameter technique=$technique_parameter + done + done + done + done + done +} + +function fplot() { + local serie + bench_run | while read seconds total plugin k m workload iteration size erasures rest ; do + if [ -z $seconds ] ; then + echo null, + elif [ $seconds = serie ] ; then + if [ "$serie" ] ; then + echo '];' + fi + local serie=`echo $total | sed 's/cauchy_\([0-9]\)/cauchy_good_\1/g'` + echo "var $serie = [" + else + local x + if [ $workload = encode ] ; then + x=$k/$m + else + x=$k/$m/$erasures + fi + echo "[ '$x', " $(echo "( $total / 1024 / 1024 ) / $seconds" | bc -ql) " ], " + fi + done + echo '];' +} + +function main() { + bench_header + bench_run +} + +if [ "$1" = fplot ] ; then + "$@" +else + main +fi +# Local Variables: +# compile-command: "\ +# CEPH_ERASURE_CODE_BENCHMARK=../../../src/ceph_erasure_code_benchmark \ +# PLUGIN_DIRECTORY=../../../build/lib \ +# ./bench.sh +# " +# End: diff --git a/qa/workunits/erasure-code/encode-decode-non-regression.sh b/qa/workunits/erasure-code/encode-decode-non-regression.sh new file mode 100755 index 000000000..7f36c91c7 --- /dev/null +++ b/qa/workunits/erasure-code/encode-decode-non-regression.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2014 Red Hat <contact@redhat.com> +# +# Author: Loic Dachary <loic@dachary.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +set -ex + +: ${CORPUS:=https://github.com/ceph/ceph-erasure-code-corpus.git} +: ${DIRECTORY:=$CEPH_ROOT/ceph-erasure-code-corpus} + +# when running from sources, the current directory must have precedence +export PATH=:$PATH + +if ! test -d $DIRECTORY ; then + git clone $CORPUS $DIRECTORY +fi + +my_version=v$(ceph --version | cut -f3 -d ' ') + +all_versions=$((ls -d $DIRECTORY/v* ; echo $DIRECTORY/$my_version ) | sort) + +for version in $all_versions ; do + if test -d $version ; then + $version/non-regression.sh + fi + if test $version = $DIRECTORY/$my_version ; then + break + fi +done diff --git a/qa/workunits/erasure-code/examples.css b/qa/workunits/erasure-code/examples.css new file mode 100644 index 000000000..ee4724778 --- /dev/null +++ b/qa/workunits/erasure-code/examples.css @@ -0,0 +1,97 @@ +* { padding: 0; margin: 0; vertical-align: top; } + +body { + background: url(background.png) repeat-x; + font: 18px/1.5em "proxima-nova", Helvetica, Arial, sans-serif; +} + +a { color: #069; } +a:hover { color: #28b; } + +h2 { + margin-top: 15px; + font: normal 32px "omnes-pro", Helvetica, Arial, sans-serif; +} + +h3 { + margin-left: 30px; + font: normal 26px "omnes-pro", Helvetica, Arial, sans-serif; + color: #666; +} + +p { + margin-top: 10px; +} + +button { + font-size: 18px; + padding: 1px 7px; +} + +input { + font-size: 18px; +} + +input[type=checkbox] { + margin: 7px; +} + +#header { + position: relative; + width: 900px; + margin: auto; +} + +#header h2 { + margin-left: 10px; + vertical-align: middle; + font-size: 42px; + font-weight: bold; + text-decoration: none; + color: #000; +} + +#content { + width: 880px; + margin: 0 auto; + padding: 10px; +} + +#footer { + margin-top: 25px; + margin-bottom: 10px; + text-align: center; + font-size: 12px; + color: #999; +} + +.demo-container { + box-sizing: border-box; + width: 850px; + height: 450px; + padding: 20px 15px 15px 15px; + margin: 15px auto 30px auto; + border: 1px solid #ddd; + background: #fff; + background: linear-gradient(#f6f6f6 0, #fff 50px); + background: -o-linear-gradient(#f6f6f6 0, #fff 50px); + background: -ms-linear-gradient(#f6f6f6 0, #fff 50px); + background: -moz-linear-gradient(#f6f6f6 0, #fff 50px); + background: -webkit-linear-gradient(#f6f6f6 0, #fff 50px); + box-shadow: 0 3px 10px rgba(0,0,0,0.15); + -o-box-shadow: 0 3px 10px rgba(0,0,0,0.1); + -ms-box-shadow: 0 3px 10px rgba(0,0,0,0.1); + -moz-box-shadow: 0 3px 10px rgba(0,0,0,0.1); + -webkit-box-shadow: 0 3px 10px rgba(0,0,0,0.1); +} + +.demo-placeholder { + width: 100%; + height: 100%; + font-size: 14px; + line-height: 1.2em; +} + +.legend table { + border-spacing: 5px; +}
\ No newline at end of file diff --git a/qa/workunits/erasure-code/jquery.flot.categories.js b/qa/workunits/erasure-code/jquery.flot.categories.js new file mode 100644 index 000000000..2f9b25797 --- /dev/null +++ b/qa/workunits/erasure-code/jquery.flot.categories.js @@ -0,0 +1,190 @@ +/* Flot plugin for plotting textual data or categories. + +Copyright (c) 2007-2014 IOLA and Ole Laursen. +Licensed under the MIT license. + +Consider a dataset like [["February", 34], ["March", 20], ...]. This plugin +allows you to plot such a dataset directly. + +To enable it, you must specify mode: "categories" on the axis with the textual +labels, e.g. + + $.plot("#placeholder", data, { xaxis: { mode: "categories" } }); + +By default, the labels are ordered as they are met in the data series. If you +need a different ordering, you can specify "categories" on the axis options +and list the categories there: + + xaxis: { + mode: "categories", + categories: ["February", "March", "April"] + } + +If you need to customize the distances between the categories, you can specify +"categories" as an object mapping labels to values + + xaxis: { + mode: "categories", + categories: { "February": 1, "March": 3, "April": 4 } + } + +If you don't specify all categories, the remaining categories will be numbered +from the max value plus 1 (with a spacing of 1 between each). + +Internally, the plugin works by transforming the input data through an auto- +generated mapping where the first category becomes 0, the second 1, etc. +Hence, a point like ["February", 34] becomes [0, 34] internally in Flot (this +is visible in hover and click events that return numbers rather than the +category labels). The plugin also overrides the tick generator to spit out the +categories as ticks instead of the values. + +If you need to map a value back to its label, the mapping is always accessible +as "categories" on the axis object, e.g. plot.getAxes().xaxis.categories. + +*/ + +(function ($) { + var options = { + xaxis: { + categories: null + }, + yaxis: { + categories: null + } + }; + + function processRawData(plot, series, data, datapoints) { + // if categories are enabled, we need to disable + // auto-transformation to numbers so the strings are intact + // for later processing + + var xCategories = series.xaxis.options.mode == "categories", + yCategories = series.yaxis.options.mode == "categories"; + + if (!(xCategories || yCategories)) + return; + + var format = datapoints.format; + + if (!format) { + // FIXME: auto-detection should really not be defined here + var s = series; + format = []; + format.push({ x: true, number: true, required: true }); + format.push({ y: true, number: true, required: true }); + + if (s.bars.show || (s.lines.show && s.lines.fill)) { + var autoscale = !!((s.bars.show && s.bars.zero) || (s.lines.show && s.lines.zero)); + format.push({ y: true, number: true, required: false, defaultValue: 0, autoscale: autoscale }); + if (s.bars.horizontal) { + delete format[format.length - 1].y; + format[format.length - 1].x = true; + } + } + + datapoints.format = format; + } + + for (var m = 0; m < format.length; ++m) { + if (format[m].x && xCategories) + format[m].number = false; + + if (format[m].y && yCategories) + format[m].number = false; + } + } + + function getNextIndex(categories) { + var index = -1; + + for (var v in categories) + if (categories[v] > index) + index = categories[v]; + + return index + 1; + } + + function categoriesTickGenerator(axis) { + var res = []; + for (var label in axis.categories) { + var v = axis.categories[label]; + if (v >= axis.min && v <= axis.max) + res.push([v, label]); + } + + res.sort(function (a, b) { return a[0] - b[0]; }); + + return res; + } + + function setupCategoriesForAxis(series, axis, datapoints) { + if (series[axis].options.mode != "categories") + return; + + if (!series[axis].categories) { + // parse options + var c = {}, o = series[axis].options.categories || {}; + if ($.isArray(o)) { + for (var i = 0; i < o.length; ++i) + c[o[i]] = i; + } + else { + for (var v in o) + c[v] = o[v]; + } + + series[axis].categories = c; + } + + // fix ticks + if (!series[axis].options.ticks) + series[axis].options.ticks = categoriesTickGenerator; + + transformPointsOnAxis(datapoints, axis, series[axis].categories); + } + + function transformPointsOnAxis(datapoints, axis, categories) { + // go through the points, transforming them + var points = datapoints.points, + ps = datapoints.pointsize, + format = datapoints.format, + formatColumn = axis.charAt(0), + index = getNextIndex(categories); + + for (var i = 0; i < points.length; i += ps) { + if (points[i] == null) + continue; + + for (var m = 0; m < ps; ++m) { + var val = points[i + m]; + + if (val == null || !format[m][formatColumn]) + continue; + + if (!(val in categories)) { + categories[val] = index; + ++index; + } + + points[i + m] = categories[val]; + } + } + } + + function processDatapoints(plot, series, datapoints) { + setupCategoriesForAxis(series, "xaxis", datapoints); + setupCategoriesForAxis(series, "yaxis", datapoints); + } + + function init(plot) { + plot.hooks.processRawData.push(processRawData); + plot.hooks.processDatapoints.push(processDatapoints); + } + + $.plot.plugins.push({ + init: init, + options: options, + name: 'categories', + version: '1.0' + }); +})(jQuery); diff --git a/qa/workunits/erasure-code/jquery.flot.js b/qa/workunits/erasure-code/jquery.flot.js new file mode 100644 index 000000000..39f3e4cf3 --- /dev/null +++ b/qa/workunits/erasure-code/jquery.flot.js @@ -0,0 +1,3168 @@ +/* Javascript plotting library for jQuery, version 0.8.3. + +Copyright (c) 2007-2014 IOLA and Ole Laursen. +Licensed under the MIT license. + +*/ + +// first an inline dependency, jquery.colorhelpers.js, we inline it here +// for convenience + +/* Plugin for jQuery for working with colors. + * + * Version 1.1. + * + * Inspiration from jQuery color animation plugin by John Resig. + * + * Released under the MIT license by Ole Laursen, October 2009. + * + * Examples: + * + * $.color.parse("#fff").scale('rgb', 0.25).add('a', -0.5).toString() + * var c = $.color.extract($("#mydiv"), 'background-color'); + * console.log(c.r, c.g, c.b, c.a); + * $.color.make(100, 50, 25, 0.4).toString() // returns "rgba(100,50,25,0.4)" + * + * Note that .scale() and .add() return the same modified object + * instead of making a new one. + * + * V. 1.1: Fix error handling so e.g. parsing an empty string does + * produce a color rather than just crashing. + */ +(function($){$.color={};$.color.make=function(r,g,b,a){var o={};o.r=r||0;o.g=g||0;o.b=b||0;o.a=a!=null?a:1;o.add=function(c,d){for(var i=0;i<c.length;++i)o[c.charAt(i)]+=d;return o.normalize()};o.scale=function(c,f){for(var i=0;i<c.length;++i)o[c.charAt(i)]*=f;return o.normalize()};o.toString=function(){if(o.a>=1){return"rgb("+[o.r,o.g,o.b].join(",")+")"}else{return"rgba("+[o.r,o.g,o.b,o.a].join(",")+")"}};o.normalize=function(){function clamp(min,value,max){return value<min?min:value>max?max:value}o.r=clamp(0,parseInt(o.r),255);o.g=clamp(0,parseInt(o.g),255);o.b=clamp(0,parseInt(o.b),255);o.a=clamp(0,o.a,1);return o};o.clone=function(){return $.color.make(o.r,o.b,o.g,o.a)};return o.normalize()};$.color.extract=function(elem,css){var c;do{c=elem.css(css).toLowerCase();if(c!=""&&c!="transparent")break;elem=elem.parent()}while(elem.length&&!$.nodeName(elem.get(0),"body"));if(c=="rgba(0, 0, 0, 0)")c="transparent";return $.color.parse(c)};$.color.parse=function(str){var res,m=$.color.make;if(res=/rgb\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*\)/.exec(str))return m(parseInt(res[1],10),parseInt(res[2],10),parseInt(res[3],10));if(res=/rgba\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(str))return m(parseInt(res[1],10),parseInt(res[2],10),parseInt(res[3],10),parseFloat(res[4]));if(res=/rgb\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*\)/.exec(str))return m(parseFloat(res[1])*2.55,parseFloat(res[2])*2.55,parseFloat(res[3])*2.55);if(res=/rgba\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(str))return m(parseFloat(res[1])*2.55,parseFloat(res[2])*2.55,parseFloat(res[3])*2.55,parseFloat(res[4]));if(res=/#([a-fA-F0-9]{2})([a-fA-F0-9]{2})([a-fA-F0-9]{2})/.exec(str))return m(parseInt(res[1],16),parseInt(res[2],16),parseInt(res[3],16));if(res=/#([a-fA-F0-9])([a-fA-F0-9])([a-fA-F0-9])/.exec(str))return m(parseInt(res[1]+res[1],16),parseInt(res[2]+res[2],16),parseInt(res[3]+res[3],16));var name=$.trim(str).toLowerCase();if(name=="transparent")return m(255,255,255,0);else{res=lookupColors[name]||[0,0,0];return m(res[0],res[1],res[2])}};var lookupColors={aqua:[0,255,255],azure:[240,255,255],beige:[245,245,220],black:[0,0,0],blue:[0,0,255],brown:[165,42,42],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgrey:[169,169,169],darkgreen:[0,100,0],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkviolet:[148,0,211],fuchsia:[255,0,255],gold:[255,215,0],green:[0,128,0],indigo:[75,0,130],khaki:[240,230,140],lightblue:[173,216,230],lightcyan:[224,255,255],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightyellow:[255,255,224],lime:[0,255,0],magenta:[255,0,255],maroon:[128,0,0],navy:[0,0,128],olive:[128,128,0],orange:[255,165,0],pink:[255,192,203],purple:[128,0,128],violet:[128,0,128],red:[255,0,0],silver:[192,192,192],white:[255,255,255],yellow:[255,255,0]}})(jQuery); + +// the actual Flot code +(function($) { + + // Cache the prototype hasOwnProperty for faster access + + var hasOwnProperty = Object.prototype.hasOwnProperty; + + // A shim to provide 'detach' to jQuery versions prior to 1.4. Using a DOM + // operation produces the same effect as detach, i.e. removing the element + // without touching its jQuery data. + + // Do not merge this into Flot 0.9, since it requires jQuery 1.4.4+. + + if (!$.fn.detach) { + $.fn.detach = function() { + return this.each(function() { + if (this.parentNode) { + this.parentNode.removeChild( this ); + } + }); + }; + } + + /////////////////////////////////////////////////////////////////////////// + // The Canvas object is a wrapper around an HTML5 <canvas> tag. + // + // @constructor + // @param {string} cls List of classes to apply to the canvas. + // @param {element} container Element onto which to append the canvas. + // + // Requiring a container is a little iffy, but unfortunately canvas + // operations don't work unless the canvas is attached to the DOM. + + function Canvas(cls, container) { + + var element = container.children("." + cls)[0]; + + if (element == null) { + + element = document.createElement("canvas"); + element.className = cls; + + $(element).css({ direction: "ltr", position: "absolute", left: 0, top: 0 }) + .appendTo(container); + + // If HTML5 Canvas isn't available, fall back to [Ex|Flash]canvas + + if (!element.getContext) { + if (window.G_vmlCanvasManager) { + element = window.G_vmlCanvasManager.initElement(element); + } else { + throw new Error("Canvas is not available. If you're using IE with a fall-back such as Excanvas, then there's either a mistake in your conditional include, or the page has no DOCTYPE and is rendering in Quirks Mode."); + } + } + } + + this.element = element; + + var context = this.context = element.getContext("2d"); + + // Determine the screen's ratio of physical to device-independent + // pixels. This is the ratio between the canvas width that the browser + // advertises and the number of pixels actually present in that space. + + // The iPhone 4, for example, has a device-independent width of 320px, + // but its screen is actually 640px wide. It therefore has a pixel + // ratio of 2, while most normal devices have a ratio of 1. + + var devicePixelRatio = window.devicePixelRatio || 1, + backingStoreRatio = + context.webkitBackingStorePixelRatio || + context.mozBackingStorePixelRatio || + context.msBackingStorePixelRatio || + context.oBackingStorePixelRatio || + context.backingStorePixelRatio || 1; + + this.pixelRatio = devicePixelRatio / backingStoreRatio; + + // Size the canvas to match the internal dimensions of its container + + this.resize(container.width(), container.height()); + + // Collection of HTML div layers for text overlaid onto the canvas + + this.textContainer = null; + this.text = {}; + + // Cache of text fragments and metrics, so we can avoid expensively + // re-calculating them when the plot is re-rendered in a loop. + + this._textCache = {}; + } + + // Resizes the canvas to the given dimensions. + // + // @param {number} width New width of the canvas, in pixels. + // @param {number} width New height of the canvas, in pixels. + + Canvas.prototype.resize = function(width, height) { + + if (width <= 0 || height <= 0) { + throw new Error("Invalid dimensions for plot, width = " + width + ", height = " + height); + } + + var element = this.element, + context = this.context, + pixelRatio = this.pixelRatio; + + // Resize the canvas, increasing its density based on the display's + // pixel ratio; basically giving it more pixels without increasing the + // size of its element, to take advantage of the fact that retina + // displays have that many more pixels in the same advertised space. + + // Resizing should reset the state (excanvas seems to be buggy though) + + if (this.width != width) { + element.width = width * pixelRatio; + element.style.width = width + "px"; + this.width = width; + } + + if (this.height != height) { + element.height = height * pixelRatio; + element.style.height = height + "px"; + this.height = height; + } + + // Save the context, so we can reset in case we get replotted. The + // restore ensure that we're really back at the initial state, and + // should be safe even if we haven't saved the initial state yet. + + context.restore(); + context.save(); + + // Scale the coordinate space to match the display density; so even though we + // may have twice as many pixels, we still want lines and other drawing to + // appear at the same size; the extra pixels will just make them crisper. + + context.scale(pixelRatio, pixelRatio); + }; + + // Clears the entire canvas area, not including any overlaid HTML text + + Canvas.prototype.clear = function() { + this.context.clearRect(0, 0, this.width, this.height); + }; + + // Finishes rendering the canvas, including managing the text overlay. + + Canvas.prototype.render = function() { + + var cache = this._textCache; + + // For each text layer, add elements marked as active that haven't + // already been rendered, and remove those that are no longer active. + + for (var layerKey in cache) { + if (hasOwnProperty.call(cache, layerKey)) { + + var layer = this.getTextLayer(layerKey), + layerCache = cache[layerKey]; + + layer.hide(); + + for (var styleKey in layerCache) { + if (hasOwnProperty.call(layerCache, styleKey)) { + var styleCache = layerCache[styleKey]; + for (var key in styleCache) { + if (hasOwnProperty.call(styleCache, key)) { + + var positions = styleCache[key].positions; + + for (var i = 0, position; position = positions[i]; i++) { + if (position.active) { + if (!position.rendered) { + layer.append(position.element); + position.rendered = true; + } + } else { + positions.splice(i--, 1); + if (position.rendered) { + position.element.detach(); + } + } + } + + if (positions.length == 0) { + delete styleCache[key]; + } + } + } + } + } + + layer.show(); + } + } + }; + + // Creates (if necessary) and returns the text overlay container. + // + // @param {string} classes String of space-separated CSS classes used to + // uniquely identify the text layer. + // @return {object} The jQuery-wrapped text-layer div. + + Canvas.prototype.getTextLayer = function(classes) { + + var layer = this.text[classes]; + + // Create the text layer if it doesn't exist + + if (layer == null) { + + // Create the text layer container, if it doesn't exist + + if (this.textContainer == null) { + this.textContainer = $("<div class='flot-text'></div>") + .css({ + position: "absolute", + top: 0, + left: 0, + bottom: 0, + right: 0, + 'font-size': "smaller", + color: "#545454" + }) + .insertAfter(this.element); + } + + layer = this.text[classes] = $("<div></div>") + .addClass(classes) + .css({ + position: "absolute", + top: 0, + left: 0, + bottom: 0, + right: 0 + }) + .appendTo(this.textContainer); + } + + return layer; + }; + + // Creates (if necessary) and returns a text info object. + // + // The object looks like this: + // + // { + // width: Width of the text's wrapper div. + // height: Height of the text's wrapper div. + // element: The jQuery-wrapped HTML div containing the text. + // positions: Array of positions at which this text is drawn. + // } + // + // The positions array contains objects that look like this: + // + // { + // active: Flag indicating whether the text should be visible. + // rendered: Flag indicating whether the text is currently visible. + // element: The jQuery-wrapped HTML div containing the text. + // x: X coordinate at which to draw the text. + // y: Y coordinate at which to draw the text. + // } + // + // Each position after the first receives a clone of the original element. + // + // The idea is that that the width, height, and general 'identity' of the + // text is constant no matter where it is placed; the placements are a + // secondary property. + // + // Canvas maintains a cache of recently-used text info objects; getTextInfo + // either returns the cached element or creates a new entry. + // + // @param {string} layer A string of space-separated CSS classes uniquely + // identifying the layer containing this text. + // @param {string} text Text string to retrieve info for. + // @param {(string|object)=} font Either a string of space-separated CSS + // classes or a font-spec object, defining the text's font and style. + // @param {number=} angle Angle at which to rotate the text, in degrees. + // Angle is currently unused, it will be implemented in the future. + // @param {number=} width Maximum width of the text before it wraps. + // @return {object} a text info object. + + Canvas.prototype.getTextInfo = function(layer, text, font, angle, width) { + + var textStyle, layerCache, styleCache, info; + + // Cast the value to a string, in case we were given a number or such + + text = "" + text; + + // If the font is a font-spec object, generate a CSS font definition + + if (typeof font === "object") { + textStyle = font.style + " " + font.variant + " " + font.weight + " " + font.size + "px/" + font.lineHeight + "px " + font.family; + } else { + textStyle = font; + } + + // Retrieve (or create) the cache for the text's layer and styles + + layerCache = this._textCache[layer]; + + if (layerCache == null) { + layerCache = this._textCache[layer] = {}; + } + + styleCache = layerCache[textStyle]; + + if (styleCache == null) { + styleCache = layerCache[textStyle] = {}; + } + + info = styleCache[text]; + + // If we can't find a matching element in our cache, create a new one + + if (info == null) { + + var element = $("<div></div>").html(text) + .css({ + position: "absolute", + 'max-width': width, + top: -9999 + }) + .appendTo(this.getTextLayer(layer)); + + if (typeof font === "object") { + element.css({ + font: textStyle, + color: font.color + }); + } else if (typeof font === "string") { + element.addClass(font); + } + + info = styleCache[text] = { + width: element.outerWidth(true), + height: element.outerHeight(true), + element: element, + positions: [] + }; + + element.detach(); + } + + return info; + }; + + // Adds a text string to the canvas text overlay. + // + // The text isn't drawn immediately; it is marked as rendering, which will + // result in its addition to the canvas on the next render pass. + // + // @param {string} layer A string of space-separated CSS classes uniquely + // identifying the layer containing this text. + // @param {number} x X coordinate at which to draw the text. + // @param {number} y Y coordinate at which to draw the text. + // @param {string} text Text string to draw. + // @param {(string|object)=} font Either a string of space-separated CSS + // classes or a font-spec object, defining the text's font and style. + // @param {number=} angle Angle at which to rotate the text, in degrees. + // Angle is currently unused, it will be implemented in the future. + // @param {number=} width Maximum width of the text before it wraps. + // @param {string=} halign Horizontal alignment of the text; either "left", + // "center" or "right". + // @param {string=} valign Vertical alignment of the text; either "top", + // "middle" or "bottom". + + Canvas.prototype.addText = function(layer, x, y, text, font, angle, width, halign, valign) { + + var info = this.getTextInfo(layer, text, font, angle, width), + positions = info.positions; + + // Tweak the div's position to match the text's alignment + + if (halign == "center") { + x -= info.width / 2; + } else if (halign == "right") { + x -= info.width; + } + + if (valign == "middle") { + y -= info.height / 2; + } else if (valign == "bottom") { + y -= info.height; + } + + // Determine whether this text already exists at this position. + // If so, mark it for inclusion in the next render pass. + + for (var i = 0, position; position = positions[i]; i++) { + if (position.x == x && position.y == y) { + position.active = true; + return; + } + } + + // If the text doesn't exist at this position, create a new entry + + // For the very first position we'll re-use the original element, + // while for subsequent ones we'll clone it. + + position = { + active: true, + rendered: false, + element: positions.length ? info.element.clone() : info.element, + x: x, + y: y + }; + + positions.push(position); + + // Move the element to its final position within the container + + position.element.css({ + top: Math.round(y), + left: Math.round(x), + 'text-align': halign // In case the text wraps + }); + }; + + // Removes one or more text strings from the canvas text overlay. + // + // If no parameters are given, all text within the layer is removed. + // + // Note that the text is not immediately removed; it is simply marked as + // inactive, which will result in its removal on the next render pass. + // This avoids the performance penalty for 'clear and redraw' behavior, + // where we potentially get rid of all text on a layer, but will likely + // add back most or all of it later, as when redrawing axes, for example. + // + // @param {string} layer A string of space-separated CSS classes uniquely + // identifying the layer containing this text. + // @param {number=} x X coordinate of the text. + // @param {number=} y Y coordinate of the text. + // @param {string=} text Text string to remove. + // @param {(string|object)=} font Either a string of space-separated CSS + // classes or a font-spec object, defining the text's font and style. + // @param {number=} angle Angle at which the text is rotated, in degrees. + // Angle is currently unused, it will be implemented in the future. + + Canvas.prototype.removeText = function(layer, x, y, text, font, angle) { + if (text == null) { + var layerCache = this._textCache[layer]; + if (layerCache != null) { + for (var styleKey in layerCache) { + if (hasOwnProperty.call(layerCache, styleKey)) { + var styleCache = layerCache[styleKey]; + for (var key in styleCache) { + if (hasOwnProperty.call(styleCache, key)) { + var positions = styleCache[key].positions; + for (var i = 0, position; position = positions[i]; i++) { + position.active = false; + } + } + } + } + } + } + } else { + var positions = this.getTextInfo(layer, text, font, angle).positions; + for (var i = 0, position; position = positions[i]; i++) { + if (position.x == x && position.y == y) { + position.active = false; + } + } + } + }; + + /////////////////////////////////////////////////////////////////////////// + // The top-level container for the entire plot. + + function Plot(placeholder, data_, options_, plugins) { + // data is on the form: + // [ series1, series2 ... ] + // where series is either just the data as [ [x1, y1], [x2, y2], ... ] + // or { data: [ [x1, y1], [x2, y2], ... ], label: "some label", ... } + + var series = [], + options = { + // the color theme used for graphs + colors: ["#edc240", "#afd8f8", "#cb4b4b", "#4da74d", "#9440ed"], + legend: { + show: true, + noColumns: 1, // number of colums in legend table + labelFormatter: null, // fn: string -> string + labelBoxBorderColor: "#ccc", // border color for the little label boxes + container: null, // container (as jQuery object) to put legend in, null means default on top of graph + position: "ne", // position of default legend container within plot + margin: 5, // distance from grid edge to default legend container within plot + backgroundColor: null, // null means auto-detect + backgroundOpacity: 0.85, // set to 0 to avoid background + sorted: null // default to no legend sorting + }, + xaxis: { + show: null, // null = auto-detect, true = always, false = never + position: "bottom", // or "top" + mode: null, // null or "time" + font: null, // null (derived from CSS in placeholder) or object like { size: 11, lineHeight: 13, style: "italic", weight: "bold", family: "sans-serif", variant: "small-caps" } + color: null, // base color, labels, ticks + tickColor: null, // possibly different color of ticks, e.g. "rgba(0,0,0,0.15)" + transform: null, // null or f: number -> number to transform axis + inverseTransform: null, // if transform is set, this should be the inverse function + min: null, // min. value to show, null means set automatically + max: null, // max. value to show, null means set automatically + autoscaleMargin: null, // margin in % to add if auto-setting min/max + ticks: null, // either [1, 3] or [[1, "a"], 3] or (fn: axis info -> ticks) or app. number of ticks for auto-ticks + tickFormatter: null, // fn: number -> string + labelWidth: null, // size of tick labels in pixels + labelHeight: null, + reserveSpace: null, // whether to reserve space even if axis isn't shown + tickLength: null, // size in pixels of ticks, or "full" for whole line + alignTicksWithAxis: null, // axis number or null for no sync + tickDecimals: null, // no. of decimals, null means auto + tickSize: null, // number or [number, "unit"] + minTickSize: null // number or [number, "unit"] + }, + yaxis: { + autoscaleMargin: 0.02, + position: "left" // or "right" + }, + xaxes: [], + yaxes: [], + series: { + points: { + show: false, + radius: 3, + lineWidth: 2, // in pixels + fill: true, + fillColor: "#ffffff", + symbol: "circle" // or callback + }, + lines: { + // we don't put in show: false so we can see + // whether lines were actively disabled + lineWidth: 2, // in pixels + fill: false, + fillColor: null, + steps: false + // Omit 'zero', so we can later default its value to + // match that of the 'fill' option. + }, + bars: { + show: false, + lineWidth: 2, // in pixels + barWidth: 1, // in units of the x axis + fill: true, + fillColor: null, + align: "left", // "left", "right", or "center" + horizontal: false, + zero: true + }, + shadowSize: 3, + highlightColor: null + }, + grid: { + show: true, + aboveData: false, + color: "#545454", // primary color used for outline and labels + backgroundColor: null, // null for transparent, else color + borderColor: null, // set if different from the grid color + tickColor: null, // color for the ticks, e.g. "rgba(0,0,0,0.15)" + margin: 0, // distance from the canvas edge to the grid + labelMargin: 5, // in pixels + axisMargin: 8, // in pixels + borderWidth: 2, // in pixels + minBorderMargin: null, // in pixels, null means taken from points radius + markings: null, // array of ranges or fn: axes -> array of ranges + markingsColor: "#f4f4f4", + markingsLineWidth: 2, + // interactive stuff + clickable: false, + hoverable: false, + autoHighlight: true, // highlight in case mouse is near + mouseActiveRadius: 10 // how far the mouse can be away to activate an item + }, + interaction: { + redrawOverlayInterval: 1000/60 // time between updates, -1 means in same flow + }, + hooks: {} + }, + surface = null, // the canvas for the plot itself + overlay = null, // canvas for interactive stuff on top of plot + eventHolder = null, // jQuery object that events should be bound to + ctx = null, octx = null, + xaxes = [], yaxes = [], + plotOffset = { left: 0, right: 0, top: 0, bottom: 0}, + plotWidth = 0, plotHeight = 0, + hooks = { + processOptions: [], + processRawData: [], + processDatapoints: [], + processOffset: [], + drawBackground: [], + drawSeries: [], + draw: [], + bindEvents: [], + drawOverlay: [], + shutdown: [] + }, + plot = this; + + // public functions + plot.setData = setData; + plot.setupGrid = setupGrid; + plot.draw = draw; + plot.getPlaceholder = function() { return placeholder; }; + plot.getCanvas = function() { return surface.element; }; + plot.getPlotOffset = function() { return plotOffset; }; + plot.width = function () { return plotWidth; }; + plot.height = function () { return plotHeight; }; + plot.offset = function () { + var o = eventHolder.offset(); + o.left += plotOffset.left; + o.top += plotOffset.top; + return o; + }; + plot.getData = function () { return series; }; + plot.getAxes = function () { + var res = {}, i; + $.each(xaxes.concat(yaxes), function (_, axis) { + if (axis) + res[axis.direction + (axis.n != 1 ? axis.n : "") + "axis"] = axis; + }); + return res; + }; + plot.getXAxes = function () { return xaxes; }; + plot.getYAxes = function () { return yaxes; }; + plot.c2p = canvasToAxisCoords; + plot.p2c = axisToCanvasCoords; + plot.getOptions = function () { return options; }; + plot.highlight = highlight; + plot.unhighlight = unhighlight; + plot.triggerRedrawOverlay = triggerRedrawOverlay; + plot.pointOffset = function(point) { + return { + left: parseInt(xaxes[axisNumber(point, "x") - 1].p2c(+point.x) + plotOffset.left, 10), + top: parseInt(yaxes[axisNumber(point, "y") - 1].p2c(+point.y) + plotOffset.top, 10) + }; + }; + plot.shutdown = shutdown; + plot.destroy = function () { + shutdown(); + placeholder.removeData("plot").empty(); + + series = []; + options = null; + surface = null; + overlay = null; + eventHolder = null; + ctx = null; + octx = null; + xaxes = []; + yaxes = []; + hooks = null; + highlights = []; + plot = null; + }; + plot.resize = function () { + var width = placeholder.width(), + height = placeholder.height(); + surface.resize(width, height); + overlay.resize(width, height); + }; + + // public attributes + plot.hooks = hooks; + + // initialize + initPlugins(plot); + parseOptions(options_); + setupCanvases(); + setData(data_); + setupGrid(); + draw(); + bindEvents(); + + + function executeHooks(hook, args) { + args = [plot].concat(args); + for (var i = 0; i < hook.length; ++i) + hook[i].apply(this, args); + } + + function initPlugins() { + + // References to key classes, allowing plugins to modify them + + var classes = { + Canvas: Canvas + }; + + for (var i = 0; i < plugins.length; ++i) { + var p = plugins[i]; + p.init(plot, classes); + if (p.options) + $.extend(true, options, p.options); + } + } + + function parseOptions(opts) { + + $.extend(true, options, opts); + + // $.extend merges arrays, rather than replacing them. When less + // colors are provided than the size of the default palette, we + // end up with those colors plus the remaining defaults, which is + // not expected behavior; avoid it by replacing them here. + + if (opts && opts.colors) { + options.colors = opts.colors; + } + + if (options.xaxis.color == null) + options.xaxis.color = $.color.parse(options.grid.color).scale('a', 0.22).toString(); + if (options.yaxis.color == null) + options.yaxis.color = $.color.parse(options.grid.color).scale('a', 0.22).toString(); + + if (options.xaxis.tickColor == null) // grid.tickColor for back-compatibility + options.xaxis.tickColor = options.grid.tickColor || options.xaxis.color; + if (options.yaxis.tickColor == null) // grid.tickColor for back-compatibility + options.yaxis.tickColor = options.grid.tickColor || options.yaxis.color; + + if (options.grid.borderColor == null) + options.grid.borderColor = options.grid.color; + if (options.grid.tickColor == null) + options.grid.tickColor = $.color.parse(options.grid.color).scale('a', 0.22).toString(); + + // Fill in defaults for axis options, including any unspecified + // font-spec fields, if a font-spec was provided. + + // If no x/y axis options were provided, create one of each anyway, + // since the rest of the code assumes that they exist. + + var i, axisOptions, axisCount, + fontSize = placeholder.css("font-size"), + fontSizeDefault = fontSize ? +fontSize.replace("px", "") : 13, + fontDefaults = { + style: placeholder.css("font-style"), + size: Math.round(0.8 * fontSizeDefault), + variant: placeholder.css("font-variant"), + weight: placeholder.css("font-weight"), + family: placeholder.css("font-family") + }; + + axisCount = options.xaxes.length || 1; + for (i = 0; i < axisCount; ++i) { + + axisOptions = options.xaxes[i]; + if (axisOptions && !axisOptions.tickColor) { + axisOptions.tickColor = axisOptions.color; + } + + axisOptions = $.extend(true, {}, options.xaxis, axisOptions); + options.xaxes[i] = axisOptions; + + if (axisOptions.font) { + axisOptions.font = $.extend({}, fontDefaults, axisOptions.font); + if (!axisOptions.font.color) { + axisOptions.font.color = axisOptions.color; + } + if (!axisOptions.font.lineHeight) { + axisOptions.font.lineHeight = Math.round(axisOptions.font.size * 1.15); + } + } + } + + axisCount = options.yaxes.length || 1; + for (i = 0; i < axisCount; ++i) { + + axisOptions = options.yaxes[i]; + if (axisOptions && !axisOptions.tickColor) { + axisOptions.tickColor = axisOptions.color; + } + + axisOptions = $.extend(true, {}, options.yaxis, axisOptions); + options.yaxes[i] = axisOptions; + + if (axisOptions.font) { + axisOptions.font = $.extend({}, fontDefaults, axisOptions.font); + if (!axisOptions.font.color) { + axisOptions.font.color = axisOptions.color; + } + if (!axisOptions.font.lineHeight) { + axisOptions.font.lineHeight = Math.round(axisOptions.font.size * 1.15); + } + } + } + + // backwards compatibility, to be removed in future + if (options.xaxis.noTicks && options.xaxis.ticks == null) + options.xaxis.ticks = options.xaxis.noTicks; + if (options.yaxis.noTicks && options.yaxis.ticks == null) + options.yaxis.ticks = options.yaxis.noTicks; + if (options.x2axis) { + options.xaxes[1] = $.extend(true, {}, options.xaxis, options.x2axis); + options.xaxes[1].position = "top"; + // Override the inherit to allow the axis to auto-scale + if (options.x2axis.min == null) { + options.xaxes[1].min = null; + } + if (options.x2axis.max == null) { + options.xaxes[1].max = null; + } + } + if (options.y2axis) { + options.yaxes[1] = $.extend(true, {}, options.yaxis, options.y2axis); + options.yaxes[1].position = "right"; + // Override the inherit to allow the axis to auto-scale + if (options.y2axis.min == null) { + options.yaxes[1].min = null; + } + if (options.y2axis.max == null) { + options.yaxes[1].max = null; + } + } + if (options.grid.coloredAreas) + options.grid.markings = options.grid.coloredAreas; + if (options.grid.coloredAreasColor) + options.grid.markingsColor = options.grid.coloredAreasColor; + if (options.lines) + $.extend(true, options.series.lines, options.lines); + if (options.points) + $.extend(true, options.series.points, options.points); + if (options.bars) + $.extend(true, options.series.bars, options.bars); + if (options.shadowSize != null) + options.series.shadowSize = options.shadowSize; + if (options.highlightColor != null) + options.series.highlightColor = options.highlightColor; + + // save options on axes for future reference + for (i = 0; i < options.xaxes.length; ++i) + getOrCreateAxis(xaxes, i + 1).options = options.xaxes[i]; + for (i = 0; i < options.yaxes.length; ++i) + getOrCreateAxis(yaxes, i + 1).options = options.yaxes[i]; + + // add hooks from options + for (var n in hooks) + if (options.hooks[n] && options.hooks[n].length) + hooks[n] = hooks[n].concat(options.hooks[n]); + + executeHooks(hooks.processOptions, [options]); + } + + function setData(d) { + series = parseData(d); + fillInSeriesOptions(); + processData(); + } + + function parseData(d) { + var res = []; + for (var i = 0; i < d.length; ++i) { + var s = $.extend(true, {}, options.series); + + if (d[i].data != null) { + s.data = d[i].data; // move the data instead of deep-copy + delete d[i].data; + + $.extend(true, s, d[i]); + + d[i].data = s.data; + } + else + s.data = d[i]; + res.push(s); + } + + return res; + } + + function axisNumber(obj, coord) { + var a = obj[coord + "axis"]; + if (typeof a == "object") // if we got a real axis, extract number + a = a.n; + if (typeof a != "number") + a = 1; // default to first axis + return a; + } + + function allAxes() { + // return flat array without annoying null entries + return $.grep(xaxes.concat(yaxes), function (a) { return a; }); + } + + function canvasToAxisCoords(pos) { + // return an object with x/y corresponding to all used axes + var res = {}, i, axis; + for (i = 0; i < xaxes.length; ++i) { + axis = xaxes[i]; + if (axis && axis.used) + res["x" + axis.n] = axis.c2p(pos.left); + } + + for (i = 0; i < yaxes.length; ++i) { + axis = yaxes[i]; + if (axis && axis.used) + res["y" + axis.n] = axis.c2p(pos.top); + } + + if (res.x1 !== undefined) + res.x = res.x1; + if (res.y1 !== undefined) + res.y = res.y1; + + return res; + } + + function axisToCanvasCoords(pos) { + // get canvas coords from the first pair of x/y found in pos + var res = {}, i, axis, key; + + for (i = 0; i < xaxes.length; ++i) { + axis = xaxes[i]; + if (axis && axis.used) { + key = "x" + axis.n; + if (pos[key] == null && axis.n == 1) + key = "x"; + + if (pos[key] != null) { + res.left = axis.p2c(pos[key]); + break; + } + } + } + + for (i = 0; i < yaxes.length; ++i) { + axis = yaxes[i]; + if (axis && axis.used) { + key = "y" + axis.n; + if (pos[key] == null && axis.n == 1) + key = "y"; + + if (pos[key] != null) { + res.top = axis.p2c(pos[key]); + break; + } + } + } + + return res; + } + + function getOrCreateAxis(axes, number) { + if (!axes[number - 1]) + axes[number - 1] = { + n: number, // save the number for future reference + direction: axes == xaxes ? "x" : "y", + options: $.extend(true, {}, axes == xaxes ? options.xaxis : options.yaxis) + }; + + return axes[number - 1]; + } + + function fillInSeriesOptions() { + + var neededColors = series.length, maxIndex = -1, i; + + // Subtract the number of series that already have fixed colors or + // color indexes from the number that we still need to generate. + + for (i = 0; i < series.length; ++i) { + var sc = series[i].color; + if (sc != null) { + neededColors--; + if (typeof sc == "number" && sc > maxIndex) { + maxIndex = sc; + } + } + } + + // If any of the series have fixed color indexes, then we need to + // generate at least as many colors as the highest index. + + if (neededColors <= maxIndex) { + neededColors = maxIndex + 1; + } + + // Generate all the colors, using first the option colors and then + // variations on those colors once they're exhausted. + + var c, colors = [], colorPool = options.colors, + colorPoolSize = colorPool.length, variation = 0; + + for (i = 0; i < neededColors; i++) { + + c = $.color.parse(colorPool[i % colorPoolSize] || "#666"); + + // Each time we exhaust the colors in the pool we adjust + // a scaling factor used to produce more variations on + // those colors. The factor alternates negative/positive + // to produce lighter/darker colors. + + // Reset the variation after every few cycles, or else + // it will end up producing only white or black colors. + + if (i % colorPoolSize == 0 && i) { + if (variation >= 0) { + if (variation < 0.5) { + variation = -variation - 0.2; + } else variation = 0; + } else variation = -variation; + } + + colors[i] = c.scale('rgb', 1 + variation); + } + + // Finalize the series options, filling in their colors + + var colori = 0, s; + for (i = 0; i < series.length; ++i) { + s = series[i]; + + // assign colors + if (s.color == null) { + s.color = colors[colori].toString(); + ++colori; + } + else if (typeof s.color == "number") + s.color = colors[s.color].toString(); + + // turn on lines automatically in case nothing is set + if (s.lines.show == null) { + var v, show = true; + for (v in s) + if (s[v] && s[v].show) { + show = false; + break; + } + if (show) + s.lines.show = true; + } + + // If nothing was provided for lines.zero, default it to match + // lines.fill, since areas by default should extend to zero. + + if (s.lines.zero == null) { + s.lines.zero = !!s.lines.fill; + } + + // setup axes + s.xaxis = getOrCreateAxis(xaxes, axisNumber(s, "x")); + s.yaxis = getOrCreateAxis(yaxes, axisNumber(s, "y")); + } + } + + function processData() { + var topSentry = Number.POSITIVE_INFINITY, + bottomSentry = Number.NEGATIVE_INFINITY, + fakeInfinity = Number.MAX_VALUE, + i, j, k, m, length, + s, points, ps, x, y, axis, val, f, p, + data, format; + + function updateAxis(axis, min, max) { + if (min < axis.datamin && min != -fakeInfinity) + axis.datamin = min; + if (max > axis.datamax && max != fakeInfinity) + axis.datamax = max; + } + + $.each(allAxes(), function (_, axis) { + // init axis + axis.datamin = topSentry; + axis.datamax = bottomSentry; + axis.used = false; + }); + + for (i = 0; i < series.length; ++i) { + s = series[i]; + s.datapoints = { points: [] }; + + executeHooks(hooks.processRawData, [ s, s.data, s.datapoints ]); + } + + // first pass: clean and copy data + for (i = 0; i < series.length; ++i) { + s = series[i]; + + data = s.data; + format = s.datapoints.format; + + if (!format) { + format = []; + // find out how to copy + format.push({ x: true, number: true, required: true }); + format.push({ y: true, number: true, required: true }); + + if (s.bars.show || (s.lines.show && s.lines.fill)) { + var autoscale = !!((s.bars.show && s.bars.zero) || (s.lines.show && s.lines.zero)); + format.push({ y: true, number: true, required: false, defaultValue: 0, autoscale: autoscale }); + if (s.bars.horizontal) { + delete format[format.length - 1].y; + format[format.length - 1].x = true; + } + } + + s.datapoints.format = format; + } + + if (s.datapoints.pointsize != null) + continue; // already filled in + + s.datapoints.pointsize = format.length; + + ps = s.datapoints.pointsize; + points = s.datapoints.points; + + var insertSteps = s.lines.show && s.lines.steps; + s.xaxis.used = s.yaxis.used = true; + + for (j = k = 0; j < data.length; ++j, k += ps) { + p = data[j]; + + var nullify = p == null; + if (!nullify) { + for (m = 0; m < ps; ++m) { + val = p[m]; + f = format[m]; + + if (f) { + if (f.number && val != null) { + val = +val; // convert to number + if (isNaN(val)) + val = null; + else if (val == Infinity) + val = fakeInfinity; + else if (val == -Infinity) + val = -fakeInfinity; + } + + if (val == null) { + if (f.required) + nullify = true; + + if (f.defaultValue != null) + val = f.defaultValue; + } + } + + points[k + m] = val; + } + } + + if (nullify) { + for (m = 0; m < ps; ++m) { + val = points[k + m]; + if (val != null) { + f = format[m]; + // extract min/max info + if (f.autoscale !== false) { + if (f.x) { + updateAxis(s.xaxis, val, val); + } + if (f.y) { + updateAxis(s.yaxis, val, val); + } + } + } + points[k + m] = null; + } + } + else { + // a little bit of line specific stuff that + // perhaps shouldn't be here, but lacking + // better means... + if (insertSteps && k > 0 + && points[k - ps] != null + && points[k - ps] != points[k] + && points[k - ps + 1] != points[k + 1]) { + // copy the point to make room for a middle point + for (m = 0; m < ps; ++m) + points[k + ps + m] = points[k + m]; + + // middle point has same y + points[k + 1] = points[k - ps + 1]; + + // we've added a point, better reflect that + k += ps; + } + } + } + } + + // give the hooks a chance to run + for (i = 0; i < series.length; ++i) { + s = series[i]; + + executeHooks(hooks.processDatapoints, [ s, s.datapoints]); + } + + // second pass: find datamax/datamin for auto-scaling + for (i = 0; i < series.length; ++i) { + s = series[i]; + points = s.datapoints.points; + ps = s.datapoints.pointsize; + format = s.datapoints.format; + + var xmin = topSentry, ymin = topSentry, + xmax = bottomSentry, ymax = bottomSentry; + + for (j = 0; j < points.length; j += ps) { + if (points[j] == null) + continue; + + for (m = 0; m < ps; ++m) { + val = points[j + m]; + f = format[m]; + if (!f || f.autoscale === false || val == fakeInfinity || val == -fakeInfinity) + continue; + + if (f.x) { + if (val < xmin) + xmin = val; + if (val > xmax) + xmax = val; + } + if (f.y) { + if (val < ymin) + ymin = val; + if (val > ymax) + ymax = val; + } + } + } + + if (s.bars.show) { + // make sure we got room for the bar on the dancing floor + var delta; + + switch (s.bars.align) { + case "left": + delta = 0; + break; + case "right": + delta = -s.bars.barWidth; + break; + default: + delta = -s.bars.barWidth / 2; + } + + if (s.bars.horizontal) { + ymin += delta; + ymax += delta + s.bars.barWidth; + } + else { + xmin += delta; + xmax += delta + s.bars.barWidth; + } + } + + updateAxis(s.xaxis, xmin, xmax); + updateAxis(s.yaxis, ymin, ymax); + } + + $.each(allAxes(), function (_, axis) { + if (axis.datamin == topSentry) + axis.datamin = null; + if (axis.datamax == bottomSentry) + axis.datamax = null; + }); + } + + function setupCanvases() { + + // Make sure the placeholder is clear of everything except canvases + // from a previous plot in this container that we'll try to re-use. + + placeholder.css("padding", 0) // padding messes up the positioning + .children().filter(function(){ + return !$(this).hasClass("flot-overlay") && !$(this).hasClass('flot-base'); + }).remove(); + + if (placeholder.css("position") == 'static') + placeholder.css("position", "relative"); // for positioning labels and overlay + + surface = new Canvas("flot-base", placeholder); + overlay = new Canvas("flot-overlay", placeholder); // overlay canvas for interactive features + + ctx = surface.context; + octx = overlay.context; + + // define which element we're listening for events on + eventHolder = $(overlay.element).unbind(); + + // If we're re-using a plot object, shut down the old one + + var existing = placeholder.data("plot"); + + if (existing) { + existing.shutdown(); + overlay.clear(); + } + + // save in case we get replotted + placeholder.data("plot", plot); + } + + function bindEvents() { + // bind events + if (options.grid.hoverable) { + eventHolder.mousemove(onMouseMove); + + // Use bind, rather than .mouseleave, because we officially + // still support jQuery 1.2.6, which doesn't define a shortcut + // for mouseenter or mouseleave. This was a bug/oversight that + // was fixed somewhere around 1.3.x. We can return to using + // .mouseleave when we drop support for 1.2.6. + + eventHolder.bind("mouseleave", onMouseLeave); + } + + if (options.grid.clickable) + eventHolder.click(onClick); + + executeHooks(hooks.bindEvents, [eventHolder]); + } + + function shutdown() { + if (redrawTimeout) + clearTimeout(redrawTimeout); + + eventHolder.unbind("mousemove", onMouseMove); + eventHolder.unbind("mouseleave", onMouseLeave); + eventHolder.unbind("click", onClick); + + executeHooks(hooks.shutdown, [eventHolder]); + } + + function setTransformationHelpers(axis) { + // set helper functions on the axis, assumes plot area + // has been computed already + + function identity(x) { return x; } + + var s, m, t = axis.options.transform || identity, + it = axis.options.inverseTransform; + + // precompute how much the axis is scaling a point + // in canvas space + if (axis.direction == "x") { + s = axis.scale = plotWidth / Math.abs(t(axis.max) - t(axis.min)); + m = Math.min(t(axis.max), t(axis.min)); + } + else { + s = axis.scale = plotHeight / Math.abs(t(axis.max) - t(axis.min)); + s = -s; + m = Math.max(t(axis.max), t(axis.min)); + } + + // data point to canvas coordinate + if (t == identity) // slight optimization + axis.p2c = function (p) { return (p - m) * s; }; + else + axis.p2c = function (p) { return (t(p) - m) * s; }; + // canvas coordinate to data point + if (!it) + axis.c2p = function (c) { return m + c / s; }; + else + axis.c2p = function (c) { return it(m + c / s); }; + } + + function measureTickLabels(axis) { + + var opts = axis.options, + ticks = axis.ticks || [], + labelWidth = opts.labelWidth || 0, + labelHeight = opts.labelHeight || 0, + maxWidth = labelWidth || (axis.direction == "x" ? Math.floor(surface.width / (ticks.length || 1)) : null), + legacyStyles = axis.direction + "Axis " + axis.direction + axis.n + "Axis", + layer = "flot-" + axis.direction + "-axis flot-" + axis.direction + axis.n + "-axis " + legacyStyles, + font = opts.font || "flot-tick-label tickLabel"; + + for (var i = 0; i < ticks.length; ++i) { + + var t = ticks[i]; + + if (!t.label) + continue; + + var info = surface.getTextInfo(layer, t.label, font, null, maxWidth); + + labelWidth = Math.max(labelWidth, info.width); + labelHeight = Math.max(labelHeight, info.height); + } + + axis.labelWidth = opts.labelWidth || labelWidth; + axis.labelHeight = opts.labelHeight || labelHeight; + } + + function allocateAxisBoxFirstPhase(axis) { + // find the bounding box of the axis by looking at label + // widths/heights and ticks, make room by diminishing the + // plotOffset; this first phase only looks at one + // dimension per axis, the other dimension depends on the + // other axes so will have to wait + + var lw = axis.labelWidth, + lh = axis.labelHeight, + pos = axis.options.position, + isXAxis = axis.direction === "x", + tickLength = axis.options.tickLength, + axisMargin = options.grid.axisMargin, + padding = options.grid.labelMargin, + innermost = true, + outermost = true, + first = true, + found = false; + + // Determine the axis's position in its direction and on its side + + $.each(isXAxis ? xaxes : yaxes, function(i, a) { + if (a && (a.show || a.reserveSpace)) { + if (a === axis) { + found = true; + } else if (a.options.position === pos) { + if (found) { + outermost = false; + } else { + innermost = false; + } + } + if (!found) { + first = false; + } + } + }); + + // The outermost axis on each side has no margin + + if (outermost) { + axisMargin = 0; + } + + // The ticks for the first axis in each direction stretch across + + if (tickLength == null) { + tickLength = first ? "full" : 5; + } + + if (!isNaN(+tickLength)) + padding += +tickLength; + + if (isXAxis) { + lh += padding; + + if (pos == "bottom") { + plotOffset.bottom += lh + axisMargin; + axis.box = { top: surface.height - plotOffset.bottom, height: lh }; + } + else { + axis.box = { top: plotOffset.top + axisMargin, height: lh }; + plotOffset.top += lh + axisMargin; + } + } + else { + lw += padding; + + if (pos == "left") { + axis.box = { left: plotOffset.left + axisMargin, width: lw }; + plotOffset.left += lw + axisMargin; + } + else { + plotOffset.right += lw + axisMargin; + axis.box = { left: surface.width - plotOffset.right, width: lw }; + } + } + + // save for future reference + axis.position = pos; + axis.tickLength = tickLength; + axis.box.padding = padding; + axis.innermost = innermost; + } + + function allocateAxisBoxSecondPhase(axis) { + // now that all axis boxes have been placed in one + // dimension, we can set the remaining dimension coordinates + if (axis.direction == "x") { + axis.box.left = plotOffset.left - axis.labelWidth / 2; + axis.box.width = surface.width - plotOffset.left - plotOffset.right + axis.labelWidth; + } + else { + axis.box.top = plotOffset.top - axis.labelHeight / 2; + axis.box.height = surface.height - plotOffset.bottom - plotOffset.top + axis.labelHeight; + } + } + + function adjustLayoutForThingsStickingOut() { + // possibly adjust plot offset to ensure everything stays + // inside the canvas and isn't clipped off + + var minMargin = options.grid.minBorderMargin, + axis, i; + + // check stuff from the plot (FIXME: this should just read + // a value from the series, otherwise it's impossible to + // customize) + if (minMargin == null) { + minMargin = 0; + for (i = 0; i < series.length; ++i) + minMargin = Math.max(minMargin, 2 * (series[i].points.radius + series[i].points.lineWidth/2)); + } + + var margins = { + left: minMargin, + right: minMargin, + top: minMargin, + bottom: minMargin + }; + + // check axis labels, note we don't check the actual + // labels but instead use the overall width/height to not + // jump as much around with replots + $.each(allAxes(), function (_, axis) { + if (axis.reserveSpace && axis.ticks && axis.ticks.length) { + if (axis.direction === "x") { + margins.left = Math.max(margins.left, axis.labelWidth / 2); + margins.right = Math.max(margins.right, axis.labelWidth / 2); + } else { + margins.bottom = Math.max(margins.bottom, axis.labelHeight / 2); + margins.top = Math.max(margins.top, axis.labelHeight / 2); + } + } + }); + + plotOffset.left = Math.ceil(Math.max(margins.left, plotOffset.left)); + plotOffset.right = Math.ceil(Math.max(margins.right, plotOffset.right)); + plotOffset.top = Math.ceil(Math.max(margins.top, plotOffset.top)); + plotOffset.bottom = Math.ceil(Math.max(margins.bottom, plotOffset.bottom)); + } + + function setupGrid() { + var i, axes = allAxes(), showGrid = options.grid.show; + + // Initialize the plot's offset from the edge of the canvas + + for (var a in plotOffset) { + var margin = options.grid.margin || 0; + plotOffset[a] = typeof margin == "number" ? margin : margin[a] || 0; + } + + executeHooks(hooks.processOffset, [plotOffset]); + + // If the grid is visible, add its border width to the offset + + for (var a in plotOffset) { + if(typeof(options.grid.borderWidth) == "object") { + plotOffset[a] += showGrid ? options.grid.borderWidth[a] : 0; + } + else { + plotOffset[a] += showGrid ? options.grid.borderWidth : 0; + } + } + + $.each(axes, function (_, axis) { + var axisOpts = axis.options; + axis.show = axisOpts.show == null ? axis.used : axisOpts.show; + axis.reserveSpace = axisOpts.reserveSpace == null ? axis.show : axisOpts.reserveSpace; + setRange(axis); + }); + + if (showGrid) { + + var allocatedAxes = $.grep(axes, function (axis) { + return axis.show || axis.reserveSpace; + }); + + $.each(allocatedAxes, function (_, axis) { + // make the ticks + setupTickGeneration(axis); + setTicks(axis); + snapRangeToTicks(axis, axis.ticks); + // find labelWidth/Height for axis + measureTickLabels(axis); + }); + + // with all dimensions calculated, we can compute the + // axis bounding boxes, start from the outside + // (reverse order) + for (i = allocatedAxes.length - 1; i >= 0; --i) + allocateAxisBoxFirstPhase(allocatedAxes[i]); + + // make sure we've got enough space for things that + // might stick out + adjustLayoutForThingsStickingOut(); + + $.each(allocatedAxes, function (_, axis) { + allocateAxisBoxSecondPhase(axis); + }); + } + + plotWidth = surface.width - plotOffset.left - plotOffset.right; + plotHeight = surface.height - plotOffset.bottom - plotOffset.top; + + // now we got the proper plot dimensions, we can compute the scaling + $.each(axes, function (_, axis) { + setTransformationHelpers(axis); + }); + + if (showGrid) { + drawAxisLabels(); + } + + insertLegend(); + } + + function setRange(axis) { + var opts = axis.options, + min = +(opts.min != null ? opts.min : axis.datamin), + max = +(opts.max != null ? opts.max : axis.datamax), + delta = max - min; + + if (delta == 0.0) { + // degenerate case + var widen = max == 0 ? 1 : 0.01; + + if (opts.min == null) + min -= widen; + // always widen max if we couldn't widen min to ensure we + // don't fall into min == max which doesn't work + if (opts.max == null || opts.min != null) + max += widen; + } + else { + // consider autoscaling + var margin = opts.autoscaleMargin; + if (margin != null) { + if (opts.min == null) { + min -= delta * margin; + // make sure we don't go below zero if all values + // are positive + if (min < 0 && axis.datamin != null && axis.datamin >= 0) + min = 0; + } + if (opts.max == null) { + max += delta * margin; + if (max > 0 && axis.datamax != null && axis.datamax <= 0) + max = 0; + } + } + } + axis.min = min; + axis.max = max; + } + + function setupTickGeneration(axis) { + var opts = axis.options; + + // estimate number of ticks + var noTicks; + if (typeof opts.ticks == "number" && opts.ticks > 0) + noTicks = opts.ticks; + else + // heuristic based on the model a*sqrt(x) fitted to + // some data points that seemed reasonable + noTicks = 0.3 * Math.sqrt(axis.direction == "x" ? surface.width : surface.height); + + var delta = (axis.max - axis.min) / noTicks, + dec = -Math.floor(Math.log(delta) / Math.LN10), + maxDec = opts.tickDecimals; + + if (maxDec != null && dec > maxDec) { + dec = maxDec; + } + + var magn = Math.pow(10, -dec), + norm = delta / magn, // norm is between 1.0 and 10.0 + size; + + if (norm < 1.5) { + size = 1; + } else if (norm < 3) { + size = 2; + // special case for 2.5, requires an extra decimal + if (norm > 2.25 && (maxDec == null || dec + 1 <= maxDec)) { + size = 2.5; + ++dec; + } + } else if (norm < 7.5) { + size = 5; + } else { + size = 10; + } + + size *= magn; + + if (opts.minTickSize != null && size < opts.minTickSize) { + size = opts.minTickSize; + } + + axis.delta = delta; + axis.tickDecimals = Math.max(0, maxDec != null ? maxDec : dec); + axis.tickSize = opts.tickSize || size; + + // Time mode was moved to a plug-in in 0.8, and since so many people use it + // we'll add an especially friendly reminder to make sure they included it. + + if (opts.mode == "time" && !axis.tickGenerator) { + throw new Error("Time mode requires the flot.time plugin."); + } + + // Flot supports base-10 axes; any other mode else is handled by a plug-in, + // like flot.time.js. + + if (!axis.tickGenerator) { + + axis.tickGenerator = function (axis) { + + var ticks = [], + start = floorInBase(axis.min, axis.tickSize), + i = 0, + v = Number.NaN, + prev; + + do { + prev = v; + v = start + i * axis.tickSize; + ticks.push(v); + ++i; + } while (v < axis.max && v != prev); + return ticks; + }; + + axis.tickFormatter = function (value, axis) { + + var factor = axis.tickDecimals ? Math.pow(10, axis.tickDecimals) : 1; + var formatted = "" + Math.round(value * factor) / factor; + + // If tickDecimals was specified, ensure that we have exactly that + // much precision; otherwise default to the value's own precision. + + if (axis.tickDecimals != null) { + var decimal = formatted.indexOf("."); + var precision = decimal == -1 ? 0 : formatted.length - decimal - 1; + if (precision < axis.tickDecimals) { + return (precision ? formatted : formatted + ".") + ("" + factor).substr(1, axis.tickDecimals - precision); + } + } + + return formatted; + }; + } + + if ($.isFunction(opts.tickFormatter)) + axis.tickFormatter = function (v, axis) { return "" + opts.tickFormatter(v, axis); }; + + if (opts.alignTicksWithAxis != null) { + var otherAxis = (axis.direction == "x" ? xaxes : yaxes)[opts.alignTicksWithAxis - 1]; + if (otherAxis && otherAxis.used && otherAxis != axis) { + // consider snapping min/max to outermost nice ticks + var niceTicks = axis.tickGenerator(axis); + if (niceTicks.length > 0) { + if (opts.min == null) + axis.min = Math.min(axis.min, niceTicks[0]); + if (opts.max == null && niceTicks.length > 1) + axis.max = Math.max(axis.max, niceTicks[niceTicks.length - 1]); + } + + axis.tickGenerator = function (axis) { + // copy ticks, scaled to this axis + var ticks = [], v, i; + for (i = 0; i < otherAxis.ticks.length; ++i) { + v = (otherAxis.ticks[i].v - otherAxis.min) / (otherAxis.max - otherAxis.min); + v = axis.min + v * (axis.max - axis.min); + ticks.push(v); + } + return ticks; + }; + + // we might need an extra decimal since forced + // ticks don't necessarily fit naturally + if (!axis.mode && opts.tickDecimals == null) { + var extraDec = Math.max(0, -Math.floor(Math.log(axis.delta) / Math.LN10) + 1), + ts = axis.tickGenerator(axis); + + // only proceed if the tick interval rounded + // with an extra decimal doesn't give us a + // zero at end + if (!(ts.length > 1 && /\..*0$/.test((ts[1] - ts[0]).toFixed(extraDec)))) + axis.tickDecimals = extraDec; + } + } + } + } + + function setTicks(axis) { + var oticks = axis.options.ticks, ticks = []; + if (oticks == null || (typeof oticks == "number" && oticks > 0)) + ticks = axis.tickGenerator(axis); + else if (oticks) { + if ($.isFunction(oticks)) + // generate the ticks + ticks = oticks(axis); + else + ticks = oticks; + } + + // clean up/labelify the supplied ticks, copy them over + var i, v; + axis.ticks = []; + for (i = 0; i < ticks.length; ++i) { + var label = null; + var t = ticks[i]; + if (typeof t == "object") { + v = +t[0]; + if (t.length > 1) + label = t[1]; + } + else + v = +t; + if (label == null) + label = axis.tickFormatter(v, axis); + if (!isNaN(v)) + axis.ticks.push({ v: v, label: label }); + } + } + + function snapRangeToTicks(axis, ticks) { + if (axis.options.autoscaleMargin && ticks.length > 0) { + // snap to ticks + if (axis.options.min == null) + axis.min = Math.min(axis.min, ticks[0].v); + if (axis.options.max == null && ticks.length > 1) + axis.max = Math.max(axis.max, ticks[ticks.length - 1].v); + } + } + + function draw() { + + surface.clear(); + + executeHooks(hooks.drawBackground, [ctx]); + + var grid = options.grid; + + // draw background, if any + if (grid.show && grid.backgroundColor) + drawBackground(); + + if (grid.show && !grid.aboveData) { + drawGrid(); + } + + for (var i = 0; i < series.length; ++i) { + executeHooks(hooks.drawSeries, [ctx, series[i]]); + drawSeries(series[i]); + } + + executeHooks(hooks.draw, [ctx]); + + if (grid.show && grid.aboveData) { + drawGrid(); + } + + surface.render(); + + // A draw implies that either the axes or data have changed, so we + // should probably update the overlay highlights as well. + + triggerRedrawOverlay(); + } + + function extractRange(ranges, coord) { + var axis, from, to, key, axes = allAxes(); + + for (var i = 0; i < axes.length; ++i) { + axis = axes[i]; + if (axis.direction == coord) { + key = coord + axis.n + "axis"; + if (!ranges[key] && axis.n == 1) + key = coord + "axis"; // support x1axis as xaxis + if (ranges[key]) { + from = ranges[key].from; + to = ranges[key].to; + break; + } + } + } + + // backwards-compat stuff - to be removed in future + if (!ranges[key]) { + axis = coord == "x" ? xaxes[0] : yaxes[0]; + from = ranges[coord + "1"]; + to = ranges[coord + "2"]; + } + + // auto-reverse as an added bonus + if (from != null && to != null && from > to) { + var tmp = from; + from = to; + to = tmp; + } + + return { from: from, to: to, axis: axis }; + } + + function drawBackground() { + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + + ctx.fillStyle = getColorOrGradient(options.grid.backgroundColor, plotHeight, 0, "rgba(255, 255, 255, 0)"); + ctx.fillRect(0, 0, plotWidth, plotHeight); + ctx.restore(); + } + + function drawGrid() { + var i, axes, bw, bc; + + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + + // draw markings + var markings = options.grid.markings; + if (markings) { + if ($.isFunction(markings)) { + axes = plot.getAxes(); + // xmin etc. is backwards compatibility, to be + // removed in the future + axes.xmin = axes.xaxis.min; + axes.xmax = axes.xaxis.max; + axes.ymin = axes.yaxis.min; + axes.ymax = axes.yaxis.max; + + markings = markings(axes); + } + + for (i = 0; i < markings.length; ++i) { + var m = markings[i], + xrange = extractRange(m, "x"), + yrange = extractRange(m, "y"); + + // fill in missing + if (xrange.from == null) + xrange.from = xrange.axis.min; + if (xrange.to == null) + xrange.to = xrange.axis.max; + if (yrange.from == null) + yrange.from = yrange.axis.min; + if (yrange.to == null) + yrange.to = yrange.axis.max; + + // clip + if (xrange.to < xrange.axis.min || xrange.from > xrange.axis.max || + yrange.to < yrange.axis.min || yrange.from > yrange.axis.max) + continue; + + xrange.from = Math.max(xrange.from, xrange.axis.min); + xrange.to = Math.min(xrange.to, xrange.axis.max); + yrange.from = Math.max(yrange.from, yrange.axis.min); + yrange.to = Math.min(yrange.to, yrange.axis.max); + + var xequal = xrange.from === xrange.to, + yequal = yrange.from === yrange.to; + + if (xequal && yequal) { + continue; + } + + // then draw + xrange.from = Math.floor(xrange.axis.p2c(xrange.from)); + xrange.to = Math.floor(xrange.axis.p2c(xrange.to)); + yrange.from = Math.floor(yrange.axis.p2c(yrange.from)); + yrange.to = Math.floor(yrange.axis.p2c(yrange.to)); + + if (xequal || yequal) { + var lineWidth = m.lineWidth || options.grid.markingsLineWidth, + subPixel = lineWidth % 2 ? 0.5 : 0; + ctx.beginPath(); + ctx.strokeStyle = m.color || options.grid.markingsColor; + ctx.lineWidth = lineWidth; + if (xequal) { + ctx.moveTo(xrange.to + subPixel, yrange.from); + ctx.lineTo(xrange.to + subPixel, yrange.to); + } else { + ctx.moveTo(xrange.from, yrange.to + subPixel); + ctx.lineTo(xrange.to, yrange.to + subPixel); + } + ctx.stroke(); + } else { + ctx.fillStyle = m.color || options.grid.markingsColor; + ctx.fillRect(xrange.from, yrange.to, + xrange.to - xrange.from, + yrange.from - yrange.to); + } + } + } + + // draw the ticks + axes = allAxes(); + bw = options.grid.borderWidth; + + for (var j = 0; j < axes.length; ++j) { + var axis = axes[j], box = axis.box, + t = axis.tickLength, x, y, xoff, yoff; + if (!axis.show || axis.ticks.length == 0) + continue; + + ctx.lineWidth = 1; + + // find the edges + if (axis.direction == "x") { + x = 0; + if (t == "full") + y = (axis.position == "top" ? 0 : plotHeight); + else + y = box.top - plotOffset.top + (axis.position == "top" ? box.height : 0); + } + else { + y = 0; + if (t == "full") + x = (axis.position == "left" ? 0 : plotWidth); + else + x = box.left - plotOffset.left + (axis.position == "left" ? box.width : 0); + } + + // draw tick bar + if (!axis.innermost) { + ctx.strokeStyle = axis.options.color; + ctx.beginPath(); + xoff = yoff = 0; + if (axis.direction == "x") + xoff = plotWidth + 1; + else + yoff = plotHeight + 1; + + if (ctx.lineWidth == 1) { + if (axis.direction == "x") { + y = Math.floor(y) + 0.5; + } else { + x = Math.floor(x) + 0.5; + } + } + + ctx.moveTo(x, y); + ctx.lineTo(x + xoff, y + yoff); + ctx.stroke(); + } + + // draw ticks + + ctx.strokeStyle = axis.options.tickColor; + + ctx.beginPath(); + for (i = 0; i < axis.ticks.length; ++i) { + var v = axis.ticks[i].v; + + xoff = yoff = 0; + + if (isNaN(v) || v < axis.min || v > axis.max + // skip those lying on the axes if we got a border + || (t == "full" + && ((typeof bw == "object" && bw[axis.position] > 0) || bw > 0) + && (v == axis.min || v == axis.max))) + continue; + + if (axis.direction == "x") { + x = axis.p2c(v); + yoff = t == "full" ? -plotHeight : t; + + if (axis.position == "top") + yoff = -yoff; + } + else { + y = axis.p2c(v); + xoff = t == "full" ? -plotWidth : t; + + if (axis.position == "left") + xoff = -xoff; + } + + if (ctx.lineWidth == 1) { + if (axis.direction == "x") + x = Math.floor(x) + 0.5; + else + y = Math.floor(y) + 0.5; + } + + ctx.moveTo(x, y); + ctx.lineTo(x + xoff, y + yoff); + } + + ctx.stroke(); + } + + + // draw border + if (bw) { + // If either borderWidth or borderColor is an object, then draw the border + // line by line instead of as one rectangle + bc = options.grid.borderColor; + if(typeof bw == "object" || typeof bc == "object") { + if (typeof bw !== "object") { + bw = {top: bw, right: bw, bottom: bw, left: bw}; + } + if (typeof bc !== "object") { + bc = {top: bc, right: bc, bottom: bc, left: bc}; + } + + if (bw.top > 0) { + ctx.strokeStyle = bc.top; + ctx.lineWidth = bw.top; + ctx.beginPath(); + ctx.moveTo(0 - bw.left, 0 - bw.top/2); + ctx.lineTo(plotWidth, 0 - bw.top/2); + ctx.stroke(); + } + + if (bw.right > 0) { + ctx.strokeStyle = bc.right; + ctx.lineWidth = bw.right; + ctx.beginPath(); + ctx.moveTo(plotWidth + bw.right / 2, 0 - bw.top); + ctx.lineTo(plotWidth + bw.right / 2, plotHeight); + ctx.stroke(); + } + + if (bw.bottom > 0) { + ctx.strokeStyle = bc.bottom; + ctx.lineWidth = bw.bottom; + ctx.beginPath(); + ctx.moveTo(plotWidth + bw.right, plotHeight + bw.bottom / 2); + ctx.lineTo(0, plotHeight + bw.bottom / 2); + ctx.stroke(); + } + + if (bw.left > 0) { + ctx.strokeStyle = bc.left; + ctx.lineWidth = bw.left; + ctx.beginPath(); + ctx.moveTo(0 - bw.left/2, plotHeight + bw.bottom); + ctx.lineTo(0- bw.left/2, 0); + ctx.stroke(); + } + } + else { + ctx.lineWidth = bw; + ctx.strokeStyle = options.grid.borderColor; + ctx.strokeRect(-bw/2, -bw/2, plotWidth + bw, plotHeight + bw); + } + } + + ctx.restore(); + } + + function drawAxisLabels() { + + $.each(allAxes(), function (_, axis) { + var box = axis.box, + legacyStyles = axis.direction + "Axis " + axis.direction + axis.n + "Axis", + layer = "flot-" + axis.direction + "-axis flot-" + axis.direction + axis.n + "-axis " + legacyStyles, + font = axis.options.font || "flot-tick-label tickLabel", + tick, x, y, halign, valign; + + // Remove text before checking for axis.show and ticks.length; + // otherwise plugins, like flot-tickrotor, that draw their own + // tick labels will end up with both theirs and the defaults. + + surface.removeText(layer); + + if (!axis.show || axis.ticks.length == 0) + return; + + for (var i = 0; i < axis.ticks.length; ++i) { + + tick = axis.ticks[i]; + if (!tick.label || tick.v < axis.min || tick.v > axis.max) + continue; + + if (axis.direction == "x") { + halign = "center"; + x = plotOffset.left + axis.p2c(tick.v); + if (axis.position == "bottom") { + y = box.top + box.padding; + } else { + y = box.top + box.height - box.padding; + valign = "bottom"; + } + } else { + valign = "middle"; + y = plotOffset.top + axis.p2c(tick.v); + if (axis.position == "left") { + x = box.left + box.width - box.padding; + halign = "right"; + } else { + x = box.left + box.padding; + } + } + + surface.addText(layer, x, y, tick.label, font, null, null, halign, valign); + } + }); + } + + function drawSeries(series) { + if (series.lines.show) + drawSeriesLines(series); + if (series.bars.show) + drawSeriesBars(series); + if (series.points.show) + drawSeriesPoints(series); + } + + function drawSeriesLines(series) { + function plotLine(datapoints, xoffset, yoffset, axisx, axisy) { + var points = datapoints.points, + ps = datapoints.pointsize, + prevx = null, prevy = null; + + ctx.beginPath(); + for (var i = ps; i < points.length; i += ps) { + var x1 = points[i - ps], y1 = points[i - ps + 1], + x2 = points[i], y2 = points[i + 1]; + + if (x1 == null || x2 == null) + continue; + + // clip with ymin + if (y1 <= y2 && y1 < axisy.min) { + if (y2 < axisy.min) + continue; // line segment is outside + // compute new intersection point + x1 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1; + y1 = axisy.min; + } + else if (y2 <= y1 && y2 < axisy.min) { + if (y1 < axisy.min) + continue; + x2 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1; + y2 = axisy.min; + } + + // clip with ymax + if (y1 >= y2 && y1 > axisy.max) { + if (y2 > axisy.max) + continue; + x1 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1; + y1 = axisy.max; + } + else if (y2 >= y1 && y2 > axisy.max) { + if (y1 > axisy.max) + continue; + x2 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1; + y2 = axisy.max; + } + + // clip with xmin + if (x1 <= x2 && x1 < axisx.min) { + if (x2 < axisx.min) + continue; + y1 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1; + x1 = axisx.min; + } + else if (x2 <= x1 && x2 < axisx.min) { + if (x1 < axisx.min) + continue; + y2 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1; + x2 = axisx.min; + } + + // clip with xmax + if (x1 >= x2 && x1 > axisx.max) { + if (x2 > axisx.max) + continue; + y1 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1; + x1 = axisx.max; + } + else if (x2 >= x1 && x2 > axisx.max) { + if (x1 > axisx.max) + continue; + y2 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1; + x2 = axisx.max; + } + + if (x1 != prevx || y1 != prevy) + ctx.moveTo(axisx.p2c(x1) + xoffset, axisy.p2c(y1) + yoffset); + + prevx = x2; + prevy = y2; + ctx.lineTo(axisx.p2c(x2) + xoffset, axisy.p2c(y2) + yoffset); + } + ctx.stroke(); + } + + function plotLineArea(datapoints, axisx, axisy) { + var points = datapoints.points, + ps = datapoints.pointsize, + bottom = Math.min(Math.max(0, axisy.min), axisy.max), + i = 0, top, areaOpen = false, + ypos = 1, segmentStart = 0, segmentEnd = 0; + + // we process each segment in two turns, first forward + // direction to sketch out top, then once we hit the + // end we go backwards to sketch the bottom + while (true) { + if (ps > 0 && i > points.length + ps) + break; + + i += ps; // ps is negative if going backwards + + var x1 = points[i - ps], + y1 = points[i - ps + ypos], + x2 = points[i], y2 = points[i + ypos]; + + if (areaOpen) { + if (ps > 0 && x1 != null && x2 == null) { + // at turning point + segmentEnd = i; + ps = -ps; + ypos = 2; + continue; + } + + if (ps < 0 && i == segmentStart + ps) { + // done with the reverse sweep + ctx.fill(); + areaOpen = false; + ps = -ps; + ypos = 1; + i = segmentStart = segmentEnd + ps; + continue; + } + } + + if (x1 == null || x2 == null) + continue; + + // clip x values + + // clip with xmin + if (x1 <= x2 && x1 < axisx.min) { + if (x2 < axisx.min) + continue; + y1 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1; + x1 = axisx.min; + } + else if (x2 <= x1 && x2 < axisx.min) { + if (x1 < axisx.min) + continue; + y2 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1; + x2 = axisx.min; + } + + // clip with xmax + if (x1 >= x2 && x1 > axisx.max) { + if (x2 > axisx.max) + continue; + y1 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1; + x1 = axisx.max; + } + else if (x2 >= x1 && x2 > axisx.max) { + if (x1 > axisx.max) + continue; + y2 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1; + x2 = axisx.max; + } + + if (!areaOpen) { + // open area + ctx.beginPath(); + ctx.moveTo(axisx.p2c(x1), axisy.p2c(bottom)); + areaOpen = true; + } + + // now first check the case where both is outside + if (y1 >= axisy.max && y2 >= axisy.max) { + ctx.lineTo(axisx.p2c(x1), axisy.p2c(axisy.max)); + ctx.lineTo(axisx.p2c(x2), axisy.p2c(axisy.max)); + continue; + } + else if (y1 <= axisy.min && y2 <= axisy.min) { + ctx.lineTo(axisx.p2c(x1), axisy.p2c(axisy.min)); + ctx.lineTo(axisx.p2c(x2), axisy.p2c(axisy.min)); + continue; + } + + // else it's a bit more complicated, there might + // be a flat maxed out rectangle first, then a + // triangular cutout or reverse; to find these + // keep track of the current x values + var x1old = x1, x2old = x2; + + // clip the y values, without shortcutting, we + // go through all cases in turn + + // clip with ymin + if (y1 <= y2 && y1 < axisy.min && y2 >= axisy.min) { + x1 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1; + y1 = axisy.min; + } + else if (y2 <= y1 && y2 < axisy.min && y1 >= axisy.min) { + x2 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1; + y2 = axisy.min; + } + + // clip with ymax + if (y1 >= y2 && y1 > axisy.max && y2 <= axisy.max) { + x1 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1; + y1 = axisy.max; + } + else if (y2 >= y1 && y2 > axisy.max && y1 <= axisy.max) { + x2 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1; + y2 = axisy.max; + } + + // if the x value was changed we got a rectangle + // to fill + if (x1 != x1old) { + ctx.lineTo(axisx.p2c(x1old), axisy.p2c(y1)); + // it goes to (x1, y1), but we fill that below + } + + // fill triangular section, this sometimes result + // in redundant points if (x1, y1) hasn't changed + // from previous line to, but we just ignore that + ctx.lineTo(axisx.p2c(x1), axisy.p2c(y1)); + ctx.lineTo(axisx.p2c(x2), axisy.p2c(y2)); + + // fill the other rectangle if it's there + if (x2 != x2old) { + ctx.lineTo(axisx.p2c(x2), axisy.p2c(y2)); + ctx.lineTo(axisx.p2c(x2old), axisy.p2c(y2)); + } + } + } + + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + ctx.lineJoin = "round"; + + var lw = series.lines.lineWidth, + sw = series.shadowSize; + // FIXME: consider another form of shadow when filling is turned on + if (lw > 0 && sw > 0) { + // draw shadow as a thick and thin line with transparency + ctx.lineWidth = sw; + ctx.strokeStyle = "rgba(0,0,0,0.1)"; + // position shadow at angle from the mid of line + var angle = Math.PI/18; + plotLine(series.datapoints, Math.sin(angle) * (lw/2 + sw/2), Math.cos(angle) * (lw/2 + sw/2), series.xaxis, series.yaxis); + ctx.lineWidth = sw/2; + plotLine(series.datapoints, Math.sin(angle) * (lw/2 + sw/4), Math.cos(angle) * (lw/2 + sw/4), series.xaxis, series.yaxis); + } + + ctx.lineWidth = lw; + ctx.strokeStyle = series.color; + var fillStyle = getFillStyle(series.lines, series.color, 0, plotHeight); + if (fillStyle) { + ctx.fillStyle = fillStyle; + plotLineArea(series.datapoints, series.xaxis, series.yaxis); + } + + if (lw > 0) + plotLine(series.datapoints, 0, 0, series.xaxis, series.yaxis); + ctx.restore(); + } + + function drawSeriesPoints(series) { + function plotPoints(datapoints, radius, fillStyle, offset, shadow, axisx, axisy, symbol) { + var points = datapoints.points, ps = datapoints.pointsize; + + for (var i = 0; i < points.length; i += ps) { + var x = points[i], y = points[i + 1]; + if (x == null || x < axisx.min || x > axisx.max || y < axisy.min || y > axisy.max) + continue; + + ctx.beginPath(); + x = axisx.p2c(x); + y = axisy.p2c(y) + offset; + if (symbol == "circle") + ctx.arc(x, y, radius, 0, shadow ? Math.PI : Math.PI * 2, false); + else + symbol(ctx, x, y, radius, shadow); + ctx.closePath(); + + if (fillStyle) { + ctx.fillStyle = fillStyle; + ctx.fill(); + } + ctx.stroke(); + } + } + + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + + var lw = series.points.lineWidth, + sw = series.shadowSize, + radius = series.points.radius, + symbol = series.points.symbol; + + // If the user sets the line width to 0, we change it to a very + // small value. A line width of 0 seems to force the default of 1. + // Doing the conditional here allows the shadow setting to still be + // optional even with a lineWidth of 0. + + if( lw == 0 ) + lw = 0.0001; + + if (lw > 0 && sw > 0) { + // draw shadow in two steps + var w = sw / 2; + ctx.lineWidth = w; + ctx.strokeStyle = "rgba(0,0,0,0.1)"; + plotPoints(series.datapoints, radius, null, w + w/2, true, + series.xaxis, series.yaxis, symbol); + + ctx.strokeStyle = "rgba(0,0,0,0.2)"; + plotPoints(series.datapoints, radius, null, w/2, true, + series.xaxis, series.yaxis, symbol); + } + + ctx.lineWidth = lw; + ctx.strokeStyle = series.color; + plotPoints(series.datapoints, radius, + getFillStyle(series.points, series.color), 0, false, + series.xaxis, series.yaxis, symbol); + ctx.restore(); + } + + function drawBar(x, y, b, barLeft, barRight, fillStyleCallback, axisx, axisy, c, horizontal, lineWidth) { + var left, right, bottom, top, + drawLeft, drawRight, drawTop, drawBottom, + tmp; + + // in horizontal mode, we start the bar from the left + // instead of from the bottom so it appears to be + // horizontal rather than vertical + if (horizontal) { + drawBottom = drawRight = drawTop = true; + drawLeft = false; + left = b; + right = x; + top = y + barLeft; + bottom = y + barRight; + + // account for negative bars + if (right < left) { + tmp = right; + right = left; + left = tmp; + drawLeft = true; + drawRight = false; + } + } + else { + drawLeft = drawRight = drawTop = true; + drawBottom = false; + left = x + barLeft; + right = x + barRight; + bottom = b; + top = y; + + // account for negative bars + if (top < bottom) { + tmp = top; + top = bottom; + bottom = tmp; + drawBottom = true; + drawTop = false; + } + } + + // clip + if (right < axisx.min || left > axisx.max || + top < axisy.min || bottom > axisy.max) + return; + + if (left < axisx.min) { + left = axisx.min; + drawLeft = false; + } + + if (right > axisx.max) { + right = axisx.max; + drawRight = false; + } + + if (bottom < axisy.min) { + bottom = axisy.min; + drawBottom = false; + } + + if (top > axisy.max) { + top = axisy.max; + drawTop = false; + } + + left = axisx.p2c(left); + bottom = axisy.p2c(bottom); + right = axisx.p2c(right); + top = axisy.p2c(top); + + // fill the bar + if (fillStyleCallback) { + c.fillStyle = fillStyleCallback(bottom, top); + c.fillRect(left, top, right - left, bottom - top) + } + + // draw outline + if (lineWidth > 0 && (drawLeft || drawRight || drawTop || drawBottom)) { + c.beginPath(); + + // FIXME: inline moveTo is buggy with excanvas + c.moveTo(left, bottom); + if (drawLeft) + c.lineTo(left, top); + else + c.moveTo(left, top); + if (drawTop) + c.lineTo(right, top); + else + c.moveTo(right, top); + if (drawRight) + c.lineTo(right, bottom); + else + c.moveTo(right, bottom); + if (drawBottom) + c.lineTo(left, bottom); + else + c.moveTo(left, bottom); + c.stroke(); + } + } + + function drawSeriesBars(series) { + function plotBars(datapoints, barLeft, barRight, fillStyleCallback, axisx, axisy) { + var points = datapoints.points, ps = datapoints.pointsize; + + for (var i = 0; i < points.length; i += ps) { + if (points[i] == null) + continue; + drawBar(points[i], points[i + 1], points[i + 2], barLeft, barRight, fillStyleCallback, axisx, axisy, ctx, series.bars.horizontal, series.bars.lineWidth); + } + } + + ctx.save(); + ctx.translate(plotOffset.left, plotOffset.top); + + // FIXME: figure out a way to add shadows (for instance along the right edge) + ctx.lineWidth = series.bars.lineWidth; + ctx.strokeStyle = series.color; + + var barLeft; + + switch (series.bars.align) { + case "left": + barLeft = 0; + break; + case "right": + barLeft = -series.bars.barWidth; + break; + default: + barLeft = -series.bars.barWidth / 2; + } + + var fillStyleCallback = series.bars.fill ? function (bottom, top) { return getFillStyle(series.bars, series.color, bottom, top); } : null; + plotBars(series.datapoints, barLeft, barLeft + series.bars.barWidth, fillStyleCallback, series.xaxis, series.yaxis); + ctx.restore(); + } + + function getFillStyle(filloptions, seriesColor, bottom, top) { + var fill = filloptions.fill; + if (!fill) + return null; + + if (filloptions.fillColor) + return getColorOrGradient(filloptions.fillColor, bottom, top, seriesColor); + + var c = $.color.parse(seriesColor); + c.a = typeof fill == "number" ? fill : 0.4; + c.normalize(); + return c.toString(); + } + + function insertLegend() { + + if (options.legend.container != null) { + $(options.legend.container).html(""); + } else { + placeholder.find(".legend").remove(); + } + + if (!options.legend.show) { + return; + } + + var fragments = [], entries = [], rowStarted = false, + lf = options.legend.labelFormatter, s, label; + + // Build a list of legend entries, with each having a label and a color + + for (var i = 0; i < series.length; ++i) { + s = series[i]; + if (s.label) { + label = lf ? lf(s.label, s) : s.label; + if (label) { + entries.push({ + label: label, + color: s.color + }); + } + } + } + + // Sort the legend using either the default or a custom comparator + + if (options.legend.sorted) { + if ($.isFunction(options.legend.sorted)) { + entries.sort(options.legend.sorted); + } else if (options.legend.sorted == "reverse") { + entries.reverse(); + } else { + var ascending = options.legend.sorted != "descending"; + entries.sort(function(a, b) { + return a.label == b.label ? 0 : ( + (a.label < b.label) != ascending ? 1 : -1 // Logical XOR + ); + }); + } + } + + // Generate markup for the list of entries, in their final order + + for (var i = 0; i < entries.length; ++i) { + + var entry = entries[i]; + + if (i % options.legend.noColumns == 0) { + if (rowStarted) + fragments.push('</tr>'); + fragments.push('<tr>'); + rowStarted = true; + } + + fragments.push( + '<td class="legendColorBox"><div style="border:1px solid ' + options.legend.labelBoxBorderColor + ';padding:1px"><div style="width:4px;height:0;border:5px solid ' + entry.color + ';overflow:hidden"></div></div></td>' + + '<td class="legendLabel">' + entry.label + '</td>' + ); + } + + if (rowStarted) + fragments.push('</tr>'); + + if (fragments.length == 0) + return; + + var table = '<table style="font-size:smaller;color:' + options.grid.color + '">' + fragments.join("") + '</table>'; + if (options.legend.container != null) + $(options.legend.container).html(table); + else { + var pos = "", + p = options.legend.position, + m = options.legend.margin; + if (m[0] == null) + m = [m, m]; + if (p.charAt(0) == "n") + pos += 'top:' + (m[1] + plotOffset.top) + 'px;'; + else if (p.charAt(0) == "s") + pos += 'bottom:' + (m[1] + plotOffset.bottom) + 'px;'; + if (p.charAt(1) == "e") + pos += 'right:' + (m[0] + plotOffset.right) + 'px;'; + else if (p.charAt(1) == "w") + pos += 'left:' + (m[0] + plotOffset.left) + 'px;'; + var legend = $('<div class="legend">' + table.replace('style="', 'style="position:absolute;' + pos +';') + '</div>').appendTo(placeholder); + if (options.legend.backgroundOpacity != 0.0) { + // put in the transparent background + // separately to avoid blended labels and + // label boxes + var c = options.legend.backgroundColor; + if (c == null) { + c = options.grid.backgroundColor; + if (c && typeof c == "string") + c = $.color.parse(c); + else + c = $.color.extract(legend, 'background-color'); + c.a = 1; + c = c.toString(); + } + var div = legend.children(); + $('<div style="position:absolute;width:' + div.width() + 'px;height:' + div.height() + 'px;' + pos +'background-color:' + c + ';"> </div>').prependTo(legend).css('opacity', options.legend.backgroundOpacity); + } + } + } + + + // interactive features + + var highlights = [], + redrawTimeout = null; + + // returns the data item the mouse is over, or null if none is found + function findNearbyItem(mouseX, mouseY, seriesFilter) { + var maxDistance = options.grid.mouseActiveRadius, + smallestDistance = maxDistance * maxDistance + 1, + item = null, foundPoint = false, i, j, ps; + + for (i = series.length - 1; i >= 0; --i) { + if (!seriesFilter(series[i])) + continue; + + var s = series[i], + axisx = s.xaxis, + axisy = s.yaxis, + points = s.datapoints.points, + mx = axisx.c2p(mouseX), // precompute some stuff to make the loop faster + my = axisy.c2p(mouseY), + maxx = maxDistance / axisx.scale, + maxy = maxDistance / axisy.scale; + + ps = s.datapoints.pointsize; + // with inverse transforms, we can't use the maxx/maxy + // optimization, sadly + if (axisx.options.inverseTransform) + maxx = Number.MAX_VALUE; + if (axisy.options.inverseTransform) + maxy = Number.MAX_VALUE; + + if (s.lines.show || s.points.show) { + for (j = 0; j < points.length; j += ps) { + var x = points[j], y = points[j + 1]; + if (x == null) + continue; + + // For points and lines, the cursor must be within a + // certain distance to the data point + if (x - mx > maxx || x - mx < -maxx || + y - my > maxy || y - my < -maxy) + continue; + + // We have to calculate distances in pixels, not in + // data units, because the scales of the axes may be different + var dx = Math.abs(axisx.p2c(x) - mouseX), + dy = Math.abs(axisy.p2c(y) - mouseY), + dist = dx * dx + dy * dy; // we save the sqrt + + // use <= to ensure last point takes precedence + // (last generally means on top of) + if (dist < smallestDistance) { + smallestDistance = dist; + item = [i, j / ps]; + } + } + } + + if (s.bars.show && !item) { // no other point can be nearby + + var barLeft, barRight; + + switch (s.bars.align) { + case "left": + barLeft = 0; + break; + case "right": + barLeft = -s.bars.barWidth; + break; + default: + barLeft = -s.bars.barWidth / 2; + } + + barRight = barLeft + s.bars.barWidth; + + for (j = 0; j < points.length; j += ps) { + var x = points[j], y = points[j + 1], b = points[j + 2]; + if (x == null) + continue; + + // for a bar graph, the cursor must be inside the bar + if (series[i].bars.horizontal ? + (mx <= Math.max(b, x) && mx >= Math.min(b, x) && + my >= y + barLeft && my <= y + barRight) : + (mx >= x + barLeft && mx <= x + barRight && + my >= Math.min(b, y) && my <= Math.max(b, y))) + item = [i, j / ps]; + } + } + } + + if (item) { + i = item[0]; + j = item[1]; + ps = series[i].datapoints.pointsize; + + return { datapoint: series[i].datapoints.points.slice(j * ps, (j + 1) * ps), + dataIndex: j, + series: series[i], + seriesIndex: i }; + } + + return null; + } + + function onMouseMove(e) { + if (options.grid.hoverable) + triggerClickHoverEvent("plothover", e, + function (s) { return s["hoverable"] != false; }); + } + + function onMouseLeave(e) { + if (options.grid.hoverable) + triggerClickHoverEvent("plothover", e, + function (s) { return false; }); + } + + function onClick(e) { + triggerClickHoverEvent("plotclick", e, + function (s) { return s["clickable"] != false; }); + } + + // trigger click or hover event (they send the same parameters + // so we share their code) + function triggerClickHoverEvent(eventname, event, seriesFilter) { + var offset = eventHolder.offset(), + canvasX = event.pageX - offset.left - plotOffset.left, + canvasY = event.pageY - offset.top - plotOffset.top, + pos = canvasToAxisCoords({ left: canvasX, top: canvasY }); + + pos.pageX = event.pageX; + pos.pageY = event.pageY; + + var item = findNearbyItem(canvasX, canvasY, seriesFilter); + + if (item) { + // fill in mouse pos for any listeners out there + item.pageX = parseInt(item.series.xaxis.p2c(item.datapoint[0]) + offset.left + plotOffset.left, 10); + item.pageY = parseInt(item.series.yaxis.p2c(item.datapoint[1]) + offset.top + plotOffset.top, 10); + } + + if (options.grid.autoHighlight) { + // clear auto-highlights + for (var i = 0; i < highlights.length; ++i) { + var h = highlights[i]; + if (h.auto == eventname && + !(item && h.series == item.series && + h.point[0] == item.datapoint[0] && + h.point[1] == item.datapoint[1])) + unhighlight(h.series, h.point); + } + + if (item) + highlight(item.series, item.datapoint, eventname); + } + + placeholder.trigger(eventname, [ pos, item ]); + } + + function triggerRedrawOverlay() { + var t = options.interaction.redrawOverlayInterval; + if (t == -1) { // skip event queue + drawOverlay(); + return; + } + + if (!redrawTimeout) + redrawTimeout = setTimeout(drawOverlay, t); + } + + function drawOverlay() { + redrawTimeout = null; + + // draw highlights + octx.save(); + overlay.clear(); + octx.translate(plotOffset.left, plotOffset.top); + + var i, hi; + for (i = 0; i < highlights.length; ++i) { + hi = highlights[i]; + + if (hi.series.bars.show) + drawBarHighlight(hi.series, hi.point); + else + drawPointHighlight(hi.series, hi.point); + } + octx.restore(); + + executeHooks(hooks.drawOverlay, [octx]); + } + + function highlight(s, point, auto) { + if (typeof s == "number") + s = series[s]; + + if (typeof point == "number") { + var ps = s.datapoints.pointsize; + point = s.datapoints.points.slice(ps * point, ps * (point + 1)); + } + + var i = indexOfHighlight(s, point); + if (i == -1) { + highlights.push({ series: s, point: point, auto: auto }); + + triggerRedrawOverlay(); + } + else if (!auto) + highlights[i].auto = false; + } + + function unhighlight(s, point) { + if (s == null && point == null) { + highlights = []; + triggerRedrawOverlay(); + return; + } + + if (typeof s == "number") + s = series[s]; + + if (typeof point == "number") { + var ps = s.datapoints.pointsize; + point = s.datapoints.points.slice(ps * point, ps * (point + 1)); + } + + var i = indexOfHighlight(s, point); + if (i != -1) { + highlights.splice(i, 1); + + triggerRedrawOverlay(); + } + } + + function indexOfHighlight(s, p) { + for (var i = 0; i < highlights.length; ++i) { + var h = highlights[i]; + if (h.series == s && h.point[0] == p[0] + && h.point[1] == p[1]) + return i; + } + return -1; + } + + function drawPointHighlight(series, point) { + var x = point[0], y = point[1], + axisx = series.xaxis, axisy = series.yaxis, + highlightColor = (typeof series.highlightColor === "string") ? series.highlightColor : $.color.parse(series.color).scale('a', 0.5).toString(); + + if (x < axisx.min || x > axisx.max || y < axisy.min || y > axisy.max) + return; + + var pointRadius = series.points.radius + series.points.lineWidth / 2; + octx.lineWidth = pointRadius; + octx.strokeStyle = highlightColor; + var radius = 1.5 * pointRadius; + x = axisx.p2c(x); + y = axisy.p2c(y); + + octx.beginPath(); + if (series.points.symbol == "circle") + octx.arc(x, y, radius, 0, 2 * Math.PI, false); + else + series.points.symbol(octx, x, y, radius, false); + octx.closePath(); + octx.stroke(); + } + + function drawBarHighlight(series, point) { + var highlightColor = (typeof series.highlightColor === "string") ? series.highlightColor : $.color.parse(series.color).scale('a', 0.5).toString(), + fillStyle = highlightColor, + barLeft; + + switch (series.bars.align) { + case "left": + barLeft = 0; + break; + case "right": + barLeft = -series.bars.barWidth; + break; + default: + barLeft = -series.bars.barWidth / 2; + } + + octx.lineWidth = series.bars.lineWidth; + octx.strokeStyle = highlightColor; + + drawBar(point[0], point[1], point[2] || 0, barLeft, barLeft + series.bars.barWidth, + function () { return fillStyle; }, series.xaxis, series.yaxis, octx, series.bars.horizontal, series.bars.lineWidth); + } + + function getColorOrGradient(spec, bottom, top, defaultColor) { + if (typeof spec == "string") + return spec; + else { + // assume this is a gradient spec; IE currently only + // supports a simple vertical gradient properly, so that's + // what we support too + var gradient = ctx.createLinearGradient(0, top, 0, bottom); + + for (var i = 0, l = spec.colors.length; i < l; ++i) { + var c = spec.colors[i]; + if (typeof c != "string") { + var co = $.color.parse(defaultColor); + if (c.brightness != null) + co = co.scale('rgb', c.brightness); + if (c.opacity != null) + co.a *= c.opacity; + c = co.toString(); + } + gradient.addColorStop(i / (l - 1), c); + } + + return gradient; + } + } + } + + // Add the plot function to the top level of the jQuery object + + $.plot = function(placeholder, data, options) { + //var t0 = new Date(); + var plot = new Plot($(placeholder), data, options, $.plot.plugins); + //(window.console ? console.log : alert)("time used (msecs): " + ((new Date()).getTime() - t0.getTime())); + return plot; + }; + + $.plot.version = "0.8.3"; + + $.plot.plugins = []; + + // Also add the plot function as a chainable property + + $.fn.plot = function(data, options) { + return this.each(function() { + $.plot(this, data, options); + }); + }; + + // round to nearby lower multiple of base + function floorInBase(n, base) { + return base * Math.floor(n / base); + } + +})(jQuery); diff --git a/qa/workunits/erasure-code/jquery.js b/qa/workunits/erasure-code/jquery.js new file mode 100644 index 000000000..8c24ffc61 --- /dev/null +++ b/qa/workunits/erasure-code/jquery.js @@ -0,0 +1,9472 @@ +/*! + * jQuery JavaScript Library v1.8.3 + * http://jquery.com/ + * + * Includes Sizzle.js + * http://sizzlejs.com/ + * + * Copyright 2012 jQuery Foundation and other contributors + * Released under the MIT license + * http://jquery.org/license + * + * Date: Tue Nov 13 2012 08:20:33 GMT-0500 (Eastern Standard Time) + */ +(function( window, undefined ) { +var + // A central reference to the root jQuery(document) + rootjQuery, + + // The deferred used on DOM ready + readyList, + + // Use the correct document accordingly with window argument (sandbox) + document = window.document, + location = window.location, + navigator = window.navigator, + + // Map over jQuery in case of overwrite + _jQuery = window.jQuery, + + // Map over the $ in case of overwrite + _$ = window.$, + + // Save a reference to some core methods + core_push = Array.prototype.push, + core_slice = Array.prototype.slice, + core_indexOf = Array.prototype.indexOf, + core_toString = Object.prototype.toString, + core_hasOwn = Object.prototype.hasOwnProperty, + core_trim = String.prototype.trim, + + // Define a local copy of jQuery + jQuery = function( selector, context ) { + // The jQuery object is actually just the init constructor 'enhanced' + return new jQuery.fn.init( selector, context, rootjQuery ); + }, + + // Used for matching numbers + core_pnum = /[\-+]?(?:\d*\.|)\d+(?:[eE][\-+]?\d+|)/.source, + + // Used for detecting and trimming whitespace + core_rnotwhite = /\S/, + core_rspace = /\s+/, + + // Make sure we trim BOM and NBSP (here's looking at you, Safari 5.0 and IE) + rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, + + // A simple way to check for HTML strings + // Prioritize #id over <tag> to avoid XSS via location.hash (#9521) + rquickExpr = /^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/, + + // Match a standalone tag + rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>|)$/, + + // JSON RegExp + rvalidchars = /^[\],:{}\s]*$/, + rvalidbraces = /(?:^|:|,)(?:\s*\[)+/g, + rvalidescape = /\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g, + rvalidtokens = /"[^"\\\r\n]*"|true|false|null|-?(?:\d\d*\.|)\d+(?:[eE][\-+]?\d+|)/g, + + // Matches dashed string for camelizing + rmsPrefix = /^-ms-/, + rdashAlpha = /-([\da-z])/gi, + + // Used by jQuery.camelCase as callback to replace() + fcamelCase = function( all, letter ) { + return ( letter + "" ).toUpperCase(); + }, + + // The ready event handler and self cleanup method + DOMContentLoaded = function() { + if ( document.addEventListener ) { + document.removeEventListener( "DOMContentLoaded", DOMContentLoaded, false ); + jQuery.ready(); + } else if ( document.readyState === "complete" ) { + // we're here because readyState === "complete" in oldIE + // which is good enough for us to call the dom ready! + document.detachEvent( "onreadystatechange", DOMContentLoaded ); + jQuery.ready(); + } + }, + + // [[Class]] -> type pairs + class2type = {}; + +jQuery.fn = jQuery.prototype = { + constructor: jQuery, + init: function( selector, context, rootjQuery ) { + var match, elem, ret, doc; + + // Handle $(""), $(null), $(undefined), $(false) + if ( !selector ) { + return this; + } + + // Handle $(DOMElement) + if ( selector.nodeType ) { + this.context = this[0] = selector; + this.length = 1; + return this; + } + + // Handle HTML strings + if ( typeof selector === "string" ) { + if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) { + // Assume that strings that start and end with <> are HTML and skip the regex check + match = [ null, selector, null ]; + + } else { + match = rquickExpr.exec( selector ); + } + + // Match html or make sure no context is specified for #id + if ( match && (match[1] || !context) ) { + + // HANDLE: $(html) -> $(array) + if ( match[1] ) { + context = context instanceof jQuery ? context[0] : context; + doc = ( context && context.nodeType ? context.ownerDocument || context : document ); + + // scripts is true for back-compat + selector = jQuery.parseHTML( match[1], doc, true ); + if ( rsingleTag.test( match[1] ) && jQuery.isPlainObject( context ) ) { + this.attr.call( selector, context, true ); + } + + return jQuery.merge( this, selector ); + + // HANDLE: $(#id) + } else { + elem = document.getElementById( match[2] ); + + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + if ( elem && elem.parentNode ) { + // Handle the case where IE and Opera return items + // by name instead of ID + if ( elem.id !== match[2] ) { + return rootjQuery.find( selector ); + } + + // Otherwise, we inject the element directly into the jQuery object + this.length = 1; + this[0] = elem; + } + + this.context = document; + this.selector = selector; + return this; + } + + // HANDLE: $(expr, $(...)) + } else if ( !context || context.jquery ) { + return ( context || rootjQuery ).find( selector ); + + // HANDLE: $(expr, context) + // (which is just equivalent to: $(context).find(expr) + } else { + return this.constructor( context ).find( selector ); + } + + // HANDLE: $(function) + // Shortcut for document ready + } else if ( jQuery.isFunction( selector ) ) { + return rootjQuery.ready( selector ); + } + + if ( selector.selector !== undefined ) { + this.selector = selector.selector; + this.context = selector.context; + } + + return jQuery.makeArray( selector, this ); + }, + + // Start with an empty selector + selector: "", + + // The current version of jQuery being used + jquery: "1.8.3", + + // The default length of a jQuery object is 0 + length: 0, + + // The number of elements contained in the matched element set + size: function() { + return this.length; + }, + + toArray: function() { + return core_slice.call( this ); + }, + + // Get the Nth element in the matched element set OR + // Get the whole matched element set as a clean array + get: function( num ) { + return num == null ? + + // Return a 'clean' array + this.toArray() : + + // Return just the object + ( num < 0 ? this[ this.length + num ] : this[ num ] ); + }, + + // Take an array of elements and push it onto the stack + // (returning the new matched element set) + pushStack: function( elems, name, selector ) { + + // Build a new jQuery matched element set + var ret = jQuery.merge( this.constructor(), elems ); + + // Add the old object onto the stack (as a reference) + ret.prevObject = this; + + ret.context = this.context; + + if ( name === "find" ) { + ret.selector = this.selector + ( this.selector ? " " : "" ) + selector; + } else if ( name ) { + ret.selector = this.selector + "." + name + "(" + selector + ")"; + } + + // Return the newly-formed element set + return ret; + }, + + // Execute a callback for every element in the matched set. + // (You can seed the arguments with an array of args, but this is + // only used internally.) + each: function( callback, args ) { + return jQuery.each( this, callback, args ); + }, + + ready: function( fn ) { + // Add the callback + jQuery.ready.promise().done( fn ); + + return this; + }, + + eq: function( i ) { + i = +i; + return i === -1 ? + this.slice( i ) : + this.slice( i, i + 1 ); + }, + + first: function() { + return this.eq( 0 ); + }, + + last: function() { + return this.eq( -1 ); + }, + + slice: function() { + return this.pushStack( core_slice.apply( this, arguments ), + "slice", core_slice.call(arguments).join(",") ); + }, + + map: function( callback ) { + return this.pushStack( jQuery.map(this, function( elem, i ) { + return callback.call( elem, i, elem ); + })); + }, + + end: function() { + return this.prevObject || this.constructor(null); + }, + + // For internal use only. + // Behaves like an Array's method, not like a jQuery method. + push: core_push, + sort: [].sort, + splice: [].splice +}; + +// Give the init function the jQuery prototype for later instantiation +jQuery.fn.init.prototype = jQuery.fn; + +jQuery.extend = jQuery.fn.extend = function() { + var options, name, src, copy, copyIsArray, clone, + target = arguments[0] || {}, + i = 1, + length = arguments.length, + deep = false; + + // Handle a deep copy situation + if ( typeof target === "boolean" ) { + deep = target; + target = arguments[1] || {}; + // skip the boolean and the target + i = 2; + } + + // Handle case when target is a string or something (possible in deep copy) + if ( typeof target !== "object" && !jQuery.isFunction(target) ) { + target = {}; + } + + // extend jQuery itself if only one argument is passed + if ( length === i ) { + target = this; + --i; + } + + for ( ; i < length; i++ ) { + // Only deal with non-null/undefined values + if ( (options = arguments[ i ]) != null ) { + // Extend the base object + for ( name in options ) { + src = target[ name ]; + copy = options[ name ]; + + // Prevent never-ending loop + if ( target === copy ) { + continue; + } + + // Recurse if we're merging plain objects or arrays + if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) { + if ( copyIsArray ) { + copyIsArray = false; + clone = src && jQuery.isArray(src) ? src : []; + + } else { + clone = src && jQuery.isPlainObject(src) ? src : {}; + } + + // Never move original objects, clone them + target[ name ] = jQuery.extend( deep, clone, copy ); + + // Don't bring in undefined values + } else if ( copy !== undefined ) { + target[ name ] = copy; + } + } + } + } + + // Return the modified object + return target; +}; + +jQuery.extend({ + noConflict: function( deep ) { + if ( window.$ === jQuery ) { + window.$ = _$; + } + + if ( deep && window.jQuery === jQuery ) { + window.jQuery = _jQuery; + } + + return jQuery; + }, + + // Is the DOM ready to be used? Set to true once it occurs. + isReady: false, + + // A counter to track how many items to wait for before + // the ready event fires. See #6781 + readyWait: 1, + + // Hold (or release) the ready event + holdReady: function( hold ) { + if ( hold ) { + jQuery.readyWait++; + } else { + jQuery.ready( true ); + } + }, + + // Handle when the DOM is ready + ready: function( wait ) { + + // Abort if there are pending holds or we're already ready + if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) { + return; + } + + // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). + if ( !document.body ) { + return setTimeout( jQuery.ready, 1 ); + } + + // Remember that the DOM is ready + jQuery.isReady = true; + + // If a normal DOM Ready event fired, decrement, and wait if need be + if ( wait !== true && --jQuery.readyWait > 0 ) { + return; + } + + // If there are functions bound, to execute + readyList.resolveWith( document, [ jQuery ] ); + + // Trigger any bound ready events + if ( jQuery.fn.trigger ) { + jQuery( document ).trigger("ready").off("ready"); + } + }, + + // See test/unit/core.js for details concerning isFunction. + // Since version 1.3, DOM methods and functions like alert + // aren't supported. They return false on IE (#2968). + isFunction: function( obj ) { + return jQuery.type(obj) === "function"; + }, + + isArray: Array.isArray || function( obj ) { + return jQuery.type(obj) === "array"; + }, + + isWindow: function( obj ) { + return obj != null && obj == obj.window; + }, + + isNumeric: function( obj ) { + return !isNaN( parseFloat(obj) ) && isFinite( obj ); + }, + + type: function( obj ) { + return obj == null ? + String( obj ) : + class2type[ core_toString.call(obj) ] || "object"; + }, + + isPlainObject: function( obj ) { + // Must be an Object. + // Because of IE, we also have to check the presence of the constructor property. + // Make sure that DOM nodes and window objects don't pass through, as well + if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) { + return false; + } + + try { + // Not own constructor property must be Object + if ( obj.constructor && + !core_hasOwn.call(obj, "constructor") && + !core_hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) { + return false; + } + } catch ( e ) { + // IE8,9 Will throw exceptions on certain host objects #9897 + return false; + } + + // Own properties are enumerated firstly, so to speed up, + // if last one is own, then all properties are own. + + var key; + for ( key in obj ) {} + + return key === undefined || core_hasOwn.call( obj, key ); + }, + + isEmptyObject: function( obj ) { + var name; + for ( name in obj ) { + return false; + } + return true; + }, + + error: function( msg ) { + throw new Error( msg ); + }, + + // data: string of html + // context (optional): If specified, the fragment will be created in this context, defaults to document + // scripts (optional): If true, will include scripts passed in the html string + parseHTML: function( data, context, scripts ) { + var parsed; + if ( !data || typeof data !== "string" ) { + return null; + } + if ( typeof context === "boolean" ) { + scripts = context; + context = 0; + } + context = context || document; + + // Single tag + if ( (parsed = rsingleTag.exec( data )) ) { + return [ context.createElement( parsed[1] ) ]; + } + + parsed = jQuery.buildFragment( [ data ], context, scripts ? null : [] ); + return jQuery.merge( [], + (parsed.cacheable ? jQuery.clone( parsed.fragment ) : parsed.fragment).childNodes ); + }, + + parseJSON: function( data ) { + if ( !data || typeof data !== "string") { + return null; + } + + // Make sure leading/trailing whitespace is removed (IE can't handle it) + data = jQuery.trim( data ); + + // Attempt to parse using the native JSON parser first + if ( window.JSON && window.JSON.parse ) { + return window.JSON.parse( data ); + } + + // Make sure the incoming data is actual JSON + // Logic borrowed from http://json.org/json2.js + if ( rvalidchars.test( data.replace( rvalidescape, "@" ) + .replace( rvalidtokens, "]" ) + .replace( rvalidbraces, "")) ) { + + return ( new Function( "return " + data ) )(); + + } + jQuery.error( "Invalid JSON: " + data ); + }, + + // Cross-browser xml parsing + parseXML: function( data ) { + var xml, tmp; + if ( !data || typeof data !== "string" ) { + return null; + } + try { + if ( window.DOMParser ) { // Standard + tmp = new DOMParser(); + xml = tmp.parseFromString( data , "text/xml" ); + } else { // IE + xml = new ActiveXObject( "Microsoft.XMLDOM" ); + xml.async = "false"; + xml.loadXML( data ); + } + } catch( e ) { + xml = undefined; + } + if ( !xml || !xml.documentElement || xml.getElementsByTagName( "parsererror" ).length ) { + jQuery.error( "Invalid XML: " + data ); + } + return xml; + }, + + noop: function() {}, + + // Evaluates a script in a global context + // Workarounds based on findings by Jim Driscoll + // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context + globalEval: function( data ) { + if ( data && core_rnotwhite.test( data ) ) { + // We use execScript on Internet Explorer + // We use an anonymous function so that context is window + // rather than jQuery in Firefox + ( window.execScript || function( data ) { + window[ "eval" ].call( window, data ); + } )( data ); + } + }, + + // Convert dashed to camelCase; used by the css and data modules + // Microsoft forgot to hump their vendor prefix (#9572) + camelCase: function( string ) { + return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); + }, + + nodeName: function( elem, name ) { + return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); + }, + + // args is for internal usage only + each: function( obj, callback, args ) { + var name, + i = 0, + length = obj.length, + isObj = length === undefined || jQuery.isFunction( obj ); + + if ( args ) { + if ( isObj ) { + for ( name in obj ) { + if ( callback.apply( obj[ name ], args ) === false ) { + break; + } + } + } else { + for ( ; i < length; ) { + if ( callback.apply( obj[ i++ ], args ) === false ) { + break; + } + } + } + + // A special, fast, case for the most common use of each + } else { + if ( isObj ) { + for ( name in obj ) { + if ( callback.call( obj[ name ], name, obj[ name ] ) === false ) { + break; + } + } + } else { + for ( ; i < length; ) { + if ( callback.call( obj[ i ], i, obj[ i++ ] ) === false ) { + break; + } + } + } + } + + return obj; + }, + + // Use native String.trim function wherever possible + trim: core_trim && !core_trim.call("\uFEFF\xA0") ? + function( text ) { + return text == null ? + "" : + core_trim.call( text ); + } : + + // Otherwise use our own trimming functionality + function( text ) { + return text == null ? + "" : + ( text + "" ).replace( rtrim, "" ); + }, + + // results is for internal usage only + makeArray: function( arr, results ) { + var type, + ret = results || []; + + if ( arr != null ) { + // The window, strings (and functions) also have 'length' + // Tweaked logic slightly to handle Blackberry 4.7 RegExp issues #6930 + type = jQuery.type( arr ); + + if ( arr.length == null || type === "string" || type === "function" || type === "regexp" || jQuery.isWindow( arr ) ) { + core_push.call( ret, arr ); + } else { + jQuery.merge( ret, arr ); + } + } + + return ret; + }, + + inArray: function( elem, arr, i ) { + var len; + + if ( arr ) { + if ( core_indexOf ) { + return core_indexOf.call( arr, elem, i ); + } + + len = arr.length; + i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0; + + for ( ; i < len; i++ ) { + // Skip accessing in sparse arrays + if ( i in arr && arr[ i ] === elem ) { + return i; + } + } + } + + return -1; + }, + + merge: function( first, second ) { + var l = second.length, + i = first.length, + j = 0; + + if ( typeof l === "number" ) { + for ( ; j < l; j++ ) { + first[ i++ ] = second[ j ]; + } + + } else { + while ( second[j] !== undefined ) { + first[ i++ ] = second[ j++ ]; + } + } + + first.length = i; + + return first; + }, + + grep: function( elems, callback, inv ) { + var retVal, + ret = [], + i = 0, + length = elems.length; + inv = !!inv; + + // Go through the array, only saving the items + // that pass the validator function + for ( ; i < length; i++ ) { + retVal = !!callback( elems[ i ], i ); + if ( inv !== retVal ) { + ret.push( elems[ i ] ); + } + } + + return ret; + }, + + // arg is for internal usage only + map: function( elems, callback, arg ) { + var value, key, + ret = [], + i = 0, + length = elems.length, + // jquery objects are treated as arrays + isArray = elems instanceof jQuery || length !== undefined && typeof length === "number" && ( ( length > 0 && elems[ 0 ] && elems[ length -1 ] ) || length === 0 || jQuery.isArray( elems ) ) ; + + // Go through the array, translating each of the items to their + if ( isArray ) { + for ( ; i < length; i++ ) { + value = callback( elems[ i ], i, arg ); + + if ( value != null ) { + ret[ ret.length ] = value; + } + } + + // Go through every key on the object, + } else { + for ( key in elems ) { + value = callback( elems[ key ], key, arg ); + + if ( value != null ) { + ret[ ret.length ] = value; + } + } + } + + // Flatten any nested arrays + return ret.concat.apply( [], ret ); + }, + + // A global GUID counter for objects + guid: 1, + + // Bind a function to a context, optionally partially applying any + // arguments. + proxy: function( fn, context ) { + var tmp, args, proxy; + + if ( typeof context === "string" ) { + tmp = fn[ context ]; + context = fn; + fn = tmp; + } + + // Quick check to determine if target is callable, in the spec + // this throws a TypeError, but we will just return undefined. + if ( !jQuery.isFunction( fn ) ) { + return undefined; + } + + // Simulated bind + args = core_slice.call( arguments, 2 ); + proxy = function() { + return fn.apply( context, args.concat( core_slice.call( arguments ) ) ); + }; + + // Set the guid of unique handler to the same of original handler, so it can be removed + proxy.guid = fn.guid = fn.guid || jQuery.guid++; + + return proxy; + }, + + // Multifunctional method to get and set values of a collection + // The value/s can optionally be executed if it's a function + access: function( elems, fn, key, value, chainable, emptyGet, pass ) { + var exec, + bulk = key == null, + i = 0, + length = elems.length; + + // Sets many values + if ( key && typeof key === "object" ) { + for ( i in key ) { + jQuery.access( elems, fn, i, key[i], 1, emptyGet, value ); + } + chainable = 1; + + // Sets one value + } else if ( value !== undefined ) { + // Optionally, function values get executed if exec is true + exec = pass === undefined && jQuery.isFunction( value ); + + if ( bulk ) { + // Bulk operations only iterate when executing function values + if ( exec ) { + exec = fn; + fn = function( elem, key, value ) { + return exec.call( jQuery( elem ), value ); + }; + + // Otherwise they run against the entire set + } else { + fn.call( elems, value ); + fn = null; + } + } + + if ( fn ) { + for (; i < length; i++ ) { + fn( elems[i], key, exec ? value.call( elems[i], i, fn( elems[i], key ) ) : value, pass ); + } + } + + chainable = 1; + } + + return chainable ? + elems : + + // Gets + bulk ? + fn.call( elems ) : + length ? fn( elems[0], key ) : emptyGet; + }, + + now: function() { + return ( new Date() ).getTime(); + } +}); + +jQuery.ready.promise = function( obj ) { + if ( !readyList ) { + + readyList = jQuery.Deferred(); + + // Catch cases where $(document).ready() is called after the browser event has already occurred. + // we once tried to use readyState "interactive" here, but it caused issues like the one + // discovered by ChrisS here: http://bugs.jquery.com/ticket/12282#comment:15 + if ( document.readyState === "complete" ) { + // Handle it asynchronously to allow scripts the opportunity to delay ready + setTimeout( jQuery.ready, 1 ); + + // Standards-based browsers support DOMContentLoaded + } else if ( document.addEventListener ) { + // Use the handy event callback + document.addEventListener( "DOMContentLoaded", DOMContentLoaded, false ); + + // A fallback to window.onload, that will always work + window.addEventListener( "load", jQuery.ready, false ); + + // If IE event model is used + } else { + // Ensure firing before onload, maybe late but safe also for iframes + document.attachEvent( "onreadystatechange", DOMContentLoaded ); + + // A fallback to window.onload, that will always work + window.attachEvent( "onload", jQuery.ready ); + + // If IE and not a frame + // continually check to see if the document is ready + var top = false; + + try { + top = window.frameElement == null && document.documentElement; + } catch(e) {} + + if ( top && top.doScroll ) { + (function doScrollCheck() { + if ( !jQuery.isReady ) { + + try { + // Use the trick by Diego Perini + // http://javascript.nwbox.com/IEContentLoaded/ + top.doScroll("left"); + } catch(e) { + return setTimeout( doScrollCheck, 50 ); + } + + // and execute any waiting functions + jQuery.ready(); + } + })(); + } + } + } + return readyList.promise( obj ); +}; + +// Populate the class2type map +jQuery.each("Boolean Number String Function Array Date RegExp Object".split(" "), function(i, name) { + class2type[ "[object " + name + "]" ] = name.toLowerCase(); +}); + +// All jQuery objects should point back to these +rootjQuery = jQuery(document); +// String to Object options format cache +var optionsCache = {}; + +// Convert String-formatted options into Object-formatted ones and store in cache +function createOptions( options ) { + var object = optionsCache[ options ] = {}; + jQuery.each( options.split( core_rspace ), function( _, flag ) { + object[ flag ] = true; + }); + return object; +} + +/* + * Create a callback list using the following parameters: + * + * options: an optional list of space-separated options that will change how + * the callback list behaves or a more traditional option object + * + * By default a callback list will act like an event callback list and can be + * "fired" multiple times. + * + * Possible options: + * + * once: will ensure the callback list can only be fired once (like a Deferred) + * + * memory: will keep track of previous values and will call any callback added + * after the list has been fired right away with the latest "memorized" + * values (like a Deferred) + * + * unique: will ensure a callback can only be added once (no duplicate in the list) + * + * stopOnFalse: interrupt callings when a callback returns false + * + */ +jQuery.Callbacks = function( options ) { + + // Convert options from String-formatted to Object-formatted if needed + // (we check in cache first) + options = typeof options === "string" ? + ( optionsCache[ options ] || createOptions( options ) ) : + jQuery.extend( {}, options ); + + var // Last fire value (for non-forgettable lists) + memory, + // Flag to know if list was already fired + fired, + // Flag to know if list is currently firing + firing, + // First callback to fire (used internally by add and fireWith) + firingStart, + // End of the loop when firing + firingLength, + // Index of currently firing callback (modified by remove if needed) + firingIndex, + // Actual callback list + list = [], + // Stack of fire calls for repeatable lists + stack = !options.once && [], + // Fire callbacks + fire = function( data ) { + memory = options.memory && data; + fired = true; + firingIndex = firingStart || 0; + firingStart = 0; + firingLength = list.length; + firing = true; + for ( ; list && firingIndex < firingLength; firingIndex++ ) { + if ( list[ firingIndex ].apply( data[ 0 ], data[ 1 ] ) === false && options.stopOnFalse ) { + memory = false; // To prevent further calls using add + break; + } + } + firing = false; + if ( list ) { + if ( stack ) { + if ( stack.length ) { + fire( stack.shift() ); + } + } else if ( memory ) { + list = []; + } else { + self.disable(); + } + } + }, + // Actual Callbacks object + self = { + // Add a callback or a collection of callbacks to the list + add: function() { + if ( list ) { + // First, we save the current length + var start = list.length; + (function add( args ) { + jQuery.each( args, function( _, arg ) { + var type = jQuery.type( arg ); + if ( type === "function" ) { + if ( !options.unique || !self.has( arg ) ) { + list.push( arg ); + } + } else if ( arg && arg.length && type !== "string" ) { + // Inspect recursively + add( arg ); + } + }); + })( arguments ); + // Do we need to add the callbacks to the + // current firing batch? + if ( firing ) { + firingLength = list.length; + // With memory, if we're not firing then + // we should call right away + } else if ( memory ) { + firingStart = start; + fire( memory ); + } + } + return this; + }, + // Remove a callback from the list + remove: function() { + if ( list ) { + jQuery.each( arguments, function( _, arg ) { + var index; + while( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) { + list.splice( index, 1 ); + // Handle firing indexes + if ( firing ) { + if ( index <= firingLength ) { + firingLength--; + } + if ( index <= firingIndex ) { + firingIndex--; + } + } + } + }); + } + return this; + }, + // Control if a given callback is in the list + has: function( fn ) { + return jQuery.inArray( fn, list ) > -1; + }, + // Remove all callbacks from the list + empty: function() { + list = []; + return this; + }, + // Have the list do nothing anymore + disable: function() { + list = stack = memory = undefined; + return this; + }, + // Is it disabled? + disabled: function() { + return !list; + }, + // Lock the list in its current state + lock: function() { + stack = undefined; + if ( !memory ) { + self.disable(); + } + return this; + }, + // Is it locked? + locked: function() { + return !stack; + }, + // Call all callbacks with the given context and arguments + fireWith: function( context, args ) { + args = args || []; + args = [ context, args.slice ? args.slice() : args ]; + if ( list && ( !fired || stack ) ) { + if ( firing ) { + stack.push( args ); + } else { + fire( args ); + } + } + return this; + }, + // Call all the callbacks with the given arguments + fire: function() { + self.fireWith( this, arguments ); + return this; + }, + // To know if the callbacks have already been called at least once + fired: function() { + return !!fired; + } + }; + + return self; +}; +jQuery.extend({ + + Deferred: function( func ) { + var tuples = [ + // action, add listener, listener list, final state + [ "resolve", "done", jQuery.Callbacks("once memory"), "resolved" ], + [ "reject", "fail", jQuery.Callbacks("once memory"), "rejected" ], + [ "notify", "progress", jQuery.Callbacks("memory") ] + ], + state = "pending", + promise = { + state: function() { + return state; + }, + always: function() { + deferred.done( arguments ).fail( arguments ); + return this; + }, + then: function( /* fnDone, fnFail, fnProgress */ ) { + var fns = arguments; + return jQuery.Deferred(function( newDefer ) { + jQuery.each( tuples, function( i, tuple ) { + var action = tuple[ 0 ], + fn = fns[ i ]; + // deferred[ done | fail | progress ] for forwarding actions to newDefer + deferred[ tuple[1] ]( jQuery.isFunction( fn ) ? + function() { + var returned = fn.apply( this, arguments ); + if ( returned && jQuery.isFunction( returned.promise ) ) { + returned.promise() + .done( newDefer.resolve ) + .fail( newDefer.reject ) + .progress( newDefer.notify ); + } else { + newDefer[ action + "With" ]( this === deferred ? newDefer : this, [ returned ] ); + } + } : + newDefer[ action ] + ); + }); + fns = null; + }).promise(); + }, + // Get a promise for this deferred + // If obj is provided, the promise aspect is added to the object + promise: function( obj ) { + return obj != null ? jQuery.extend( obj, promise ) : promise; + } + }, + deferred = {}; + + // Keep pipe for back-compat + promise.pipe = promise.then; + + // Add list-specific methods + jQuery.each( tuples, function( i, tuple ) { + var list = tuple[ 2 ], + stateString = tuple[ 3 ]; + + // promise[ done | fail | progress ] = list.add + promise[ tuple[1] ] = list.add; + + // Handle state + if ( stateString ) { + list.add(function() { + // state = [ resolved | rejected ] + state = stateString; + + // [ reject_list | resolve_list ].disable; progress_list.lock + }, tuples[ i ^ 1 ][ 2 ].disable, tuples[ 2 ][ 2 ].lock ); + } + + // deferred[ resolve | reject | notify ] = list.fire + deferred[ tuple[0] ] = list.fire; + deferred[ tuple[0] + "With" ] = list.fireWith; + }); + + // Make the deferred a promise + promise.promise( deferred ); + + // Call given func if any + if ( func ) { + func.call( deferred, deferred ); + } + + // All done! + return deferred; + }, + + // Deferred helper + when: function( subordinate /* , ..., subordinateN */ ) { + var i = 0, + resolveValues = core_slice.call( arguments ), + length = resolveValues.length, + + // the count of uncompleted subordinates + remaining = length !== 1 || ( subordinate && jQuery.isFunction( subordinate.promise ) ) ? length : 0, + + // the master Deferred. If resolveValues consist of only a single Deferred, just use that. + deferred = remaining === 1 ? subordinate : jQuery.Deferred(), + + // Update function for both resolve and progress values + updateFunc = function( i, contexts, values ) { + return function( value ) { + contexts[ i ] = this; + values[ i ] = arguments.length > 1 ? core_slice.call( arguments ) : value; + if( values === progressValues ) { + deferred.notifyWith( contexts, values ); + } else if ( !( --remaining ) ) { + deferred.resolveWith( contexts, values ); + } + }; + }, + + progressValues, progressContexts, resolveContexts; + + // add listeners to Deferred subordinates; treat others as resolved + if ( length > 1 ) { + progressValues = new Array( length ); + progressContexts = new Array( length ); + resolveContexts = new Array( length ); + for ( ; i < length; i++ ) { + if ( resolveValues[ i ] && jQuery.isFunction( resolveValues[ i ].promise ) ) { + resolveValues[ i ].promise() + .done( updateFunc( i, resolveContexts, resolveValues ) ) + .fail( deferred.reject ) + .progress( updateFunc( i, progressContexts, progressValues ) ); + } else { + --remaining; + } + } + } + + // if we're not waiting on anything, resolve the master + if ( !remaining ) { + deferred.resolveWith( resolveContexts, resolveValues ); + } + + return deferred.promise(); + } +}); +jQuery.support = (function() { + + var support, + all, + a, + select, + opt, + input, + fragment, + eventName, + i, + isSupported, + clickFn, + div = document.createElement("div"); + + // Setup + div.setAttribute( "className", "t" ); + div.innerHTML = " <link/><table></table><a href='/a'>a</a><input type='checkbox'/>"; + + // Support tests won't run in some limited or non-browser environments + all = div.getElementsByTagName("*"); + a = div.getElementsByTagName("a")[ 0 ]; + if ( !all || !a || !all.length ) { + return {}; + } + + // First batch of tests + select = document.createElement("select"); + opt = select.appendChild( document.createElement("option") ); + input = div.getElementsByTagName("input")[ 0 ]; + + a.style.cssText = "top:1px;float:left;opacity:.5"; + support = { + // IE strips leading whitespace when .innerHTML is used + leadingWhitespace: ( div.firstChild.nodeType === 3 ), + + // Make sure that tbody elements aren't automatically inserted + // IE will insert them into empty tables + tbody: !div.getElementsByTagName("tbody").length, + + // Make sure that link elements get serialized correctly by innerHTML + // This requires a wrapper element in IE + htmlSerialize: !!div.getElementsByTagName("link").length, + + // Get the style information from getAttribute + // (IE uses .cssText instead) + style: /top/.test( a.getAttribute("style") ), + + // Make sure that URLs aren't manipulated + // (IE normalizes it by default) + hrefNormalized: ( a.getAttribute("href") === "/a" ), + + // Make sure that element opacity exists + // (IE uses filter instead) + // Use a regex to work around a WebKit issue. See #5145 + opacity: /^0.5/.test( a.style.opacity ), + + // Verify style float existence + // (IE uses styleFloat instead of cssFloat) + cssFloat: !!a.style.cssFloat, + + // Make sure that if no value is specified for a checkbox + // that it defaults to "on". + // (WebKit defaults to "" instead) + checkOn: ( input.value === "on" ), + + // Make sure that a selected-by-default option has a working selected property. + // (WebKit defaults to false instead of true, IE too, if it's in an optgroup) + optSelected: opt.selected, + + // Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7) + getSetAttribute: div.className !== "t", + + // Tests for enctype support on a form (#6743) + enctype: !!document.createElement("form").enctype, + + // Makes sure cloning an html5 element does not cause problems + // Where outerHTML is undefined, this still works + html5Clone: document.createElement("nav").cloneNode( true ).outerHTML !== "<:nav></:nav>", + + // jQuery.support.boxModel DEPRECATED in 1.8 since we don't support Quirks Mode + boxModel: ( document.compatMode === "CSS1Compat" ), + + // Will be defined later + submitBubbles: true, + changeBubbles: true, + focusinBubbles: false, + deleteExpando: true, + noCloneEvent: true, + inlineBlockNeedsLayout: false, + shrinkWrapBlocks: false, + reliableMarginRight: true, + boxSizingReliable: true, + pixelPosition: false + }; + + // Make sure checked status is properly cloned + input.checked = true; + support.noCloneChecked = input.cloneNode( true ).checked; + + // Make sure that the options inside disabled selects aren't marked as disabled + // (WebKit marks them as disabled) + select.disabled = true; + support.optDisabled = !opt.disabled; + + // Test to see if it's possible to delete an expando from an element + // Fails in Internet Explorer + try { + delete div.test; + } catch( e ) { + support.deleteExpando = false; + } + + if ( !div.addEventListener && div.attachEvent && div.fireEvent ) { + div.attachEvent( "onclick", clickFn = function() { + // Cloning a node shouldn't copy over any + // bound event handlers (IE does this) + support.noCloneEvent = false; + }); + div.cloneNode( true ).fireEvent("onclick"); + div.detachEvent( "onclick", clickFn ); + } + + // Check if a radio maintains its value + // after being appended to the DOM + input = document.createElement("input"); + input.value = "t"; + input.setAttribute( "type", "radio" ); + support.radioValue = input.value === "t"; + + input.setAttribute( "checked", "checked" ); + + // #11217 - WebKit loses check when the name is after the checked attribute + input.setAttribute( "name", "t" ); + + div.appendChild( input ); + fragment = document.createDocumentFragment(); + fragment.appendChild( div.lastChild ); + + // WebKit doesn't clone checked state correctly in fragments + support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked; + + // Check if a disconnected checkbox will retain its checked + // value of true after appended to the DOM (IE6/7) + support.appendChecked = input.checked; + + fragment.removeChild( input ); + fragment.appendChild( div ); + + // Technique from Juriy Zaytsev + // http://perfectionkills.com/detecting-event-support-without-browser-sniffing/ + // We only care about the case where non-standard event systems + // are used, namely in IE. Short-circuiting here helps us to + // avoid an eval call (in setAttribute) which can cause CSP + // to go haywire. See: https://developer.mozilla.org/en/Security/CSP + if ( div.attachEvent ) { + for ( i in { + submit: true, + change: true, + focusin: true + }) { + eventName = "on" + i; + isSupported = ( eventName in div ); + if ( !isSupported ) { + div.setAttribute( eventName, "return;" ); + isSupported = ( typeof div[ eventName ] === "function" ); + } + support[ i + "Bubbles" ] = isSupported; + } + } + + // Run tests that need a body at doc ready + jQuery(function() { + var container, div, tds, marginDiv, + divReset = "padding:0;margin:0;border:0;display:block;overflow:hidden;", + body = document.getElementsByTagName("body")[0]; + + if ( !body ) { + // Return for frameset docs that don't have a body + return; + } + + container = document.createElement("div"); + container.style.cssText = "visibility:hidden;border:0;width:0;height:0;position:static;top:0;margin-top:1px"; + body.insertBefore( container, body.firstChild ); + + // Construct the test element + div = document.createElement("div"); + container.appendChild( div ); + + // Check if table cells still have offsetWidth/Height when they are set + // to display:none and there are still other visible table cells in a + // table row; if so, offsetWidth/Height are not reliable for use when + // determining if an element has been hidden directly using + // display:none (it is still safe to use offsets if a parent element is + // hidden; don safety goggles and see bug #4512 for more information). + // (only IE 8 fails this test) + div.innerHTML = "<table><tr><td></td><td>t</td></tr></table>"; + tds = div.getElementsByTagName("td"); + tds[ 0 ].style.cssText = "padding:0;margin:0;border:0;display:none"; + isSupported = ( tds[ 0 ].offsetHeight === 0 ); + + tds[ 0 ].style.display = ""; + tds[ 1 ].style.display = "none"; + + // Check if empty table cells still have offsetWidth/Height + // (IE <= 8 fail this test) + support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 ); + + // Check box-sizing and margin behavior + div.innerHTML = ""; + div.style.cssText = "box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;"; + support.boxSizing = ( div.offsetWidth === 4 ); + support.doesNotIncludeMarginInBodyOffset = ( body.offsetTop !== 1 ); + + // NOTE: To any future maintainer, we've window.getComputedStyle + // because jsdom on node.js will break without it. + if ( window.getComputedStyle ) { + support.pixelPosition = ( window.getComputedStyle( div, null ) || {} ).top !== "1%"; + support.boxSizingReliable = ( window.getComputedStyle( div, null ) || { width: "4px" } ).width === "4px"; + + // Check if div with explicit width and no margin-right incorrectly + // gets computed margin-right based on width of container. For more + // info see bug #3333 + // Fails in WebKit before Feb 2011 nightlies + // WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right + marginDiv = document.createElement("div"); + marginDiv.style.cssText = div.style.cssText = divReset; + marginDiv.style.marginRight = marginDiv.style.width = "0"; + div.style.width = "1px"; + div.appendChild( marginDiv ); + support.reliableMarginRight = + !parseFloat( ( window.getComputedStyle( marginDiv, null ) || {} ).marginRight ); + } + + if ( typeof div.style.zoom !== "undefined" ) { + // Check if natively block-level elements act like inline-block + // elements when setting their display to 'inline' and giving + // them layout + // (IE < 8 does this) + div.innerHTML = ""; + div.style.cssText = divReset + "width:1px;padding:1px;display:inline;zoom:1"; + support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 ); + + // Check if elements with layout shrink-wrap their children + // (IE 6 does this) + div.style.display = "block"; + div.style.overflow = "visible"; + div.innerHTML = "<div></div>"; + div.firstChild.style.width = "5px"; + support.shrinkWrapBlocks = ( div.offsetWidth !== 3 ); + + container.style.zoom = 1; + } + + // Null elements to avoid leaks in IE + body.removeChild( container ); + container = div = tds = marginDiv = null; + }); + + // Null elements to avoid leaks in IE + fragment.removeChild( div ); + all = a = select = opt = input = fragment = div = null; + + return support; +})(); +var rbrace = /(?:\{[\s\S]*\}|\[[\s\S]*\])$/, + rmultiDash = /([A-Z])/g; + +jQuery.extend({ + cache: {}, + + deletedIds: [], + + // Remove at next major release (1.9/2.0) + uuid: 0, + + // Unique for each copy of jQuery on the page + // Non-digits removed to match rinlinejQuery + expando: "jQuery" + ( jQuery.fn.jquery + Math.random() ).replace( /\D/g, "" ), + + // The following elements throw uncatchable exceptions if you + // attempt to add expando properties to them. + noData: { + "embed": true, + // Ban all objects except for Flash (which handle expandos) + "object": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000", + "applet": true + }, + + hasData: function( elem ) { + elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ]; + return !!elem && !isEmptyDataObject( elem ); + }, + + data: function( elem, name, data, pvt /* Internal Use Only */ ) { + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var thisCache, ret, + internalKey = jQuery.expando, + getByName = typeof name === "string", + + // We have to handle DOM nodes and JS objects differently because IE6-7 + // can't GC object references properly across the DOM-JS boundary + isNode = elem.nodeType, + + // Only DOM nodes need the global jQuery cache; JS object data is + // attached directly to the object so GC can occur automatically + cache = isNode ? jQuery.cache : elem, + + // Only defining an ID for JS objects if its cache already exists allows + // the code to shortcut on the same path as a DOM node with no cache + id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey; + + // Avoid doing any more work than we need to when trying to get data on an + // object that has no data at all + if ( (!id || !cache[id] || (!pvt && !cache[id].data)) && getByName && data === undefined ) { + return; + } + + if ( !id ) { + // Only DOM nodes need a new unique ID for each element since their data + // ends up in the global cache + if ( isNode ) { + elem[ internalKey ] = id = jQuery.deletedIds.pop() || jQuery.guid++; + } else { + id = internalKey; + } + } + + if ( !cache[ id ] ) { + cache[ id ] = {}; + + // Avoids exposing jQuery metadata on plain JS objects when the object + // is serialized using JSON.stringify + if ( !isNode ) { + cache[ id ].toJSON = jQuery.noop; + } + } + + // An object can be passed to jQuery.data instead of a key/value pair; this gets + // shallow copied over onto the existing cache + if ( typeof name === "object" || typeof name === "function" ) { + if ( pvt ) { + cache[ id ] = jQuery.extend( cache[ id ], name ); + } else { + cache[ id ].data = jQuery.extend( cache[ id ].data, name ); + } + } + + thisCache = cache[ id ]; + + // jQuery data() is stored in a separate object inside the object's internal data + // cache in order to avoid key collisions between internal data and user-defined + // data. + if ( !pvt ) { + if ( !thisCache.data ) { + thisCache.data = {}; + } + + thisCache = thisCache.data; + } + + if ( data !== undefined ) { + thisCache[ jQuery.camelCase( name ) ] = data; + } + + // Check for both converted-to-camel and non-converted data property names + // If a data property was specified + if ( getByName ) { + + // First Try to find as-is property data + ret = thisCache[ name ]; + + // Test for null|undefined property data + if ( ret == null ) { + + // Try to find the camelCased property + ret = thisCache[ jQuery.camelCase( name ) ]; + } + } else { + ret = thisCache; + } + + return ret; + }, + + removeData: function( elem, name, pvt /* Internal Use Only */ ) { + if ( !jQuery.acceptData( elem ) ) { + return; + } + + var thisCache, i, l, + + isNode = elem.nodeType, + + // See jQuery.data for more information + cache = isNode ? jQuery.cache : elem, + id = isNode ? elem[ jQuery.expando ] : jQuery.expando; + + // If there is already no cache entry for this object, there is no + // purpose in continuing + if ( !cache[ id ] ) { + return; + } + + if ( name ) { + + thisCache = pvt ? cache[ id ] : cache[ id ].data; + + if ( thisCache ) { + + // Support array or space separated string names for data keys + if ( !jQuery.isArray( name ) ) { + + // try the string as a key before any manipulation + if ( name in thisCache ) { + name = [ name ]; + } else { + + // split the camel cased version by spaces unless a key with the spaces exists + name = jQuery.camelCase( name ); + if ( name in thisCache ) { + name = [ name ]; + } else { + name = name.split(" "); + } + } + } + + for ( i = 0, l = name.length; i < l; i++ ) { + delete thisCache[ name[i] ]; + } + + // If there is no data left in the cache, we want to continue + // and let the cache object itself get destroyed + if ( !( pvt ? isEmptyDataObject : jQuery.isEmptyObject )( thisCache ) ) { + return; + } + } + } + + // See jQuery.data for more information + if ( !pvt ) { + delete cache[ id ].data; + + // Don't destroy the parent cache unless the internal data object + // had been the only thing left in it + if ( !isEmptyDataObject( cache[ id ] ) ) { + return; + } + } + + // Destroy the cache + if ( isNode ) { + jQuery.cleanData( [ elem ], true ); + + // Use delete when supported for expandos or `cache` is not a window per isWindow (#10080) + } else if ( jQuery.support.deleteExpando || cache != cache.window ) { + delete cache[ id ]; + + // When all else fails, null + } else { + cache[ id ] = null; + } + }, + + // For internal use only. + _data: function( elem, name, data ) { + return jQuery.data( elem, name, data, true ); + }, + + // A method for determining if a DOM node can handle the data expando + acceptData: function( elem ) { + var noData = elem.nodeName && jQuery.noData[ elem.nodeName.toLowerCase() ]; + + // nodes accept data unless otherwise specified; rejection can be conditional + return !noData || noData !== true && elem.getAttribute("classid") === noData; + } +}); + +jQuery.fn.extend({ + data: function( key, value ) { + var parts, part, attr, name, l, + elem = this[0], + i = 0, + data = null; + + // Gets all values + if ( key === undefined ) { + if ( this.length ) { + data = jQuery.data( elem ); + + if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) { + attr = elem.attributes; + for ( l = attr.length; i < l; i++ ) { + name = attr[i].name; + + if ( !name.indexOf( "data-" ) ) { + name = jQuery.camelCase( name.substring(5) ); + + dataAttr( elem, name, data[ name ] ); + } + } + jQuery._data( elem, "parsedAttrs", true ); + } + } + + return data; + } + + // Sets multiple values + if ( typeof key === "object" ) { + return this.each(function() { + jQuery.data( this, key ); + }); + } + + parts = key.split( ".", 2 ); + parts[1] = parts[1] ? "." + parts[1] : ""; + part = parts[1] + "!"; + + return jQuery.access( this, function( value ) { + + if ( value === undefined ) { + data = this.triggerHandler( "getData" + part, [ parts[0] ] ); + + // Try to fetch any internally stored data first + if ( data === undefined && elem ) { + data = jQuery.data( elem, key ); + data = dataAttr( elem, key, data ); + } + + return data === undefined && parts[1] ? + this.data( parts[0] ) : + data; + } + + parts[1] = value; + this.each(function() { + var self = jQuery( this ); + + self.triggerHandler( "setData" + part, parts ); + jQuery.data( this, key, value ); + self.triggerHandler( "changeData" + part, parts ); + }); + }, null, value, arguments.length > 1, null, false ); + }, + + removeData: function( key ) { + return this.each(function() { + jQuery.removeData( this, key ); + }); + } +}); + +function dataAttr( elem, key, data ) { + // If nothing was found internally, try to fetch any + // data from the HTML5 data-* attribute + if ( data === undefined && elem.nodeType === 1 ) { + + var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase(); + + data = elem.getAttribute( name ); + + if ( typeof data === "string" ) { + try { + data = data === "true" ? true : + data === "false" ? false : + data === "null" ? null : + // Only convert to a number if it doesn't change the string + +data + "" === data ? +data : + rbrace.test( data ) ? jQuery.parseJSON( data ) : + data; + } catch( e ) {} + + // Make sure we set the data so it isn't changed later + jQuery.data( elem, key, data ); + + } else { + data = undefined; + } + } + + return data; +} + +// checks a cache object for emptiness +function isEmptyDataObject( obj ) { + var name; + for ( name in obj ) { + + // if the public data object is empty, the private is still empty + if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) { + continue; + } + if ( name !== "toJSON" ) { + return false; + } + } + + return true; +} +jQuery.extend({ + queue: function( elem, type, data ) { + var queue; + + if ( elem ) { + type = ( type || "fx" ) + "queue"; + queue = jQuery._data( elem, type ); + + // Speed up dequeue by getting out quickly if this is just a lookup + if ( data ) { + if ( !queue || jQuery.isArray(data) ) { + queue = jQuery._data( elem, type, jQuery.makeArray(data) ); + } else { + queue.push( data ); + } + } + return queue || []; + } + }, + + dequeue: function( elem, type ) { + type = type || "fx"; + + var queue = jQuery.queue( elem, type ), + startLength = queue.length, + fn = queue.shift(), + hooks = jQuery._queueHooks( elem, type ), + next = function() { + jQuery.dequeue( elem, type ); + }; + + // If the fx queue is dequeued, always remove the progress sentinel + if ( fn === "inprogress" ) { + fn = queue.shift(); + startLength--; + } + + if ( fn ) { + + // Add a progress sentinel to prevent the fx queue from being + // automatically dequeued + if ( type === "fx" ) { + queue.unshift( "inprogress" ); + } + + // clear up the last queue stop function + delete hooks.stop; + fn.call( elem, next, hooks ); + } + + if ( !startLength && hooks ) { + hooks.empty.fire(); + } + }, + + // not intended for public consumption - generates a queueHooks object, or returns the current one + _queueHooks: function( elem, type ) { + var key = type + "queueHooks"; + return jQuery._data( elem, key ) || jQuery._data( elem, key, { + empty: jQuery.Callbacks("once memory").add(function() { + jQuery.removeData( elem, type + "queue", true ); + jQuery.removeData( elem, key, true ); + }) + }); + } +}); + +jQuery.fn.extend({ + queue: function( type, data ) { + var setter = 2; + + if ( typeof type !== "string" ) { + data = type; + type = "fx"; + setter--; + } + + if ( arguments.length < setter ) { + return jQuery.queue( this[0], type ); + } + + return data === undefined ? + this : + this.each(function() { + var queue = jQuery.queue( this, type, data ); + + // ensure a hooks for this queue + jQuery._queueHooks( this, type ); + + if ( type === "fx" && queue[0] !== "inprogress" ) { + jQuery.dequeue( this, type ); + } + }); + }, + dequeue: function( type ) { + return this.each(function() { + jQuery.dequeue( this, type ); + }); + }, + // Based off of the plugin by Clint Helfers, with permission. + // http://blindsignals.com/index.php/2009/07/jquery-delay/ + delay: function( time, type ) { + time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time; + type = type || "fx"; + + return this.queue( type, function( next, hooks ) { + var timeout = setTimeout( next, time ); + hooks.stop = function() { + clearTimeout( timeout ); + }; + }); + }, + clearQueue: function( type ) { + return this.queue( type || "fx", [] ); + }, + // Get a promise resolved when queues of a certain type + // are emptied (fx is the type by default) + promise: function( type, obj ) { + var tmp, + count = 1, + defer = jQuery.Deferred(), + elements = this, + i = this.length, + resolve = function() { + if ( !( --count ) ) { + defer.resolveWith( elements, [ elements ] ); + } + }; + + if ( typeof type !== "string" ) { + obj = type; + type = undefined; + } + type = type || "fx"; + + while( i-- ) { + tmp = jQuery._data( elements[ i ], type + "queueHooks" ); + if ( tmp && tmp.empty ) { + count++; + tmp.empty.add( resolve ); + } + } + resolve(); + return defer.promise( obj ); + } +}); +var nodeHook, boolHook, fixSpecified, + rclass = /[\t\r\n]/g, + rreturn = /\r/g, + rtype = /^(?:button|input)$/i, + rfocusable = /^(?:button|input|object|select|textarea)$/i, + rclickable = /^a(?:rea|)$/i, + rboolean = /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i, + getSetAttribute = jQuery.support.getSetAttribute; + +jQuery.fn.extend({ + attr: function( name, value ) { + return jQuery.access( this, jQuery.attr, name, value, arguments.length > 1 ); + }, + + removeAttr: function( name ) { + return this.each(function() { + jQuery.removeAttr( this, name ); + }); + }, + + prop: function( name, value ) { + return jQuery.access( this, jQuery.prop, name, value, arguments.length > 1 ); + }, + + removeProp: function( name ) { + name = jQuery.propFix[ name ] || name; + return this.each(function() { + // try/catch handles cases where IE balks (such as removing a property on window) + try { + this[ name ] = undefined; + delete this[ name ]; + } catch( e ) {} + }); + }, + + addClass: function( value ) { + var classNames, i, l, elem, + setClass, c, cl; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( j ) { + jQuery( this ).addClass( value.call(this, j, this.className) ); + }); + } + + if ( value && typeof value === "string" ) { + classNames = value.split( core_rspace ); + + for ( i = 0, l = this.length; i < l; i++ ) { + elem = this[ i ]; + + if ( elem.nodeType === 1 ) { + if ( !elem.className && classNames.length === 1 ) { + elem.className = value; + + } else { + setClass = " " + elem.className + " "; + + for ( c = 0, cl = classNames.length; c < cl; c++ ) { + if ( setClass.indexOf( " " + classNames[ c ] + " " ) < 0 ) { + setClass += classNames[ c ] + " "; + } + } + elem.className = jQuery.trim( setClass ); + } + } + } + } + + return this; + }, + + removeClass: function( value ) { + var removes, className, elem, c, cl, i, l; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( j ) { + jQuery( this ).removeClass( value.call(this, j, this.className) ); + }); + } + if ( (value && typeof value === "string") || value === undefined ) { + removes = ( value || "" ).split( core_rspace ); + + for ( i = 0, l = this.length; i < l; i++ ) { + elem = this[ i ]; + if ( elem.nodeType === 1 && elem.className ) { + + className = (" " + elem.className + " ").replace( rclass, " " ); + + // loop over each item in the removal list + for ( c = 0, cl = removes.length; c < cl; c++ ) { + // Remove until there is nothing to remove, + while ( className.indexOf(" " + removes[ c ] + " ") >= 0 ) { + className = className.replace( " " + removes[ c ] + " " , " " ); + } + } + elem.className = value ? jQuery.trim( className ) : ""; + } + } + } + + return this; + }, + + toggleClass: function( value, stateVal ) { + var type = typeof value, + isBool = typeof stateVal === "boolean"; + + if ( jQuery.isFunction( value ) ) { + return this.each(function( i ) { + jQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal ); + }); + } + + return this.each(function() { + if ( type === "string" ) { + // toggle individual class names + var className, + i = 0, + self = jQuery( this ), + state = stateVal, + classNames = value.split( core_rspace ); + + while ( (className = classNames[ i++ ]) ) { + // check each className given, space separated list + state = isBool ? state : !self.hasClass( className ); + self[ state ? "addClass" : "removeClass" ]( className ); + } + + } else if ( type === "undefined" || type === "boolean" ) { + if ( this.className ) { + // store className if set + jQuery._data( this, "__className__", this.className ); + } + + // toggle whole className + this.className = this.className || value === false ? "" : jQuery._data( this, "__className__" ) || ""; + } + }); + }, + + hasClass: function( selector ) { + var className = " " + selector + " ", + i = 0, + l = this.length; + for ( ; i < l; i++ ) { + if ( this[i].nodeType === 1 && (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) >= 0 ) { + return true; + } + } + + return false; + }, + + val: function( value ) { + var hooks, ret, isFunction, + elem = this[0]; + + if ( !arguments.length ) { + if ( elem ) { + hooks = jQuery.valHooks[ elem.type ] || jQuery.valHooks[ elem.nodeName.toLowerCase() ]; + + if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) { + return ret; + } + + ret = elem.value; + + return typeof ret === "string" ? + // handle most common string cases + ret.replace(rreturn, "") : + // handle cases where value is null/undef or number + ret == null ? "" : ret; + } + + return; + } + + isFunction = jQuery.isFunction( value ); + + return this.each(function( i ) { + var val, + self = jQuery(this); + + if ( this.nodeType !== 1 ) { + return; + } + + if ( isFunction ) { + val = value.call( this, i, self.val() ); + } else { + val = value; + } + + // Treat null/undefined as ""; convert numbers to string + if ( val == null ) { + val = ""; + } else if ( typeof val === "number" ) { + val += ""; + } else if ( jQuery.isArray( val ) ) { + val = jQuery.map(val, function ( value ) { + return value == null ? "" : value + ""; + }); + } + + hooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ]; + + // If set returns undefined, fall back to normal setting + if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) { + this.value = val; + } + }); + } +}); + +jQuery.extend({ + valHooks: { + option: { + get: function( elem ) { + // attributes.value is undefined in Blackberry 4.7 but + // uses .value. See #6932 + var val = elem.attributes.value; + return !val || val.specified ? elem.value : elem.text; + } + }, + select: { + get: function( elem ) { + var value, option, + options = elem.options, + index = elem.selectedIndex, + one = elem.type === "select-one" || index < 0, + values = one ? null : [], + max = one ? index + 1 : options.length, + i = index < 0 ? + max : + one ? index : 0; + + // Loop through all the selected options + for ( ; i < max; i++ ) { + option = options[ i ]; + + // oldIE doesn't update selected after form reset (#2551) + if ( ( option.selected || i === index ) && + // Don't return options that are disabled or in a disabled optgroup + ( jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null ) && + ( !option.parentNode.disabled || !jQuery.nodeName( option.parentNode, "optgroup" ) ) ) { + + // Get the specific value for the option + value = jQuery( option ).val(); + + // We don't need an array for one selects + if ( one ) { + return value; + } + + // Multi-Selects return an array + values.push( value ); + } + } + + return values; + }, + + set: function( elem, value ) { + var values = jQuery.makeArray( value ); + + jQuery(elem).find("option").each(function() { + this.selected = jQuery.inArray( jQuery(this).val(), values ) >= 0; + }); + + if ( !values.length ) { + elem.selectedIndex = -1; + } + return values; + } + } + }, + + // Unused in 1.8, left in so attrFn-stabbers won't die; remove in 1.9 + attrFn: {}, + + attr: function( elem, name, value, pass ) { + var ret, hooks, notxml, + nType = elem.nodeType; + + // don't get/set attributes on text, comment and attribute nodes + if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { + return; + } + + if ( pass && jQuery.isFunction( jQuery.fn[ name ] ) ) { + return jQuery( elem )[ name ]( value ); + } + + // Fallback to prop when attributes are not supported + if ( typeof elem.getAttribute === "undefined" ) { + return jQuery.prop( elem, name, value ); + } + + notxml = nType !== 1 || !jQuery.isXMLDoc( elem ); + + // All attributes are lowercase + // Grab necessary hook if one is defined + if ( notxml ) { + name = name.toLowerCase(); + hooks = jQuery.attrHooks[ name ] || ( rboolean.test( name ) ? boolHook : nodeHook ); + } + + if ( value !== undefined ) { + + if ( value === null ) { + jQuery.removeAttr( elem, name ); + return; + + } else if ( hooks && "set" in hooks && notxml && (ret = hooks.set( elem, value, name )) !== undefined ) { + return ret; + + } else { + elem.setAttribute( name, value + "" ); + return value; + } + + } else if ( hooks && "get" in hooks && notxml && (ret = hooks.get( elem, name )) !== null ) { + return ret; + + } else { + + ret = elem.getAttribute( name ); + + // Non-existent attributes return null, we normalize to undefined + return ret === null ? + undefined : + ret; + } + }, + + removeAttr: function( elem, value ) { + var propName, attrNames, name, isBool, + i = 0; + + if ( value && elem.nodeType === 1 ) { + + attrNames = value.split( core_rspace ); + + for ( ; i < attrNames.length; i++ ) { + name = attrNames[ i ]; + + if ( name ) { + propName = jQuery.propFix[ name ] || name; + isBool = rboolean.test( name ); + + // See #9699 for explanation of this approach (setting first, then removal) + // Do not do this for boolean attributes (see #10870) + if ( !isBool ) { + jQuery.attr( elem, name, "" ); + } + elem.removeAttribute( getSetAttribute ? name : propName ); + + // Set corresponding property to false for boolean attributes + if ( isBool && propName in elem ) { + elem[ propName ] = false; + } + } + } + } + }, + + attrHooks: { + type: { + set: function( elem, value ) { + // We can't allow the type property to be changed (since it causes problems in IE) + if ( rtype.test( elem.nodeName ) && elem.parentNode ) { + jQuery.error( "type property can't be changed" ); + } else if ( !jQuery.support.radioValue && value === "radio" && jQuery.nodeName(elem, "input") ) { + // Setting the type on a radio button after the value resets the value in IE6-9 + // Reset value to it's default in case type is set after value + // This is for element creation + var val = elem.value; + elem.setAttribute( "type", value ); + if ( val ) { + elem.value = val; + } + return value; + } + } + }, + // Use the value property for back compat + // Use the nodeHook for button elements in IE6/7 (#1954) + value: { + get: function( elem, name ) { + if ( nodeHook && jQuery.nodeName( elem, "button" ) ) { + return nodeHook.get( elem, name ); + } + return name in elem ? + elem.value : + null; + }, + set: function( elem, value, name ) { + if ( nodeHook && jQuery.nodeName( elem, "button" ) ) { + return nodeHook.set( elem, value, name ); + } + // Does not return so that setAttribute is also used + elem.value = value; + } + } + }, + + propFix: { + tabindex: "tabIndex", + readonly: "readOnly", + "for": "htmlFor", + "class": "className", + maxlength: "maxLength", + cellspacing: "cellSpacing", + cellpadding: "cellPadding", + rowspan: "rowSpan", + colspan: "colSpan", + usemap: "useMap", + frameborder: "frameBorder", + contenteditable: "contentEditable" + }, + + prop: function( elem, name, value ) { + var ret, hooks, notxml, + nType = elem.nodeType; + + // don't get/set properties on text, comment and attribute nodes + if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { + return; + } + + notxml = nType !== 1 || !jQuery.isXMLDoc( elem ); + + if ( notxml ) { + // Fix name and attach hooks + name = jQuery.propFix[ name ] || name; + hooks = jQuery.propHooks[ name ]; + } + + if ( value !== undefined ) { + if ( hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) { + return ret; + + } else { + return ( elem[ name ] = value ); + } + + } else { + if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ) { + return ret; + + } else { + return elem[ name ]; + } + } + }, + + propHooks: { + tabIndex: { + get: function( elem ) { + // elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set + // http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/ + var attributeNode = elem.getAttributeNode("tabindex"); + + return attributeNode && attributeNode.specified ? + parseInt( attributeNode.value, 10 ) : + rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ? + 0 : + undefined; + } + } + } +}); + +// Hook for boolean attributes +boolHook = { + get: function( elem, name ) { + // Align boolean attributes with corresponding properties + // Fall back to attribute presence where some booleans are not supported + var attrNode, + property = jQuery.prop( elem, name ); + return property === true || typeof property !== "boolean" && ( attrNode = elem.getAttributeNode(name) ) && attrNode.nodeValue !== false ? + name.toLowerCase() : + undefined; + }, + set: function( elem, value, name ) { + var propName; + if ( value === false ) { + // Remove boolean attributes when set to false + jQuery.removeAttr( elem, name ); + } else { + // value is true since we know at this point it's type boolean and not false + // Set boolean attributes to the same name and set the DOM property + propName = jQuery.propFix[ name ] || name; + if ( propName in elem ) { + // Only set the IDL specifically if it already exists on the element + elem[ propName ] = true; + } + + elem.setAttribute( name, name.toLowerCase() ); + } + return name; + } +}; + +// IE6/7 do not support getting/setting some attributes with get/setAttribute +if ( !getSetAttribute ) { + + fixSpecified = { + name: true, + id: true, + coords: true + }; + + // Use this for any attribute in IE6/7 + // This fixes almost every IE6/7 issue + nodeHook = jQuery.valHooks.button = { + get: function( elem, name ) { + var ret; + ret = elem.getAttributeNode( name ); + return ret && ( fixSpecified[ name ] ? ret.value !== "" : ret.specified ) ? + ret.value : + undefined; + }, + set: function( elem, value, name ) { + // Set the existing or create a new attribute node + var ret = elem.getAttributeNode( name ); + if ( !ret ) { + ret = document.createAttribute( name ); + elem.setAttributeNode( ret ); + } + return ( ret.value = value + "" ); + } + }; + + // Set width and height to auto instead of 0 on empty string( Bug #8150 ) + // This is for removals + jQuery.each([ "width", "height" ], function( i, name ) { + jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], { + set: function( elem, value ) { + if ( value === "" ) { + elem.setAttribute( name, "auto" ); + return value; + } + } + }); + }); + + // Set contenteditable to false on removals(#10429) + // Setting to empty string throws an error as an invalid value + jQuery.attrHooks.contenteditable = { + get: nodeHook.get, + set: function( elem, value, name ) { + if ( value === "" ) { + value = "false"; + } + nodeHook.set( elem, value, name ); + } + }; +} + + +// Some attributes require a special call on IE +if ( !jQuery.support.hrefNormalized ) { + jQuery.each([ "href", "src", "width", "height" ], function( i, name ) { + jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], { + get: function( elem ) { + var ret = elem.getAttribute( name, 2 ); + return ret === null ? undefined : ret; + } + }); + }); +} + +if ( !jQuery.support.style ) { + jQuery.attrHooks.style = { + get: function( elem ) { + // Return undefined in the case of empty string + // Normalize to lowercase since IE uppercases css property names + return elem.style.cssText.toLowerCase() || undefined; + }, + set: function( elem, value ) { + return ( elem.style.cssText = value + "" ); + } + }; +} + +// Safari mis-reports the default selected property of an option +// Accessing the parent's selectedIndex property fixes it +if ( !jQuery.support.optSelected ) { + jQuery.propHooks.selected = jQuery.extend( jQuery.propHooks.selected, { + get: function( elem ) { + var parent = elem.parentNode; + + if ( parent ) { + parent.selectedIndex; + + // Make sure that it also works with optgroups, see #5701 + if ( parent.parentNode ) { + parent.parentNode.selectedIndex; + } + } + return null; + } + }); +} + +// IE6/7 call enctype encoding +if ( !jQuery.support.enctype ) { + jQuery.propFix.enctype = "encoding"; +} + +// Radios and checkboxes getter/setter +if ( !jQuery.support.checkOn ) { + jQuery.each([ "radio", "checkbox" ], function() { + jQuery.valHooks[ this ] = { + get: function( elem ) { + // Handle the case where in Webkit "" is returned instead of "on" if a value isn't specified + return elem.getAttribute("value") === null ? "on" : elem.value; + } + }; + }); +} +jQuery.each([ "radio", "checkbox" ], function() { + jQuery.valHooks[ this ] = jQuery.extend( jQuery.valHooks[ this ], { + set: function( elem, value ) { + if ( jQuery.isArray( value ) ) { + return ( elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0 ); + } + } + }); +}); +var rformElems = /^(?:textarea|input|select)$/i, + rtypenamespace = /^([^\.]*|)(?:\.(.+)|)$/, + rhoverHack = /(?:^|\s)hover(\.\S+|)\b/, + rkeyEvent = /^key/, + rmouseEvent = /^(?:mouse|contextmenu)|click/, + rfocusMorph = /^(?:focusinfocus|focusoutblur)$/, + hoverHack = function( events ) { + return jQuery.event.special.hover ? events : events.replace( rhoverHack, "mouseenter$1 mouseleave$1" ); + }; + +/* + * Helper functions for managing events -- not part of the public interface. + * Props to Dean Edwards' addEvent library for many of the ideas. + */ +jQuery.event = { + + add: function( elem, types, handler, data, selector ) { + + var elemData, eventHandle, events, + t, tns, type, namespaces, handleObj, + handleObjIn, handlers, special; + + // Don't attach events to noData or text/comment nodes (allow plain objects tho) + if ( elem.nodeType === 3 || elem.nodeType === 8 || !types || !handler || !(elemData = jQuery._data( elem )) ) { + return; + } + + // Caller can pass in an object of custom data in lieu of the handler + if ( handler.handler ) { + handleObjIn = handler; + handler = handleObjIn.handler; + selector = handleObjIn.selector; + } + + // Make sure that the handler has a unique ID, used to find/remove it later + if ( !handler.guid ) { + handler.guid = jQuery.guid++; + } + + // Init the element's event structure and main handler, if this is the first + events = elemData.events; + if ( !events ) { + elemData.events = events = {}; + } + eventHandle = elemData.handle; + if ( !eventHandle ) { + elemData.handle = eventHandle = function( e ) { + // Discard the second event of a jQuery.event.trigger() and + // when an event is called after a page has unloaded + return typeof jQuery !== "undefined" && (!e || jQuery.event.triggered !== e.type) ? + jQuery.event.dispatch.apply( eventHandle.elem, arguments ) : + undefined; + }; + // Add elem as a property of the handle fn to prevent a memory leak with IE non-native events + eventHandle.elem = elem; + } + + // Handle multiple events separated by a space + // jQuery(...).bind("mouseover mouseout", fn); + types = jQuery.trim( hoverHack(types) ).split( " " ); + for ( t = 0; t < types.length; t++ ) { + + tns = rtypenamespace.exec( types[t] ) || []; + type = tns[1]; + namespaces = ( tns[2] || "" ).split( "." ).sort(); + + // If event changes its type, use the special event handlers for the changed type + special = jQuery.event.special[ type ] || {}; + + // If selector defined, determine special event api type, otherwise given type + type = ( selector ? special.delegateType : special.bindType ) || type; + + // Update special based on newly reset type + special = jQuery.event.special[ type ] || {}; + + // handleObj is passed to all event handlers + handleObj = jQuery.extend({ + type: type, + origType: tns[1], + data: data, + handler: handler, + guid: handler.guid, + selector: selector, + needsContext: selector && jQuery.expr.match.needsContext.test( selector ), + namespace: namespaces.join(".") + }, handleObjIn ); + + // Init the event handler queue if we're the first + handlers = events[ type ]; + if ( !handlers ) { + handlers = events[ type ] = []; + handlers.delegateCount = 0; + + // Only use addEventListener/attachEvent if the special events handler returns false + if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { + // Bind the global event handler to the element + if ( elem.addEventListener ) { + elem.addEventListener( type, eventHandle, false ); + + } else if ( elem.attachEvent ) { + elem.attachEvent( "on" + type, eventHandle ); + } + } + } + + if ( special.add ) { + special.add.call( elem, handleObj ); + + if ( !handleObj.handler.guid ) { + handleObj.handler.guid = handler.guid; + } + } + + // Add to the element's handler list, delegates in front + if ( selector ) { + handlers.splice( handlers.delegateCount++, 0, handleObj ); + } else { + handlers.push( handleObj ); + } + + // Keep track of which events have ever been used, for event optimization + jQuery.event.global[ type ] = true; + } + + // Nullify elem to prevent memory leaks in IE + elem = null; + }, + + global: {}, + + // Detach an event or set of events from an element + remove: function( elem, types, handler, selector, mappedTypes ) { + + var t, tns, type, origType, namespaces, origCount, + j, events, special, eventType, handleObj, + elemData = jQuery.hasData( elem ) && jQuery._data( elem ); + + if ( !elemData || !(events = elemData.events) ) { + return; + } + + // Once for each type.namespace in types; type may be omitted + types = jQuery.trim( hoverHack( types || "" ) ).split(" "); + for ( t = 0; t < types.length; t++ ) { + tns = rtypenamespace.exec( types[t] ) || []; + type = origType = tns[1]; + namespaces = tns[2]; + + // Unbind all events (on this namespace, if provided) for the element + if ( !type ) { + for ( type in events ) { + jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); + } + continue; + } + + special = jQuery.event.special[ type ] || {}; + type = ( selector? special.delegateType : special.bindType ) || type; + eventType = events[ type ] || []; + origCount = eventType.length; + namespaces = namespaces ? new RegExp("(^|\\.)" + namespaces.split(".").sort().join("\\.(?:.*\\.|)") + "(\\.|$)") : null; + + // Remove matching events + for ( j = 0; j < eventType.length; j++ ) { + handleObj = eventType[ j ]; + + if ( ( mappedTypes || origType === handleObj.origType ) && + ( !handler || handler.guid === handleObj.guid ) && + ( !namespaces || namespaces.test( handleObj.namespace ) ) && + ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) { + eventType.splice( j--, 1 ); + + if ( handleObj.selector ) { + eventType.delegateCount--; + } + if ( special.remove ) { + special.remove.call( elem, handleObj ); + } + } + } + + // Remove generic event handler if we removed something and no more handlers exist + // (avoids potential for endless recursion during removal of special event handlers) + if ( eventType.length === 0 && origCount !== eventType.length ) { + if ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) { + jQuery.removeEvent( elem, type, elemData.handle ); + } + + delete events[ type ]; + } + } + + // Remove the expando if it's no longer used + if ( jQuery.isEmptyObject( events ) ) { + delete elemData.handle; + + // removeData also checks for emptiness and clears the expando if empty + // so use it instead of delete + jQuery.removeData( elem, "events", true ); + } + }, + + // Events that are safe to short-circuit if no handlers are attached. + // Native DOM events should not be added, they may have inline handlers. + customEvent: { + "getData": true, + "setData": true, + "changeData": true + }, + + trigger: function( event, data, elem, onlyHandlers ) { + // Don't do events on text and comment nodes + if ( elem && (elem.nodeType === 3 || elem.nodeType === 8) ) { + return; + } + + // Event object or event type + var cache, exclusive, i, cur, old, ontype, special, handle, eventPath, bubbleType, + type = event.type || event, + namespaces = []; + + // focus/blur morphs to focusin/out; ensure we're not firing them right now + if ( rfocusMorph.test( type + jQuery.event.triggered ) ) { + return; + } + + if ( type.indexOf( "!" ) >= 0 ) { + // Exclusive events trigger only for the exact event (no namespaces) + type = type.slice(0, -1); + exclusive = true; + } + + if ( type.indexOf( "." ) >= 0 ) { + // Namespaced trigger; create a regexp to match event type in handle() + namespaces = type.split("."); + type = namespaces.shift(); + namespaces.sort(); + } + + if ( (!elem || jQuery.event.customEvent[ type ]) && !jQuery.event.global[ type ] ) { + // No jQuery handlers for this event type, and it can't have inline handlers + return; + } + + // Caller can pass in an Event, Object, or just an event type string + event = typeof event === "object" ? + // jQuery.Event object + event[ jQuery.expando ] ? event : + // Object literal + new jQuery.Event( type, event ) : + // Just the event type (string) + new jQuery.Event( type ); + + event.type = type; + event.isTrigger = true; + event.exclusive = exclusive; + event.namespace = namespaces.join( "." ); + event.namespace_re = event.namespace? new RegExp("(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)") : null; + ontype = type.indexOf( ":" ) < 0 ? "on" + type : ""; + + // Handle a global trigger + if ( !elem ) { + + // TODO: Stop taunting the data cache; remove global events and always attach to document + cache = jQuery.cache; + for ( i in cache ) { + if ( cache[ i ].events && cache[ i ].events[ type ] ) { + jQuery.event.trigger( event, data, cache[ i ].handle.elem, true ); + } + } + return; + } + + // Clean up the event in case it is being reused + event.result = undefined; + if ( !event.target ) { + event.target = elem; + } + + // Clone any incoming data and prepend the event, creating the handler arg list + data = data != null ? jQuery.makeArray( data ) : []; + data.unshift( event ); + + // Allow special events to draw outside the lines + special = jQuery.event.special[ type ] || {}; + if ( special.trigger && special.trigger.apply( elem, data ) === false ) { + return; + } + + // Determine event propagation path in advance, per W3C events spec (#9951) + // Bubble up to document, then to window; watch for a global ownerDocument var (#9724) + eventPath = [[ elem, special.bindType || type ]]; + if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) { + + bubbleType = special.delegateType || type; + cur = rfocusMorph.test( bubbleType + type ) ? elem : elem.parentNode; + for ( old = elem; cur; cur = cur.parentNode ) { + eventPath.push([ cur, bubbleType ]); + old = cur; + } + + // Only add window if we got to document (e.g., not plain obj or detached DOM) + if ( old === (elem.ownerDocument || document) ) { + eventPath.push([ old.defaultView || old.parentWindow || window, bubbleType ]); + } + } + + // Fire handlers on the event path + for ( i = 0; i < eventPath.length && !event.isPropagationStopped(); i++ ) { + + cur = eventPath[i][0]; + event.type = eventPath[i][1]; + + handle = ( jQuery._data( cur, "events" ) || {} )[ event.type ] && jQuery._data( cur, "handle" ); + if ( handle ) { + handle.apply( cur, data ); + } + // Note that this is a bare JS function and not a jQuery handler + handle = ontype && cur[ ontype ]; + if ( handle && jQuery.acceptData( cur ) && handle.apply && handle.apply( cur, data ) === false ) { + event.preventDefault(); + } + } + event.type = type; + + // If nobody prevented the default action, do it now + if ( !onlyHandlers && !event.isDefaultPrevented() ) { + + if ( (!special._default || special._default.apply( elem.ownerDocument, data ) === false) && + !(type === "click" && jQuery.nodeName( elem, "a" )) && jQuery.acceptData( elem ) ) { + + // Call a native DOM method on the target with the same name name as the event. + // Can't use an .isFunction() check here because IE6/7 fails that test. + // Don't do default actions on window, that's where global variables be (#6170) + // IE<9 dies on focus/blur to hidden element (#1486) + if ( ontype && elem[ type ] && ((type !== "focus" && type !== "blur") || event.target.offsetWidth !== 0) && !jQuery.isWindow( elem ) ) { + + // Don't re-trigger an onFOO event when we call its FOO() method + old = elem[ ontype ]; + + if ( old ) { + elem[ ontype ] = null; + } + + // Prevent re-triggering of the same event, since we already bubbled it above + jQuery.event.triggered = type; + elem[ type ](); + jQuery.event.triggered = undefined; + + if ( old ) { + elem[ ontype ] = old; + } + } + } + } + + return event.result; + }, + + dispatch: function( event ) { + + // Make a writable jQuery.Event from the native event object + event = jQuery.event.fix( event || window.event ); + + var i, j, cur, ret, selMatch, matched, matches, handleObj, sel, related, + handlers = ( (jQuery._data( this, "events" ) || {} )[ event.type ] || []), + delegateCount = handlers.delegateCount, + args = core_slice.call( arguments ), + run_all = !event.exclusive && !event.namespace, + special = jQuery.event.special[ event.type ] || {}, + handlerQueue = []; + + // Use the fix-ed jQuery.Event rather than the (read-only) native event + args[0] = event; + event.delegateTarget = this; + + // Call the preDispatch hook for the mapped type, and let it bail if desired + if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { + return; + } + + // Determine handlers that should run if there are delegated events + // Avoid non-left-click bubbling in Firefox (#3861) + if ( delegateCount && !(event.button && event.type === "click") ) { + + for ( cur = event.target; cur != this; cur = cur.parentNode || this ) { + + // Don't process clicks (ONLY) on disabled elements (#6911, #8165, #11382, #11764) + if ( cur.disabled !== true || event.type !== "click" ) { + selMatch = {}; + matches = []; + for ( i = 0; i < delegateCount; i++ ) { + handleObj = handlers[ i ]; + sel = handleObj.selector; + + if ( selMatch[ sel ] === undefined ) { + selMatch[ sel ] = handleObj.needsContext ? + jQuery( sel, this ).index( cur ) >= 0 : + jQuery.find( sel, this, null, [ cur ] ).length; + } + if ( selMatch[ sel ] ) { + matches.push( handleObj ); + } + } + if ( matches.length ) { + handlerQueue.push({ elem: cur, matches: matches }); + } + } + } + } + + // Add the remaining (directly-bound) handlers + if ( handlers.length > delegateCount ) { + handlerQueue.push({ elem: this, matches: handlers.slice( delegateCount ) }); + } + + // Run delegates first; they may want to stop propagation beneath us + for ( i = 0; i < handlerQueue.length && !event.isPropagationStopped(); i++ ) { + matched = handlerQueue[ i ]; + event.currentTarget = matched.elem; + + for ( j = 0; j < matched.matches.length && !event.isImmediatePropagationStopped(); j++ ) { + handleObj = matched.matches[ j ]; + + // Triggered event must either 1) be non-exclusive and have no namespace, or + // 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace). + if ( run_all || (!event.namespace && !handleObj.namespace) || event.namespace_re && event.namespace_re.test( handleObj.namespace ) ) { + + event.data = handleObj.data; + event.handleObj = handleObj; + + ret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler ) + .apply( matched.elem, args ); + + if ( ret !== undefined ) { + event.result = ret; + if ( ret === false ) { + event.preventDefault(); + event.stopPropagation(); + } + } + } + } + } + + // Call the postDispatch hook for the mapped type + if ( special.postDispatch ) { + special.postDispatch.call( this, event ); + } + + return event.result; + }, + + // Includes some event props shared by KeyEvent and MouseEvent + // *** attrChange attrName relatedNode srcElement are not normalized, non-W3C, deprecated, will be removed in 1.8 *** + props: "attrChange attrName relatedNode srcElement altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "), + + fixHooks: {}, + + keyHooks: { + props: "char charCode key keyCode".split(" "), + filter: function( event, original ) { + + // Add which for key events + if ( event.which == null ) { + event.which = original.charCode != null ? original.charCode : original.keyCode; + } + + return event; + } + }, + + mouseHooks: { + props: "button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "), + filter: function( event, original ) { + var eventDoc, doc, body, + button = original.button, + fromElement = original.fromElement; + + // Calculate pageX/Y if missing and clientX/Y available + if ( event.pageX == null && original.clientX != null ) { + eventDoc = event.target.ownerDocument || document; + doc = eventDoc.documentElement; + body = eventDoc.body; + + event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 ); + event.pageY = original.clientY + ( doc && doc.scrollTop || body && body.scrollTop || 0 ) - ( doc && doc.clientTop || body && body.clientTop || 0 ); + } + + // Add relatedTarget, if necessary + if ( !event.relatedTarget && fromElement ) { + event.relatedTarget = fromElement === event.target ? original.toElement : fromElement; + } + + // Add which for click: 1 === left; 2 === middle; 3 === right + // Note: button is not normalized, so don't use it + if ( !event.which && button !== undefined ) { + event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) ); + } + + return event; + } + }, + + fix: function( event ) { + if ( event[ jQuery.expando ] ) { + return event; + } + + // Create a writable copy of the event object and normalize some properties + var i, prop, + originalEvent = event, + fixHook = jQuery.event.fixHooks[ event.type ] || {}, + copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props; + + event = jQuery.Event( originalEvent ); + + for ( i = copy.length; i; ) { + prop = copy[ --i ]; + event[ prop ] = originalEvent[ prop ]; + } + + // Fix target property, if necessary (#1925, IE 6/7/8 & Safari2) + if ( !event.target ) { + event.target = originalEvent.srcElement || document; + } + + // Target should not be a text node (#504, Safari) + if ( event.target.nodeType === 3 ) { + event.target = event.target.parentNode; + } + + // For mouse/key events, metaKey==false if it's undefined (#3368, #11328; IE6/7/8) + event.metaKey = !!event.metaKey; + + return fixHook.filter? fixHook.filter( event, originalEvent ) : event; + }, + + special: { + load: { + // Prevent triggered image.load events from bubbling to window.load + noBubble: true + }, + + focus: { + delegateType: "focusin" + }, + blur: { + delegateType: "focusout" + }, + + beforeunload: { + setup: function( data, namespaces, eventHandle ) { + // We only want to do this special case on windows + if ( jQuery.isWindow( this ) ) { + this.onbeforeunload = eventHandle; + } + }, + + teardown: function( namespaces, eventHandle ) { + if ( this.onbeforeunload === eventHandle ) { + this.onbeforeunload = null; + } + } + } + }, + + simulate: function( type, elem, event, bubble ) { + // Piggyback on a donor event to simulate a different one. + // Fake originalEvent to avoid donor's stopPropagation, but if the + // simulated event prevents default then we do the same on the donor. + var e = jQuery.extend( + new jQuery.Event(), + event, + { type: type, + isSimulated: true, + originalEvent: {} + } + ); + if ( bubble ) { + jQuery.event.trigger( e, null, elem ); + } else { + jQuery.event.dispatch.call( elem, e ); + } + if ( e.isDefaultPrevented() ) { + event.preventDefault(); + } + } +}; + +// Some plugins are using, but it's undocumented/deprecated and will be removed. +// The 1.7 special event interface should provide all the hooks needed now. +jQuery.event.handle = jQuery.event.dispatch; + +jQuery.removeEvent = document.removeEventListener ? + function( elem, type, handle ) { + if ( elem.removeEventListener ) { + elem.removeEventListener( type, handle, false ); + } + } : + function( elem, type, handle ) { + var name = "on" + type; + + if ( elem.detachEvent ) { + + // #8545, #7054, preventing memory leaks for custom events in IE6-8 + // detachEvent needed property on element, by name of that event, to properly expose it to GC + if ( typeof elem[ name ] === "undefined" ) { + elem[ name ] = null; + } + + elem.detachEvent( name, handle ); + } + }; + +jQuery.Event = function( src, props ) { + // Allow instantiation without the 'new' keyword + if ( !(this instanceof jQuery.Event) ) { + return new jQuery.Event( src, props ); + } + + // Event object + if ( src && src.type ) { + this.originalEvent = src; + this.type = src.type; + + // Events bubbling up the document may have been marked as prevented + // by a handler lower down the tree; reflect the correct value. + this.isDefaultPrevented = ( src.defaultPrevented || src.returnValue === false || + src.getPreventDefault && src.getPreventDefault() ) ? returnTrue : returnFalse; + + // Event type + } else { + this.type = src; + } + + // Put explicitly provided properties onto the event object + if ( props ) { + jQuery.extend( this, props ); + } + + // Create a timestamp if incoming event doesn't have one + this.timeStamp = src && src.timeStamp || jQuery.now(); + + // Mark it as fixed + this[ jQuery.expando ] = true; +}; + +function returnFalse() { + return false; +} +function returnTrue() { + return true; +} + +// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding +// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html +jQuery.Event.prototype = { + preventDefault: function() { + this.isDefaultPrevented = returnTrue; + + var e = this.originalEvent; + if ( !e ) { + return; + } + + // if preventDefault exists run it on the original event + if ( e.preventDefault ) { + e.preventDefault(); + + // otherwise set the returnValue property of the original event to false (IE) + } else { + e.returnValue = false; + } + }, + stopPropagation: function() { + this.isPropagationStopped = returnTrue; + + var e = this.originalEvent; + if ( !e ) { + return; + } + // if stopPropagation exists run it on the original event + if ( e.stopPropagation ) { + e.stopPropagation(); + } + // otherwise set the cancelBubble property of the original event to true (IE) + e.cancelBubble = true; + }, + stopImmediatePropagation: function() { + this.isImmediatePropagationStopped = returnTrue; + this.stopPropagation(); + }, + isDefaultPrevented: returnFalse, + isPropagationStopped: returnFalse, + isImmediatePropagationStopped: returnFalse +}; + +// Create mouseenter/leave events using mouseover/out and event-time checks +jQuery.each({ + mouseenter: "mouseover", + mouseleave: "mouseout" +}, function( orig, fix ) { + jQuery.event.special[ orig ] = { + delegateType: fix, + bindType: fix, + + handle: function( event ) { + var ret, + target = this, + related = event.relatedTarget, + handleObj = event.handleObj, + selector = handleObj.selector; + + // For mousenter/leave call the handler if related is outside the target. + // NB: No relatedTarget if the mouse left/entered the browser window + if ( !related || (related !== target && !jQuery.contains( target, related )) ) { + event.type = handleObj.origType; + ret = handleObj.handler.apply( this, arguments ); + event.type = fix; + } + return ret; + } + }; +}); + +// IE submit delegation +if ( !jQuery.support.submitBubbles ) { + + jQuery.event.special.submit = { + setup: function() { + // Only need this for delegated form submit events + if ( jQuery.nodeName( this, "form" ) ) { + return false; + } + + // Lazy-add a submit handler when a descendant form may potentially be submitted + jQuery.event.add( this, "click._submit keypress._submit", function( e ) { + // Node name check avoids a VML-related crash in IE (#9807) + var elem = e.target, + form = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.form : undefined; + if ( form && !jQuery._data( form, "_submit_attached" ) ) { + jQuery.event.add( form, "submit._submit", function( event ) { + event._submit_bubble = true; + }); + jQuery._data( form, "_submit_attached", true ); + } + }); + // return undefined since we don't need an event listener + }, + + postDispatch: function( event ) { + // If form was submitted by the user, bubble the event up the tree + if ( event._submit_bubble ) { + delete event._submit_bubble; + if ( this.parentNode && !event.isTrigger ) { + jQuery.event.simulate( "submit", this.parentNode, event, true ); + } + } + }, + + teardown: function() { + // Only need this for delegated form submit events + if ( jQuery.nodeName( this, "form" ) ) { + return false; + } + + // Remove delegated handlers; cleanData eventually reaps submit handlers attached above + jQuery.event.remove( this, "._submit" ); + } + }; +} + +// IE change delegation and checkbox/radio fix +if ( !jQuery.support.changeBubbles ) { + + jQuery.event.special.change = { + + setup: function() { + + if ( rformElems.test( this.nodeName ) ) { + // IE doesn't fire change on a check/radio until blur; trigger it on click + // after a propertychange. Eat the blur-change in special.change.handle. + // This still fires onchange a second time for check/radio after blur. + if ( this.type === "checkbox" || this.type === "radio" ) { + jQuery.event.add( this, "propertychange._change", function( event ) { + if ( event.originalEvent.propertyName === "checked" ) { + this._just_changed = true; + } + }); + jQuery.event.add( this, "click._change", function( event ) { + if ( this._just_changed && !event.isTrigger ) { + this._just_changed = false; + } + // Allow triggered, simulated change events (#11500) + jQuery.event.simulate( "change", this, event, true ); + }); + } + return false; + } + // Delegated event; lazy-add a change handler on descendant inputs + jQuery.event.add( this, "beforeactivate._change", function( e ) { + var elem = e.target; + + if ( rformElems.test( elem.nodeName ) && !jQuery._data( elem, "_change_attached" ) ) { + jQuery.event.add( elem, "change._change", function( event ) { + if ( this.parentNode && !event.isSimulated && !event.isTrigger ) { + jQuery.event.simulate( "change", this.parentNode, event, true ); + } + }); + jQuery._data( elem, "_change_attached", true ); + } + }); + }, + + handle: function( event ) { + var elem = event.target; + + // Swallow native change events from checkbox/radio, we already triggered them above + if ( this !== elem || event.isSimulated || event.isTrigger || (elem.type !== "radio" && elem.type !== "checkbox") ) { + return event.handleObj.handler.apply( this, arguments ); + } + }, + + teardown: function() { + jQuery.event.remove( this, "._change" ); + + return !rformElems.test( this.nodeName ); + } + }; +} + +// Create "bubbling" focus and blur events +if ( !jQuery.support.focusinBubbles ) { + jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) { + + // Attach a single capturing handler while someone wants focusin/focusout + var attaches = 0, + handler = function( event ) { + jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true ); + }; + + jQuery.event.special[ fix ] = { + setup: function() { + if ( attaches++ === 0 ) { + document.addEventListener( orig, handler, true ); + } + }, + teardown: function() { + if ( --attaches === 0 ) { + document.removeEventListener( orig, handler, true ); + } + } + }; + }); +} + +jQuery.fn.extend({ + + on: function( types, selector, data, fn, /*INTERNAL*/ one ) { + var origFn, type; + + // Types can be a map of types/handlers + if ( typeof types === "object" ) { + // ( types-Object, selector, data ) + if ( typeof selector !== "string" ) { // && selector != null + // ( types-Object, data ) + data = data || selector; + selector = undefined; + } + for ( type in types ) { + this.on( type, selector, data, types[ type ], one ); + } + return this; + } + + if ( data == null && fn == null ) { + // ( types, fn ) + fn = selector; + data = selector = undefined; + } else if ( fn == null ) { + if ( typeof selector === "string" ) { + // ( types, selector, fn ) + fn = data; + data = undefined; + } else { + // ( types, data, fn ) + fn = data; + data = selector; + selector = undefined; + } + } + if ( fn === false ) { + fn = returnFalse; + } else if ( !fn ) { + return this; + } + + if ( one === 1 ) { + origFn = fn; + fn = function( event ) { + // Can use an empty set, since event contains the info + jQuery().off( event ); + return origFn.apply( this, arguments ); + }; + // Use same guid so caller can remove using origFn + fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); + } + return this.each( function() { + jQuery.event.add( this, types, fn, data, selector ); + }); + }, + one: function( types, selector, data, fn ) { + return this.on( types, selector, data, fn, 1 ); + }, + off: function( types, selector, fn ) { + var handleObj, type; + if ( types && types.preventDefault && types.handleObj ) { + // ( event ) dispatched jQuery.Event + handleObj = types.handleObj; + jQuery( types.delegateTarget ).off( + handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType, + handleObj.selector, + handleObj.handler + ); + return this; + } + if ( typeof types === "object" ) { + // ( types-object [, selector] ) + for ( type in types ) { + this.off( type, selector, types[ type ] ); + } + return this; + } + if ( selector === false || typeof selector === "function" ) { + // ( types [, fn] ) + fn = selector; + selector = undefined; + } + if ( fn === false ) { + fn = returnFalse; + } + return this.each(function() { + jQuery.event.remove( this, types, fn, selector ); + }); + }, + + bind: function( types, data, fn ) { + return this.on( types, null, data, fn ); + }, + unbind: function( types, fn ) { + return this.off( types, null, fn ); + }, + + live: function( types, data, fn ) { + jQuery( this.context ).on( types, this.selector, data, fn ); + return this; + }, + die: function( types, fn ) { + jQuery( this.context ).off( types, this.selector || "**", fn ); + return this; + }, + + delegate: function( selector, types, data, fn ) { + return this.on( types, selector, data, fn ); + }, + undelegate: function( selector, types, fn ) { + // ( namespace ) or ( selector, types [, fn] ) + return arguments.length === 1 ? this.off( selector, "**" ) : this.off( types, selector || "**", fn ); + }, + + trigger: function( type, data ) { + return this.each(function() { + jQuery.event.trigger( type, data, this ); + }); + }, + triggerHandler: function( type, data ) { + if ( this[0] ) { + return jQuery.event.trigger( type, data, this[0], true ); + } + }, + + toggle: function( fn ) { + // Save reference to arguments for access in closure + var args = arguments, + guid = fn.guid || jQuery.guid++, + i = 0, + toggler = function( event ) { + // Figure out which function to execute + var lastToggle = ( jQuery._data( this, "lastToggle" + fn.guid ) || 0 ) % i; + jQuery._data( this, "lastToggle" + fn.guid, lastToggle + 1 ); + + // Make sure that clicks stop + event.preventDefault(); + + // and execute the function + return args[ lastToggle ].apply( this, arguments ) || false; + }; + + // link all the functions, so any of them can unbind this click handler + toggler.guid = guid; + while ( i < args.length ) { + args[ i++ ].guid = guid; + } + + return this.click( toggler ); + }, + + hover: function( fnOver, fnOut ) { + return this.mouseenter( fnOver ).mouseleave( fnOut || fnOver ); + } +}); + +jQuery.each( ("blur focus focusin focusout load resize scroll unload click dblclick " + + "mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave " + + "change select submit keydown keypress keyup error contextmenu").split(" "), function( i, name ) { + + // Handle event binding + jQuery.fn[ name ] = function( data, fn ) { + if ( fn == null ) { + fn = data; + data = null; + } + + return arguments.length > 0 ? + this.on( name, null, data, fn ) : + this.trigger( name ); + }; + + if ( rkeyEvent.test( name ) ) { + jQuery.event.fixHooks[ name ] = jQuery.event.keyHooks; + } + + if ( rmouseEvent.test( name ) ) { + jQuery.event.fixHooks[ name ] = jQuery.event.mouseHooks; + } +}); +/*! + * Sizzle CSS Selector Engine + * Copyright 2012 jQuery Foundation and other contributors + * Released under the MIT license + * http://sizzlejs.com/ + */ +(function( window, undefined ) { + +var cachedruns, + assertGetIdNotName, + Expr, + getText, + isXML, + contains, + compile, + sortOrder, + hasDuplicate, + outermostContext, + + baseHasDuplicate = true, + strundefined = "undefined", + + expando = ( "sizcache" + Math.random() ).replace( ".", "" ), + + Token = String, + document = window.document, + docElem = document.documentElement, + dirruns = 0, + done = 0, + pop = [].pop, + push = [].push, + slice = [].slice, + // Use a stripped-down indexOf if a native one is unavailable + indexOf = [].indexOf || function( elem ) { + var i = 0, + len = this.length; + for ( ; i < len; i++ ) { + if ( this[i] === elem ) { + return i; + } + } + return -1; + }, + + // Augment a function for special use by Sizzle + markFunction = function( fn, value ) { + fn[ expando ] = value == null || value; + return fn; + }, + + createCache = function() { + var cache = {}, + keys = []; + + return markFunction(function( key, value ) { + // Only keep the most recent entries + if ( keys.push( key ) > Expr.cacheLength ) { + delete cache[ keys.shift() ]; + } + + // Retrieve with (key + " ") to avoid collision with native Object.prototype properties (see Issue #157) + return (cache[ key + " " ] = value); + }, cache ); + }, + + classCache = createCache(), + tokenCache = createCache(), + compilerCache = createCache(), + + // Regex + + // Whitespace characters http://www.w3.org/TR/css3-selectors/#whitespace + whitespace = "[\\x20\\t\\r\\n\\f]", + // http://www.w3.org/TR/css3-syntax/#characters + characterEncoding = "(?:\\\\.|[-\\w]|[^\\x00-\\xa0])+", + + // Loosely modeled on CSS identifier characters + // An unquoted value should be a CSS identifier (http://www.w3.org/TR/css3-selectors/#attribute-selectors) + // Proper syntax: http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier + identifier = characterEncoding.replace( "w", "w#" ), + + // Acceptable operators http://www.w3.org/TR/selectors/#attribute-selectors + operators = "([*^$|!~]?=)", + attributes = "\\[" + whitespace + "*(" + characterEncoding + ")" + whitespace + + "*(?:" + operators + whitespace + "*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|(" + identifier + ")|)|)" + whitespace + "*\\]", + + // Prefer arguments not in parens/brackets, + // then attribute selectors and non-pseudos (denoted by :), + // then anything else + // These preferences are here to reduce the number of selectors + // needing tokenize in the PSEUDO preFilter + pseudos = ":(" + characterEncoding + ")(?:\\((?:(['\"])((?:\\\\.|[^\\\\])*?)\\2|([^()[\\]]*|(?:(?:" + attributes + ")|[^:]|\\\\.)*|.*))\\)|)", + + // For matchExpr.POS and matchExpr.needsContext + pos = ":(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + whitespace + + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", + + // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter + rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), + + rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), + rcombinators = new RegExp( "^" + whitespace + "*([\\x20\\t\\r\\n\\f>+~])" + whitespace + "*" ), + rpseudo = new RegExp( pseudos ), + + // Easily-parseable/retrievable ID or TAG or CLASS selectors + rquickExpr = /^(?:#([\w\-]+)|(\w+)|\.([\w\-]+))$/, + + rnot = /^:not/, + rsibling = /[\x20\t\r\n\f]*[+~]/, + rendsWithNot = /:not\($/, + + rheader = /h\d/i, + rinputs = /input|select|textarea|button/i, + + rbackslash = /\\(?!\\)/g, + + matchExpr = { + "ID": new RegExp( "^#(" + characterEncoding + ")" ), + "CLASS": new RegExp( "^\\.(" + characterEncoding + ")" ), + "NAME": new RegExp( "^\\[name=['\"]?(" + characterEncoding + ")['\"]?\\]" ), + "TAG": new RegExp( "^(" + characterEncoding.replace( "w", "w*" ) + ")" ), + "ATTR": new RegExp( "^" + attributes ), + "PSEUDO": new RegExp( "^" + pseudos ), + "POS": new RegExp( pos, "i" ), + "CHILD": new RegExp( "^:(only|nth|first|last)-child(?:\\(" + whitespace + + "*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace + + "*(\\d+)|))" + whitespace + "*\\)|)", "i" ), + // For use in libraries implementing .is() + "needsContext": new RegExp( "^" + whitespace + "*[>+~]|" + pos, "i" ) + }, + + // Support + + // Used for testing something on an element + assert = function( fn ) { + var div = document.createElement("div"); + + try { + return fn( div ); + } catch (e) { + return false; + } finally { + // release memory in IE + div = null; + } + }, + + // Check if getElementsByTagName("*") returns only elements + assertTagNameNoComments = assert(function( div ) { + div.appendChild( document.createComment("") ); + return !div.getElementsByTagName("*").length; + }), + + // Check if getAttribute returns normalized href attributes + assertHrefNotNormalized = assert(function( div ) { + div.innerHTML = "<a href='#'></a>"; + return div.firstChild && typeof div.firstChild.getAttribute !== strundefined && + div.firstChild.getAttribute("href") === "#"; + }), + + // Check if attributes should be retrieved by attribute nodes + assertAttributes = assert(function( div ) { + div.innerHTML = "<select></select>"; + var type = typeof div.lastChild.getAttribute("multiple"); + // IE8 returns a string for some attributes even when not present + return type !== "boolean" && type !== "string"; + }), + + // Check if getElementsByClassName can be trusted + assertUsableClassName = assert(function( div ) { + // Opera can't find a second classname (in 9.6) + div.innerHTML = "<div class='hidden e'></div><div class='hidden'></div>"; + if ( !div.getElementsByClassName || !div.getElementsByClassName("e").length ) { + return false; + } + + // Safari 3.2 caches class attributes and doesn't catch changes + div.lastChild.className = "e"; + return div.getElementsByClassName("e").length === 2; + }), + + // Check if getElementById returns elements by name + // Check if getElementsByName privileges form controls or returns elements by ID + assertUsableName = assert(function( div ) { + // Inject content + div.id = expando + 0; + div.innerHTML = "<a name='" + expando + "'></a><div name='" + expando + "'></div>"; + docElem.insertBefore( div, docElem.firstChild ); + + // Test + var pass = document.getElementsByName && + // buggy browsers will return fewer than the correct 2 + document.getElementsByName( expando ).length === 2 + + // buggy browsers will return more than the correct 0 + document.getElementsByName( expando + 0 ).length; + assertGetIdNotName = !document.getElementById( expando ); + + // Cleanup + docElem.removeChild( div ); + + return pass; + }); + +// If slice is not available, provide a backup +try { + slice.call( docElem.childNodes, 0 )[0].nodeType; +} catch ( e ) { + slice = function( i ) { + var elem, + results = []; + for ( ; (elem = this[i]); i++ ) { + results.push( elem ); + } + return results; + }; +} + +function Sizzle( selector, context, results, seed ) { + results = results || []; + context = context || document; + var match, elem, xml, m, + nodeType = context.nodeType; + + if ( !selector || typeof selector !== "string" ) { + return results; + } + + if ( nodeType !== 1 && nodeType !== 9 ) { + return []; + } + + xml = isXML( context ); + + if ( !xml && !seed ) { + if ( (match = rquickExpr.exec( selector )) ) { + // Speed-up: Sizzle("#ID") + if ( (m = match[1]) ) { + if ( nodeType === 9 ) { + elem = context.getElementById( m ); + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + if ( elem && elem.parentNode ) { + // Handle the case where IE, Opera, and Webkit return items + // by name instead of ID + if ( elem.id === m ) { + results.push( elem ); + return results; + } + } else { + return results; + } + } else { + // Context is not a document + if ( context.ownerDocument && (elem = context.ownerDocument.getElementById( m )) && + contains( context, elem ) && elem.id === m ) { + results.push( elem ); + return results; + } + } + + // Speed-up: Sizzle("TAG") + } else if ( match[2] ) { + push.apply( results, slice.call(context.getElementsByTagName( selector ), 0) ); + return results; + + // Speed-up: Sizzle(".CLASS") + } else if ( (m = match[3]) && assertUsableClassName && context.getElementsByClassName ) { + push.apply( results, slice.call(context.getElementsByClassName( m ), 0) ); + return results; + } + } + } + + // All others + return select( selector.replace( rtrim, "$1" ), context, results, seed, xml ); +} + +Sizzle.matches = function( expr, elements ) { + return Sizzle( expr, null, null, elements ); +}; + +Sizzle.matchesSelector = function( elem, expr ) { + return Sizzle( expr, null, null, [ elem ] ).length > 0; +}; + +// Returns a function to use in pseudos for input types +function createInputPseudo( type ) { + return function( elem ) { + var name = elem.nodeName.toLowerCase(); + return name === "input" && elem.type === type; + }; +} + +// Returns a function to use in pseudos for buttons +function createButtonPseudo( type ) { + return function( elem ) { + var name = elem.nodeName.toLowerCase(); + return (name === "input" || name === "button") && elem.type === type; + }; +} + +// Returns a function to use in pseudos for positionals +function createPositionalPseudo( fn ) { + return markFunction(function( argument ) { + argument = +argument; + return markFunction(function( seed, matches ) { + var j, + matchIndexes = fn( [], seed.length, argument ), + i = matchIndexes.length; + + // Match elements found at the specified indexes + while ( i-- ) { + if ( seed[ (j = matchIndexes[i]) ] ) { + seed[j] = !(matches[j] = seed[j]); + } + } + }); + }); +} + +/** + * Utility function for retrieving the text value of an array of DOM nodes + * @param {Array|Element} elem + */ +getText = Sizzle.getText = function( elem ) { + var node, + ret = "", + i = 0, + nodeType = elem.nodeType; + + if ( nodeType ) { + if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) { + // Use textContent for elements + // innerText usage removed for consistency of new lines (see #11153) + if ( typeof elem.textContent === "string" ) { + return elem.textContent; + } else { + // Traverse its children + for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { + ret += getText( elem ); + } + } + } else if ( nodeType === 3 || nodeType === 4 ) { + return elem.nodeValue; + } + // Do not include comment or processing instruction nodes + } else { + + // If no nodeType, this is expected to be an array + for ( ; (node = elem[i]); i++ ) { + // Do not traverse comment nodes + ret += getText( node ); + } + } + return ret; +}; + +isXML = Sizzle.isXML = function( elem ) { + // documentElement is verified for cases where it doesn't yet exist + // (such as loading iframes in IE - #4833) + var documentElement = elem && (elem.ownerDocument || elem).documentElement; + return documentElement ? documentElement.nodeName !== "HTML" : false; +}; + +// Element contains another +contains = Sizzle.contains = docElem.contains ? + function( a, b ) { + var adown = a.nodeType === 9 ? a.documentElement : a, + bup = b && b.parentNode; + return a === bup || !!( bup && bup.nodeType === 1 && adown.contains && adown.contains(bup) ); + } : + docElem.compareDocumentPosition ? + function( a, b ) { + return b && !!( a.compareDocumentPosition( b ) & 16 ); + } : + function( a, b ) { + while ( (b = b.parentNode) ) { + if ( b === a ) { + return true; + } + } + return false; + }; + +Sizzle.attr = function( elem, name ) { + var val, + xml = isXML( elem ); + + if ( !xml ) { + name = name.toLowerCase(); + } + if ( (val = Expr.attrHandle[ name ]) ) { + return val( elem ); + } + if ( xml || assertAttributes ) { + return elem.getAttribute( name ); + } + val = elem.getAttributeNode( name ); + return val ? + typeof elem[ name ] === "boolean" ? + elem[ name ] ? name : null : + val.specified ? val.value : null : + null; +}; + +Expr = Sizzle.selectors = { + + // Can be adjusted by the user + cacheLength: 50, + + createPseudo: markFunction, + + match: matchExpr, + + // IE6/7 return a modified href + attrHandle: assertHrefNotNormalized ? + {} : + { + "href": function( elem ) { + return elem.getAttribute( "href", 2 ); + }, + "type": function( elem ) { + return elem.getAttribute("type"); + } + }, + + find: { + "ID": assertGetIdNotName ? + function( id, context, xml ) { + if ( typeof context.getElementById !== strundefined && !xml ) { + var m = context.getElementById( id ); + // Check parentNode to catch when Blackberry 4.6 returns + // nodes that are no longer in the document #6963 + return m && m.parentNode ? [m] : []; + } + } : + function( id, context, xml ) { + if ( typeof context.getElementById !== strundefined && !xml ) { + var m = context.getElementById( id ); + + return m ? + m.id === id || typeof m.getAttributeNode !== strundefined && m.getAttributeNode("id").value === id ? + [m] : + undefined : + []; + } + }, + + "TAG": assertTagNameNoComments ? + function( tag, context ) { + if ( typeof context.getElementsByTagName !== strundefined ) { + return context.getElementsByTagName( tag ); + } + } : + function( tag, context ) { + var results = context.getElementsByTagName( tag ); + + // Filter out possible comments + if ( tag === "*" ) { + var elem, + tmp = [], + i = 0; + + for ( ; (elem = results[i]); i++ ) { + if ( elem.nodeType === 1 ) { + tmp.push( elem ); + } + } + + return tmp; + } + return results; + }, + + "NAME": assertUsableName && function( tag, context ) { + if ( typeof context.getElementsByName !== strundefined ) { + return context.getElementsByName( name ); + } + }, + + "CLASS": assertUsableClassName && function( className, context, xml ) { + if ( typeof context.getElementsByClassName !== strundefined && !xml ) { + return context.getElementsByClassName( className ); + } + } + }, + + relative: { + ">": { dir: "parentNode", first: true }, + " ": { dir: "parentNode" }, + "+": { dir: "previousSibling", first: true }, + "~": { dir: "previousSibling" } + }, + + preFilter: { + "ATTR": function( match ) { + match[1] = match[1].replace( rbackslash, "" ); + + // Move the given value to match[3] whether quoted or unquoted + match[3] = ( match[4] || match[5] || "" ).replace( rbackslash, "" ); + + if ( match[2] === "~=" ) { + match[3] = " " + match[3] + " "; + } + + return match.slice( 0, 4 ); + }, + + "CHILD": function( match ) { + /* matches from matchExpr["CHILD"] + 1 type (only|nth|...) + 2 argument (even|odd|\d*|\d*n([+-]\d+)?|...) + 3 xn-component of xn+y argument ([+-]?\d*n|) + 4 sign of xn-component + 5 x of xn-component + 6 sign of y-component + 7 y of y-component + */ + match[1] = match[1].toLowerCase(); + + if ( match[1] === "nth" ) { + // nth-child requires argument + if ( !match[2] ) { + Sizzle.error( match[0] ); + } + + // numeric x and y parameters for Expr.filter.CHILD + // remember that false/true cast respectively to 0/1 + match[3] = +( match[3] ? match[4] + (match[5] || 1) : 2 * ( match[2] === "even" || match[2] === "odd" ) ); + match[4] = +( ( match[6] + match[7] ) || match[2] === "odd" ); + + // other types prohibit arguments + } else if ( match[2] ) { + Sizzle.error( match[0] ); + } + + return match; + }, + + "PSEUDO": function( match ) { + var unquoted, excess; + if ( matchExpr["CHILD"].test( match[0] ) ) { + return null; + } + + if ( match[3] ) { + match[2] = match[3]; + } else if ( (unquoted = match[4]) ) { + // Only check arguments that contain a pseudo + if ( rpseudo.test(unquoted) && + // Get excess from tokenize (recursively) + (excess = tokenize( unquoted, true )) && + // advance to the next closing parenthesis + (excess = unquoted.indexOf( ")", unquoted.length - excess ) - unquoted.length) ) { + + // excess is a negative index + unquoted = unquoted.slice( 0, excess ); + match[0] = match[0].slice( 0, excess ); + } + match[2] = unquoted; + } + + // Return only captures needed by the pseudo filter method (type and argument) + return match.slice( 0, 3 ); + } + }, + + filter: { + "ID": assertGetIdNotName ? + function( id ) { + id = id.replace( rbackslash, "" ); + return function( elem ) { + return elem.getAttribute("id") === id; + }; + } : + function( id ) { + id = id.replace( rbackslash, "" ); + return function( elem ) { + var node = typeof elem.getAttributeNode !== strundefined && elem.getAttributeNode("id"); + return node && node.value === id; + }; + }, + + "TAG": function( nodeName ) { + if ( nodeName === "*" ) { + return function() { return true; }; + } + nodeName = nodeName.replace( rbackslash, "" ).toLowerCase(); + + return function( elem ) { + return elem.nodeName && elem.nodeName.toLowerCase() === nodeName; + }; + }, + + "CLASS": function( className ) { + var pattern = classCache[ expando ][ className + " " ]; + + return pattern || + (pattern = new RegExp( "(^|" + whitespace + ")" + className + "(" + whitespace + "|$)" )) && + classCache( className, function( elem ) { + return pattern.test( elem.className || (typeof elem.getAttribute !== strundefined && elem.getAttribute("class")) || "" ); + }); + }, + + "ATTR": function( name, operator, check ) { + return function( elem, context ) { + var result = Sizzle.attr( elem, name ); + + if ( result == null ) { + return operator === "!="; + } + if ( !operator ) { + return true; + } + + result += ""; + + return operator === "=" ? result === check : + operator === "!=" ? result !== check : + operator === "^=" ? check && result.indexOf( check ) === 0 : + operator === "*=" ? check && result.indexOf( check ) > -1 : + operator === "$=" ? check && result.substr( result.length - check.length ) === check : + operator === "~=" ? ( " " + result + " " ).indexOf( check ) > -1 : + operator === "|=" ? result === check || result.substr( 0, check.length + 1 ) === check + "-" : + false; + }; + }, + + "CHILD": function( type, argument, first, last ) { + + if ( type === "nth" ) { + return function( elem ) { + var node, diff, + parent = elem.parentNode; + + if ( first === 1 && last === 0 ) { + return true; + } + + if ( parent ) { + diff = 0; + for ( node = parent.firstChild; node; node = node.nextSibling ) { + if ( node.nodeType === 1 ) { + diff++; + if ( elem === node ) { + break; + } + } + } + } + + // Incorporate the offset (or cast to NaN), then check against cycle size + diff -= last; + return diff === first || ( diff % first === 0 && diff / first >= 0 ); + }; + } + + return function( elem ) { + var node = elem; + + switch ( type ) { + case "only": + case "first": + while ( (node = node.previousSibling) ) { + if ( node.nodeType === 1 ) { + return false; + } + } + + if ( type === "first" ) { + return true; + } + + node = elem; + + /* falls through */ + case "last": + while ( (node = node.nextSibling) ) { + if ( node.nodeType === 1 ) { + return false; + } + } + + return true; + } + }; + }, + + "PSEUDO": function( pseudo, argument ) { + // pseudo-class names are case-insensitive + // http://www.w3.org/TR/selectors/#pseudo-classes + // Prioritize by case sensitivity in case custom pseudos are added with uppercase letters + // Remember that setFilters inherits from pseudos + var args, + fn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] || + Sizzle.error( "unsupported pseudo: " + pseudo ); + + // The user may use createPseudo to indicate that + // arguments are needed to create the filter function + // just as Sizzle does + if ( fn[ expando ] ) { + return fn( argument ); + } + + // But maintain support for old signatures + if ( fn.length > 1 ) { + args = [ pseudo, pseudo, "", argument ]; + return Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ? + markFunction(function( seed, matches ) { + var idx, + matched = fn( seed, argument ), + i = matched.length; + while ( i-- ) { + idx = indexOf.call( seed, matched[i] ); + seed[ idx ] = !( matches[ idx ] = matched[i] ); + } + }) : + function( elem ) { + return fn( elem, 0, args ); + }; + } + + return fn; + } + }, + + pseudos: { + "not": markFunction(function( selector ) { + // Trim the selector passed to compile + // to avoid treating leading and trailing + // spaces as combinators + var input = [], + results = [], + matcher = compile( selector.replace( rtrim, "$1" ) ); + + return matcher[ expando ] ? + markFunction(function( seed, matches, context, xml ) { + var elem, + unmatched = matcher( seed, null, xml, [] ), + i = seed.length; + + // Match elements unmatched by `matcher` + while ( i-- ) { + if ( (elem = unmatched[i]) ) { + seed[i] = !(matches[i] = elem); + } + } + }) : + function( elem, context, xml ) { + input[0] = elem; + matcher( input, null, xml, results ); + return !results.pop(); + }; + }), + + "has": markFunction(function( selector ) { + return function( elem ) { + return Sizzle( selector, elem ).length > 0; + }; + }), + + "contains": markFunction(function( text ) { + return function( elem ) { + return ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1; + }; + }), + + "enabled": function( elem ) { + return elem.disabled === false; + }, + + "disabled": function( elem ) { + return elem.disabled === true; + }, + + "checked": function( elem ) { + // In CSS3, :checked should return both checked and selected elements + // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked + var nodeName = elem.nodeName.toLowerCase(); + return (nodeName === "input" && !!elem.checked) || (nodeName === "option" && !!elem.selected); + }, + + "selected": function( elem ) { + // Accessing this property makes selected-by-default + // options in Safari work properly + if ( elem.parentNode ) { + elem.parentNode.selectedIndex; + } + + return elem.selected === true; + }, + + "parent": function( elem ) { + return !Expr.pseudos["empty"]( elem ); + }, + + "empty": function( elem ) { + // http://www.w3.org/TR/selectors/#empty-pseudo + // :empty is only affected by element nodes and content nodes(including text(3), cdata(4)), + // not comment, processing instructions, or others + // Thanks to Diego Perini for the nodeName shortcut + // Greater than "@" means alpha characters (specifically not starting with "#" or "?") + var nodeType; + elem = elem.firstChild; + while ( elem ) { + if ( elem.nodeName > "@" || (nodeType = elem.nodeType) === 3 || nodeType === 4 ) { + return false; + } + elem = elem.nextSibling; + } + return true; + }, + + "header": function( elem ) { + return rheader.test( elem.nodeName ); + }, + + "text": function( elem ) { + var type, attr; + // IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc) + // use getAttribute instead to test this case + return elem.nodeName.toLowerCase() === "input" && + (type = elem.type) === "text" && + ( (attr = elem.getAttribute("type")) == null || attr.toLowerCase() === type ); + }, + + // Input types + "radio": createInputPseudo("radio"), + "checkbox": createInputPseudo("checkbox"), + "file": createInputPseudo("file"), + "password": createInputPseudo("password"), + "image": createInputPseudo("image"), + + "submit": createButtonPseudo("submit"), + "reset": createButtonPseudo("reset"), + + "button": function( elem ) { + var name = elem.nodeName.toLowerCase(); + return name === "input" && elem.type === "button" || name === "button"; + }, + + "input": function( elem ) { + return rinputs.test( elem.nodeName ); + }, + + "focus": function( elem ) { + var doc = elem.ownerDocument; + return elem === doc.activeElement && (!doc.hasFocus || doc.hasFocus()) && !!(elem.type || elem.href || ~elem.tabIndex); + }, + + "active": function( elem ) { + return elem === elem.ownerDocument.activeElement; + }, + + // Positional types + "first": createPositionalPseudo(function() { + return [ 0 ]; + }), + + "last": createPositionalPseudo(function( matchIndexes, length ) { + return [ length - 1 ]; + }), + + "eq": createPositionalPseudo(function( matchIndexes, length, argument ) { + return [ argument < 0 ? argument + length : argument ]; + }), + + "even": createPositionalPseudo(function( matchIndexes, length ) { + for ( var i = 0; i < length; i += 2 ) { + matchIndexes.push( i ); + } + return matchIndexes; + }), + + "odd": createPositionalPseudo(function( matchIndexes, length ) { + for ( var i = 1; i < length; i += 2 ) { + matchIndexes.push( i ); + } + return matchIndexes; + }), + + "lt": createPositionalPseudo(function( matchIndexes, length, argument ) { + for ( var i = argument < 0 ? argument + length : argument; --i >= 0; ) { + matchIndexes.push( i ); + } + return matchIndexes; + }), + + "gt": createPositionalPseudo(function( matchIndexes, length, argument ) { + for ( var i = argument < 0 ? argument + length : argument; ++i < length; ) { + matchIndexes.push( i ); + } + return matchIndexes; + }) + } +}; + +function siblingCheck( a, b, ret ) { + if ( a === b ) { + return ret; + } + + var cur = a.nextSibling; + + while ( cur ) { + if ( cur === b ) { + return -1; + } + + cur = cur.nextSibling; + } + + return 1; +} + +sortOrder = docElem.compareDocumentPosition ? + function( a, b ) { + if ( a === b ) { + hasDuplicate = true; + return 0; + } + + return ( !a.compareDocumentPosition || !b.compareDocumentPosition ? + a.compareDocumentPosition : + a.compareDocumentPosition(b) & 4 + ) ? -1 : 1; + } : + function( a, b ) { + // The nodes are identical, we can exit early + if ( a === b ) { + hasDuplicate = true; + return 0; + + // Fallback to using sourceIndex (in IE) if it's available on both nodes + } else if ( a.sourceIndex && b.sourceIndex ) { + return a.sourceIndex - b.sourceIndex; + } + + var al, bl, + ap = [], + bp = [], + aup = a.parentNode, + bup = b.parentNode, + cur = aup; + + // If the nodes are siblings (or identical) we can do a quick check + if ( aup === bup ) { + return siblingCheck( a, b ); + + // If no parents were found then the nodes are disconnected + } else if ( !aup ) { + return -1; + + } else if ( !bup ) { + return 1; + } + + // Otherwise they're somewhere else in the tree so we need + // to build up a full list of the parentNodes for comparison + while ( cur ) { + ap.unshift( cur ); + cur = cur.parentNode; + } + + cur = bup; + + while ( cur ) { + bp.unshift( cur ); + cur = cur.parentNode; + } + + al = ap.length; + bl = bp.length; + + // Start walking down the tree looking for a discrepancy + for ( var i = 0; i < al && i < bl; i++ ) { + if ( ap[i] !== bp[i] ) { + return siblingCheck( ap[i], bp[i] ); + } + } + + // We ended someplace up the tree so do a sibling check + return i === al ? + siblingCheck( a, bp[i], -1 ) : + siblingCheck( ap[i], b, 1 ); + }; + +// Always assume the presence of duplicates if sort doesn't +// pass them to our comparison function (as in Google Chrome). +[0, 0].sort( sortOrder ); +baseHasDuplicate = !hasDuplicate; + +// Document sorting and removing duplicates +Sizzle.uniqueSort = function( results ) { + var elem, + duplicates = [], + i = 1, + j = 0; + + hasDuplicate = baseHasDuplicate; + results.sort( sortOrder ); + + if ( hasDuplicate ) { + for ( ; (elem = results[i]); i++ ) { + if ( elem === results[ i - 1 ] ) { + j = duplicates.push( i ); + } + } + while ( j-- ) { + results.splice( duplicates[ j ], 1 ); + } + } + + return results; +}; + +Sizzle.error = function( msg ) { + throw new Error( "Syntax error, unrecognized expression: " + msg ); +}; + +function tokenize( selector, parseOnly ) { + var matched, match, tokens, type, + soFar, groups, preFilters, + cached = tokenCache[ expando ][ selector + " " ]; + + if ( cached ) { + return parseOnly ? 0 : cached.slice( 0 ); + } + + soFar = selector; + groups = []; + preFilters = Expr.preFilter; + + while ( soFar ) { + + // Comma and first run + if ( !matched || (match = rcomma.exec( soFar )) ) { + if ( match ) { + // Don't consume trailing commas as valid + soFar = soFar.slice( match[0].length ) || soFar; + } + groups.push( tokens = [] ); + } + + matched = false; + + // Combinators + if ( (match = rcombinators.exec( soFar )) ) { + tokens.push( matched = new Token( match.shift() ) ); + soFar = soFar.slice( matched.length ); + + // Cast descendant combinators to space + matched.type = match[0].replace( rtrim, " " ); + } + + // Filters + for ( type in Expr.filter ) { + if ( (match = matchExpr[ type ].exec( soFar )) && (!preFilters[ type ] || + (match = preFilters[ type ]( match ))) ) { + + tokens.push( matched = new Token( match.shift() ) ); + soFar = soFar.slice( matched.length ); + matched.type = type; + matched.matches = match; + } + } + + if ( !matched ) { + break; + } + } + + // Return the length of the invalid excess + // if we're just parsing + // Otherwise, throw an error or return tokens + return parseOnly ? + soFar.length : + soFar ? + Sizzle.error( selector ) : + // Cache the tokens + tokenCache( selector, groups ).slice( 0 ); +} + +function addCombinator( matcher, combinator, base ) { + var dir = combinator.dir, + checkNonElements = base && combinator.dir === "parentNode", + doneName = done++; + + return combinator.first ? + // Check against closest ancestor/preceding element + function( elem, context, xml ) { + while ( (elem = elem[ dir ]) ) { + if ( checkNonElements || elem.nodeType === 1 ) { + return matcher( elem, context, xml ); + } + } + } : + + // Check against all ancestor/preceding elements + function( elem, context, xml ) { + // We can't set arbitrary data on XML nodes, so they don't benefit from dir caching + if ( !xml ) { + var cache, + dirkey = dirruns + " " + doneName + " ", + cachedkey = dirkey + cachedruns; + while ( (elem = elem[ dir ]) ) { + if ( checkNonElements || elem.nodeType === 1 ) { + if ( (cache = elem[ expando ]) === cachedkey ) { + return elem.sizset; + } else if ( typeof cache === "string" && cache.indexOf(dirkey) === 0 ) { + if ( elem.sizset ) { + return elem; + } + } else { + elem[ expando ] = cachedkey; + if ( matcher( elem, context, xml ) ) { + elem.sizset = true; + return elem; + } + elem.sizset = false; + } + } + } + } else { + while ( (elem = elem[ dir ]) ) { + if ( checkNonElements || elem.nodeType === 1 ) { + if ( matcher( elem, context, xml ) ) { + return elem; + } + } + } + } + }; +} + +function elementMatcher( matchers ) { + return matchers.length > 1 ? + function( elem, context, xml ) { + var i = matchers.length; + while ( i-- ) { + if ( !matchers[i]( elem, context, xml ) ) { + return false; + } + } + return true; + } : + matchers[0]; +} + +function condense( unmatched, map, filter, context, xml ) { + var elem, + newUnmatched = [], + i = 0, + len = unmatched.length, + mapped = map != null; + + for ( ; i < len; i++ ) { + if ( (elem = unmatched[i]) ) { + if ( !filter || filter( elem, context, xml ) ) { + newUnmatched.push( elem ); + if ( mapped ) { + map.push( i ); + } + } + } + } + + return newUnmatched; +} + +function setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) { + if ( postFilter && !postFilter[ expando ] ) { + postFilter = setMatcher( postFilter ); + } + if ( postFinder && !postFinder[ expando ] ) { + postFinder = setMatcher( postFinder, postSelector ); + } + return markFunction(function( seed, results, context, xml ) { + var temp, i, elem, + preMap = [], + postMap = [], + preexisting = results.length, + + // Get initial elements from seed or context + elems = seed || multipleContexts( selector || "*", context.nodeType ? [ context ] : context, [] ), + + // Prefilter to get matcher input, preserving a map for seed-results synchronization + matcherIn = preFilter && ( seed || !selector ) ? + condense( elems, preMap, preFilter, context, xml ) : + elems, + + matcherOut = matcher ? + // If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results, + postFinder || ( seed ? preFilter : preexisting || postFilter ) ? + + // ...intermediate processing is necessary + [] : + + // ...otherwise use results directly + results : + matcherIn; + + // Find primary matches + if ( matcher ) { + matcher( matcherIn, matcherOut, context, xml ); + } + + // Apply postFilter + if ( postFilter ) { + temp = condense( matcherOut, postMap ); + postFilter( temp, [], context, xml ); + + // Un-match failing elements by moving them back to matcherIn + i = temp.length; + while ( i-- ) { + if ( (elem = temp[i]) ) { + matcherOut[ postMap[i] ] = !(matcherIn[ postMap[i] ] = elem); + } + } + } + + if ( seed ) { + if ( postFinder || preFilter ) { + if ( postFinder ) { + // Get the final matcherOut by condensing this intermediate into postFinder contexts + temp = []; + i = matcherOut.length; + while ( i-- ) { + if ( (elem = matcherOut[i]) ) { + // Restore matcherIn since elem is not yet a final match + temp.push( (matcherIn[i] = elem) ); + } + } + postFinder( null, (matcherOut = []), temp, xml ); + } + + // Move matched elements from seed to results to keep them synchronized + i = matcherOut.length; + while ( i-- ) { + if ( (elem = matcherOut[i]) && + (temp = postFinder ? indexOf.call( seed, elem ) : preMap[i]) > -1 ) { + + seed[temp] = !(results[temp] = elem); + } + } + } + + // Add elements to results, through postFinder if defined + } else { + matcherOut = condense( + matcherOut === results ? + matcherOut.splice( preexisting, matcherOut.length ) : + matcherOut + ); + if ( postFinder ) { + postFinder( null, results, matcherOut, xml ); + } else { + push.apply( results, matcherOut ); + } + } + }); +} + +function matcherFromTokens( tokens ) { + var checkContext, matcher, j, + len = tokens.length, + leadingRelative = Expr.relative[ tokens[0].type ], + implicitRelative = leadingRelative || Expr.relative[" "], + i = leadingRelative ? 1 : 0, + + // The foundational matcher ensures that elements are reachable from top-level context(s) + matchContext = addCombinator( function( elem ) { + return elem === checkContext; + }, implicitRelative, true ), + matchAnyContext = addCombinator( function( elem ) { + return indexOf.call( checkContext, elem ) > -1; + }, implicitRelative, true ), + matchers = [ function( elem, context, xml ) { + return ( !leadingRelative && ( xml || context !== outermostContext ) ) || ( + (checkContext = context).nodeType ? + matchContext( elem, context, xml ) : + matchAnyContext( elem, context, xml ) ); + } ]; + + for ( ; i < len; i++ ) { + if ( (matcher = Expr.relative[ tokens[i].type ]) ) { + matchers = [ addCombinator( elementMatcher( matchers ), matcher ) ]; + } else { + matcher = Expr.filter[ tokens[i].type ].apply( null, tokens[i].matches ); + + // Return special upon seeing a positional matcher + if ( matcher[ expando ] ) { + // Find the next relative operator (if any) for proper handling + j = ++i; + for ( ; j < len; j++ ) { + if ( Expr.relative[ tokens[j].type ] ) { + break; + } + } + return setMatcher( + i > 1 && elementMatcher( matchers ), + i > 1 && tokens.slice( 0, i - 1 ).join("").replace( rtrim, "$1" ), + matcher, + i < j && matcherFromTokens( tokens.slice( i, j ) ), + j < len && matcherFromTokens( (tokens = tokens.slice( j )) ), + j < len && tokens.join("") + ); + } + matchers.push( matcher ); + } + } + + return elementMatcher( matchers ); +} + +function matcherFromGroupMatchers( elementMatchers, setMatchers ) { + var bySet = setMatchers.length > 0, + byElement = elementMatchers.length > 0, + superMatcher = function( seed, context, xml, results, expandContext ) { + var elem, j, matcher, + setMatched = [], + matchedCount = 0, + i = "0", + unmatched = seed && [], + outermost = expandContext != null, + contextBackup = outermostContext, + // We must always have either seed elements or context + elems = seed || byElement && Expr.find["TAG"]( "*", expandContext && context.parentNode || context ), + // Nested matchers should use non-integer dirruns + dirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.E); + + if ( outermost ) { + outermostContext = context !== document && context; + cachedruns = superMatcher.el; + } + + // Add elements passing elementMatchers directly to results + for ( ; (elem = elems[i]) != null; i++ ) { + if ( byElement && elem ) { + for ( j = 0; (matcher = elementMatchers[j]); j++ ) { + if ( matcher( elem, context, xml ) ) { + results.push( elem ); + break; + } + } + if ( outermost ) { + dirruns = dirrunsUnique; + cachedruns = ++superMatcher.el; + } + } + + // Track unmatched elements for set filters + if ( bySet ) { + // They will have gone through all possible matchers + if ( (elem = !matcher && elem) ) { + matchedCount--; + } + + // Lengthen the array for every element, matched or not + if ( seed ) { + unmatched.push( elem ); + } + } + } + + // Apply set filters to unmatched elements + matchedCount += i; + if ( bySet && i !== matchedCount ) { + for ( j = 0; (matcher = setMatchers[j]); j++ ) { + matcher( unmatched, setMatched, context, xml ); + } + + if ( seed ) { + // Reintegrate element matches to eliminate the need for sorting + if ( matchedCount > 0 ) { + while ( i-- ) { + if ( !(unmatched[i] || setMatched[i]) ) { + setMatched[i] = pop.call( results ); + } + } + } + + // Discard index placeholder values to get only actual matches + setMatched = condense( setMatched ); + } + + // Add matches to results + push.apply( results, setMatched ); + + // Seedless set matches succeeding multiple successful matchers stipulate sorting + if ( outermost && !seed && setMatched.length > 0 && + ( matchedCount + setMatchers.length ) > 1 ) { + + Sizzle.uniqueSort( results ); + } + } + + // Override manipulation of globals by nested matchers + if ( outermost ) { + dirruns = dirrunsUnique; + outermostContext = contextBackup; + } + + return unmatched; + }; + + superMatcher.el = 0; + return bySet ? + markFunction( superMatcher ) : + superMatcher; +} + +compile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) { + var i, + setMatchers = [], + elementMatchers = [], + cached = compilerCache[ expando ][ selector + " " ]; + + if ( !cached ) { + // Generate a function of recursive functions that can be used to check each element + if ( !group ) { + group = tokenize( selector ); + } + i = group.length; + while ( i-- ) { + cached = matcherFromTokens( group[i] ); + if ( cached[ expando ] ) { + setMatchers.push( cached ); + } else { + elementMatchers.push( cached ); + } + } + + // Cache the compiled function + cached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) ); + } + return cached; +}; + +function multipleContexts( selector, contexts, results ) { + var i = 0, + len = contexts.length; + for ( ; i < len; i++ ) { + Sizzle( selector, contexts[i], results ); + } + return results; +} + +function select( selector, context, results, seed, xml ) { + var i, tokens, token, type, find, + match = tokenize( selector ), + j = match.length; + + if ( !seed ) { + // Try to minimize operations if there is only one group + if ( match.length === 1 ) { + + // Take a shortcut and set the context if the root selector is an ID + tokens = match[0] = match[0].slice( 0 ); + if ( tokens.length > 2 && (token = tokens[0]).type === "ID" && + context.nodeType === 9 && !xml && + Expr.relative[ tokens[1].type ] ) { + + context = Expr.find["ID"]( token.matches[0].replace( rbackslash, "" ), context, xml )[0]; + if ( !context ) { + return results; + } + + selector = selector.slice( tokens.shift().length ); + } + + // Fetch a seed set for right-to-left matching + for ( i = matchExpr["POS"].test( selector ) ? -1 : tokens.length - 1; i >= 0; i-- ) { + token = tokens[i]; + + // Abort if we hit a combinator + if ( Expr.relative[ (type = token.type) ] ) { + break; + } + if ( (find = Expr.find[ type ]) ) { + // Search, expanding context for leading sibling combinators + if ( (seed = find( + token.matches[0].replace( rbackslash, "" ), + rsibling.test( tokens[0].type ) && context.parentNode || context, + xml + )) ) { + + // If seed is empty or no tokens remain, we can return early + tokens.splice( i, 1 ); + selector = seed.length && tokens.join(""); + if ( !selector ) { + push.apply( results, slice.call( seed, 0 ) ); + return results; + } + + break; + } + } + } + } + } + + // Compile and execute a filtering function + // Provide `match` to avoid retokenization if we modified the selector above + compile( selector, match )( + seed, + context, + xml, + results, + rsibling.test( selector ) + ); + return results; +} + +if ( document.querySelectorAll ) { + (function() { + var disconnectedMatch, + oldSelect = select, + rescape = /'|\\/g, + rattributeQuotes = /\=[\x20\t\r\n\f]*([^'"\]]*)[\x20\t\r\n\f]*\]/g, + + // qSa(:focus) reports false when true (Chrome 21), no need to also add to buggyMatches since matches checks buggyQSA + // A support test would require too much code (would include document ready) + rbuggyQSA = [ ":focus" ], + + // matchesSelector(:active) reports false when true (IE9/Opera 11.5) + // A support test would require too much code (would include document ready) + // just skip matchesSelector for :active + rbuggyMatches = [ ":active" ], + matches = docElem.matchesSelector || + docElem.mozMatchesSelector || + docElem.webkitMatchesSelector || + docElem.oMatchesSelector || + docElem.msMatchesSelector; + + // Build QSA regex + // Regex strategy adopted from Diego Perini + assert(function( div ) { + // Select is set to empty string on purpose + // This is to test IE's treatment of not explictly + // setting a boolean content attribute, + // since its presence should be enough + // http://bugs.jquery.com/ticket/12359 + div.innerHTML = "<select><option selected=''></option></select>"; + + // IE8 - Some boolean attributes are not treated correctly + if ( !div.querySelectorAll("[selected]").length ) { + rbuggyQSA.push( "\\[" + whitespace + "*(?:checked|disabled|ismap|multiple|readonly|selected|value)" ); + } + + // Webkit/Opera - :checked should return selected option elements + // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked + // IE8 throws error here (do not put tests after this one) + if ( !div.querySelectorAll(":checked").length ) { + rbuggyQSA.push(":checked"); + } + }); + + assert(function( div ) { + + // Opera 10-12/IE9 - ^= $= *= and empty values + // Should not select anything + div.innerHTML = "<p test=''></p>"; + if ( div.querySelectorAll("[test^='']").length ) { + rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:\"\"|'')" ); + } + + // FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled) + // IE8 throws error here (do not put tests after this one) + div.innerHTML = "<input type='hidden'/>"; + if ( !div.querySelectorAll(":enabled").length ) { + rbuggyQSA.push(":enabled", ":disabled"); + } + }); + + // rbuggyQSA always contains :focus, so no need for a length check + rbuggyQSA = /* rbuggyQSA.length && */ new RegExp( rbuggyQSA.join("|") ); + + select = function( selector, context, results, seed, xml ) { + // Only use querySelectorAll when not filtering, + // when this is not xml, + // and when no QSA bugs apply + if ( !seed && !xml && !rbuggyQSA.test( selector ) ) { + var groups, i, + old = true, + nid = expando, + newContext = context, + newSelector = context.nodeType === 9 && selector; + + // qSA works strangely on Element-rooted queries + // We can work around this by specifying an extra ID on the root + // and working up from there (Thanks to Andrew Dupont for the technique) + // IE 8 doesn't work on object elements + if ( context.nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) { + groups = tokenize( selector ); + + if ( (old = context.getAttribute("id")) ) { + nid = old.replace( rescape, "\\$&" ); + } else { + context.setAttribute( "id", nid ); + } + nid = "[id='" + nid + "'] "; + + i = groups.length; + while ( i-- ) { + groups[i] = nid + groups[i].join(""); + } + newContext = rsibling.test( selector ) && context.parentNode || context; + newSelector = groups.join(","); + } + + if ( newSelector ) { + try { + push.apply( results, slice.call( newContext.querySelectorAll( + newSelector + ), 0 ) ); + return results; + } catch(qsaError) { + } finally { + if ( !old ) { + context.removeAttribute("id"); + } + } + } + } + + return oldSelect( selector, context, results, seed, xml ); + }; + + if ( matches ) { + assert(function( div ) { + // Check to see if it's possible to do matchesSelector + // on a disconnected node (IE 9) + disconnectedMatch = matches.call( div, "div" ); + + // This should fail with an exception + // Gecko does not error, returns false instead + try { + matches.call( div, "[test!='']:sizzle" ); + rbuggyMatches.push( "!=", pseudos ); + } catch ( e ) {} + }); + + // rbuggyMatches always contains :active and :focus, so no need for a length check + rbuggyMatches = /* rbuggyMatches.length && */ new RegExp( rbuggyMatches.join("|") ); + + Sizzle.matchesSelector = function( elem, expr ) { + // Make sure that attribute selectors are quoted + expr = expr.replace( rattributeQuotes, "='$1']" ); + + // rbuggyMatches always contains :active, so no need for an existence check + if ( !isXML( elem ) && !rbuggyMatches.test( expr ) && !rbuggyQSA.test( expr ) ) { + try { + var ret = matches.call( elem, expr ); + + // IE 9's matchesSelector returns false on disconnected nodes + if ( ret || disconnectedMatch || + // As well, disconnected nodes are said to be in a document + // fragment in IE 9 + elem.document && elem.document.nodeType !== 11 ) { + return ret; + } + } catch(e) {} + } + + return Sizzle( expr, null, null, [ elem ] ).length > 0; + }; + } + })(); +} + +// Deprecated +Expr.pseudos["nth"] = Expr.pseudos["eq"]; + +// Back-compat +function setFilters() {} +Expr.filters = setFilters.prototype = Expr.pseudos; +Expr.setFilters = new setFilters(); + +// Override sizzle attribute retrieval +Sizzle.attr = jQuery.attr; +jQuery.find = Sizzle; +jQuery.expr = Sizzle.selectors; +jQuery.expr[":"] = jQuery.expr.pseudos; +jQuery.unique = Sizzle.uniqueSort; +jQuery.text = Sizzle.getText; +jQuery.isXMLDoc = Sizzle.isXML; +jQuery.contains = Sizzle.contains; + + +})( window ); +var runtil = /Until$/, + rparentsprev = /^(?:parents|prev(?:Until|All))/, + isSimple = /^.[^:#\[\.,]*$/, + rneedsContext = jQuery.expr.match.needsContext, + // methods guaranteed to produce a unique set when starting from a unique set + guaranteedUnique = { + children: true, + contents: true, + next: true, + prev: true + }; + +jQuery.fn.extend({ + find: function( selector ) { + var i, l, length, n, r, ret, + self = this; + + if ( typeof selector !== "string" ) { + return jQuery( selector ).filter(function() { + for ( i = 0, l = self.length; i < l; i++ ) { + if ( jQuery.contains( self[ i ], this ) ) { + return true; + } + } + }); + } + + ret = this.pushStack( "", "find", selector ); + + for ( i = 0, l = this.length; i < l; i++ ) { + length = ret.length; + jQuery.find( selector, this[i], ret ); + + if ( i > 0 ) { + // Make sure that the results are unique + for ( n = length; n < ret.length; n++ ) { + for ( r = 0; r < length; r++ ) { + if ( ret[r] === ret[n] ) { + ret.splice(n--, 1); + break; + } + } + } + } + } + + return ret; + }, + + has: function( target ) { + var i, + targets = jQuery( target, this ), + len = targets.length; + + return this.filter(function() { + for ( i = 0; i < len; i++ ) { + if ( jQuery.contains( this, targets[i] ) ) { + return true; + } + } + }); + }, + + not: function( selector ) { + return this.pushStack( winnow(this, selector, false), "not", selector); + }, + + filter: function( selector ) { + return this.pushStack( winnow(this, selector, true), "filter", selector ); + }, + + is: function( selector ) { + return !!selector && ( + typeof selector === "string" ? + // If this is a positional/relative selector, check membership in the returned set + // so $("p:first").is("p:last") won't return true for a doc with two "p". + rneedsContext.test( selector ) ? + jQuery( selector, this.context ).index( this[0] ) >= 0 : + jQuery.filter( selector, this ).length > 0 : + this.filter( selector ).length > 0 ); + }, + + closest: function( selectors, context ) { + var cur, + i = 0, + l = this.length, + ret = [], + pos = rneedsContext.test( selectors ) || typeof selectors !== "string" ? + jQuery( selectors, context || this.context ) : + 0; + + for ( ; i < l; i++ ) { + cur = this[i]; + + while ( cur && cur.ownerDocument && cur !== context && cur.nodeType !== 11 ) { + if ( pos ? pos.index(cur) > -1 : jQuery.find.matchesSelector(cur, selectors) ) { + ret.push( cur ); + break; + } + cur = cur.parentNode; + } + } + + ret = ret.length > 1 ? jQuery.unique( ret ) : ret; + + return this.pushStack( ret, "closest", selectors ); + }, + + // Determine the position of an element within + // the matched set of elements + index: function( elem ) { + + // No argument, return index in parent + if ( !elem ) { + return ( this[0] && this[0].parentNode ) ? this.prevAll().length : -1; + } + + // index in selector + if ( typeof elem === "string" ) { + return jQuery.inArray( this[0], jQuery( elem ) ); + } + + // Locate the position of the desired element + return jQuery.inArray( + // If it receives a jQuery object, the first element is used + elem.jquery ? elem[0] : elem, this ); + }, + + add: function( selector, context ) { + var set = typeof selector === "string" ? + jQuery( selector, context ) : + jQuery.makeArray( selector && selector.nodeType ? [ selector ] : selector ), + all = jQuery.merge( this.get(), set ); + + return this.pushStack( isDisconnected( set[0] ) || isDisconnected( all[0] ) ? + all : + jQuery.unique( all ) ); + }, + + addBack: function( selector ) { + return this.add( selector == null ? + this.prevObject : this.prevObject.filter(selector) + ); + } +}); + +jQuery.fn.andSelf = jQuery.fn.addBack; + +// A painfully simple check to see if an element is disconnected +// from a document (should be improved, where feasible). +function isDisconnected( node ) { + return !node || !node.parentNode || node.parentNode.nodeType === 11; +} + +function sibling( cur, dir ) { + do { + cur = cur[ dir ]; + } while ( cur && cur.nodeType !== 1 ); + + return cur; +} + +jQuery.each({ + parent: function( elem ) { + var parent = elem.parentNode; + return parent && parent.nodeType !== 11 ? parent : null; + }, + parents: function( elem ) { + return jQuery.dir( elem, "parentNode" ); + }, + parentsUntil: function( elem, i, until ) { + return jQuery.dir( elem, "parentNode", until ); + }, + next: function( elem ) { + return sibling( elem, "nextSibling" ); + }, + prev: function( elem ) { + return sibling( elem, "previousSibling" ); + }, + nextAll: function( elem ) { + return jQuery.dir( elem, "nextSibling" ); + }, + prevAll: function( elem ) { + return jQuery.dir( elem, "previousSibling" ); + }, + nextUntil: function( elem, i, until ) { + return jQuery.dir( elem, "nextSibling", until ); + }, + prevUntil: function( elem, i, until ) { + return jQuery.dir( elem, "previousSibling", until ); + }, + siblings: function( elem ) { + return jQuery.sibling( ( elem.parentNode || {} ).firstChild, elem ); + }, + children: function( elem ) { + return jQuery.sibling( elem.firstChild ); + }, + contents: function( elem ) { + return jQuery.nodeName( elem, "iframe" ) ? + elem.contentDocument || elem.contentWindow.document : + jQuery.merge( [], elem.childNodes ); + } +}, function( name, fn ) { + jQuery.fn[ name ] = function( until, selector ) { + var ret = jQuery.map( this, fn, until ); + + if ( !runtil.test( name ) ) { + selector = until; + } + + if ( selector && typeof selector === "string" ) { + ret = jQuery.filter( selector, ret ); + } + + ret = this.length > 1 && !guaranteedUnique[ name ] ? jQuery.unique( ret ) : ret; + + if ( this.length > 1 && rparentsprev.test( name ) ) { + ret = ret.reverse(); + } + + return this.pushStack( ret, name, core_slice.call( arguments ).join(",") ); + }; +}); + +jQuery.extend({ + filter: function( expr, elems, not ) { + if ( not ) { + expr = ":not(" + expr + ")"; + } + + return elems.length === 1 ? + jQuery.find.matchesSelector(elems[0], expr) ? [ elems[0] ] : [] : + jQuery.find.matches(expr, elems); + }, + + dir: function( elem, dir, until ) { + var matched = [], + cur = elem[ dir ]; + + while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) { + if ( cur.nodeType === 1 ) { + matched.push( cur ); + } + cur = cur[dir]; + } + return matched; + }, + + sibling: function( n, elem ) { + var r = []; + + for ( ; n; n = n.nextSibling ) { + if ( n.nodeType === 1 && n !== elem ) { + r.push( n ); + } + } + + return r; + } +}); + +// Implement the identical functionality for filter and not +function winnow( elements, qualifier, keep ) { + + // Can't pass null or undefined to indexOf in Firefox 4 + // Set to 0 to skip string check + qualifier = qualifier || 0; + + if ( jQuery.isFunction( qualifier ) ) { + return jQuery.grep(elements, function( elem, i ) { + var retVal = !!qualifier.call( elem, i, elem ); + return retVal === keep; + }); + + } else if ( qualifier.nodeType ) { + return jQuery.grep(elements, function( elem, i ) { + return ( elem === qualifier ) === keep; + }); + + } else if ( typeof qualifier === "string" ) { + var filtered = jQuery.grep(elements, function( elem ) { + return elem.nodeType === 1; + }); + + if ( isSimple.test( qualifier ) ) { + return jQuery.filter(qualifier, filtered, !keep); + } else { + qualifier = jQuery.filter( qualifier, filtered ); + } + } + + return jQuery.grep(elements, function( elem, i ) { + return ( jQuery.inArray( elem, qualifier ) >= 0 ) === keep; + }); +} +function createSafeFragment( document ) { + var list = nodeNames.split( "|" ), + safeFrag = document.createDocumentFragment(); + + if ( safeFrag.createElement ) { + while ( list.length ) { + safeFrag.createElement( + list.pop() + ); + } + } + return safeFrag; +} + +var nodeNames = "abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|" + + "header|hgroup|mark|meter|nav|output|progress|section|summary|time|video", + rinlinejQuery = / jQuery\d+="(?:null|\d+)"/g, + rleadingWhitespace = /^\s+/, + rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi, + rtagName = /<([\w:]+)/, + rtbody = /<tbody/i, + rhtml = /<|&#?\w+;/, + rnoInnerhtml = /<(?:script|style|link)/i, + rnocache = /<(?:script|object|embed|option|style)/i, + rnoshimcache = new RegExp("<(?:" + nodeNames + ")[\\s/>]", "i"), + rcheckableType = /^(?:checkbox|radio)$/, + // checked="checked" or checked + rchecked = /checked\s*(?:[^=]|=\s*.checked.)/i, + rscriptType = /\/(java|ecma)script/i, + rcleanScript = /^\s*<!(?:\[CDATA\[|\-\-)|[\]\-]{2}>\s*$/g, + wrapMap = { + option: [ 1, "<select multiple='multiple'>", "</select>" ], + legend: [ 1, "<fieldset>", "</fieldset>" ], + thead: [ 1, "<table>", "</table>" ], + tr: [ 2, "<table><tbody>", "</tbody></table>" ], + td: [ 3, "<table><tbody><tr>", "</tr></tbody></table>" ], + col: [ 2, "<table><tbody></tbody><colgroup>", "</colgroup></table>" ], + area: [ 1, "<map>", "</map>" ], + _default: [ 0, "", "" ] + }, + safeFragment = createSafeFragment( document ), + fragmentDiv = safeFragment.appendChild( document.createElement("div") ); + +wrapMap.optgroup = wrapMap.option; +wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; +wrapMap.th = wrapMap.td; + +// IE6-8 can't serialize link, script, style, or any html5 (NoScope) tags, +// unless wrapped in a div with non-breaking characters in front of it. +if ( !jQuery.support.htmlSerialize ) { + wrapMap._default = [ 1, "X<div>", "</div>" ]; +} + +jQuery.fn.extend({ + text: function( value ) { + return jQuery.access( this, function( value ) { + return value === undefined ? + jQuery.text( this ) : + this.empty().append( ( this[0] && this[0].ownerDocument || document ).createTextNode( value ) ); + }, null, value, arguments.length ); + }, + + wrapAll: function( html ) { + if ( jQuery.isFunction( html ) ) { + return this.each(function(i) { + jQuery(this).wrapAll( html.call(this, i) ); + }); + } + + if ( this[0] ) { + // The elements to wrap the target around + var wrap = jQuery( html, this[0].ownerDocument ).eq(0).clone(true); + + if ( this[0].parentNode ) { + wrap.insertBefore( this[0] ); + } + + wrap.map(function() { + var elem = this; + + while ( elem.firstChild && elem.firstChild.nodeType === 1 ) { + elem = elem.firstChild; + } + + return elem; + }).append( this ); + } + + return this; + }, + + wrapInner: function( html ) { + if ( jQuery.isFunction( html ) ) { + return this.each(function(i) { + jQuery(this).wrapInner( html.call(this, i) ); + }); + } + + return this.each(function() { + var self = jQuery( this ), + contents = self.contents(); + + if ( contents.length ) { + contents.wrapAll( html ); + + } else { + self.append( html ); + } + }); + }, + + wrap: function( html ) { + var isFunction = jQuery.isFunction( html ); + + return this.each(function(i) { + jQuery( this ).wrapAll( isFunction ? html.call(this, i) : html ); + }); + }, + + unwrap: function() { + return this.parent().each(function() { + if ( !jQuery.nodeName( this, "body" ) ) { + jQuery( this ).replaceWith( this.childNodes ); + } + }).end(); + }, + + append: function() { + return this.domManip(arguments, true, function( elem ) { + if ( this.nodeType === 1 || this.nodeType === 11 ) { + this.appendChild( elem ); + } + }); + }, + + prepend: function() { + return this.domManip(arguments, true, function( elem ) { + if ( this.nodeType === 1 || this.nodeType === 11 ) { + this.insertBefore( elem, this.firstChild ); + } + }); + }, + + before: function() { + if ( !isDisconnected( this[0] ) ) { + return this.domManip(arguments, false, function( elem ) { + this.parentNode.insertBefore( elem, this ); + }); + } + + if ( arguments.length ) { + var set = jQuery.clean( arguments ); + return this.pushStack( jQuery.merge( set, this ), "before", this.selector ); + } + }, + + after: function() { + if ( !isDisconnected( this[0] ) ) { + return this.domManip(arguments, false, function( elem ) { + this.parentNode.insertBefore( elem, this.nextSibling ); + }); + } + + if ( arguments.length ) { + var set = jQuery.clean( arguments ); + return this.pushStack( jQuery.merge( this, set ), "after", this.selector ); + } + }, + + // keepData is for internal use only--do not document + remove: function( selector, keepData ) { + var elem, + i = 0; + + for ( ; (elem = this[i]) != null; i++ ) { + if ( !selector || jQuery.filter( selector, [ elem ] ).length ) { + if ( !keepData && elem.nodeType === 1 ) { + jQuery.cleanData( elem.getElementsByTagName("*") ); + jQuery.cleanData( [ elem ] ); + } + + if ( elem.parentNode ) { + elem.parentNode.removeChild( elem ); + } + } + } + + return this; + }, + + empty: function() { + var elem, + i = 0; + + for ( ; (elem = this[i]) != null; i++ ) { + // Remove element nodes and prevent memory leaks + if ( elem.nodeType === 1 ) { + jQuery.cleanData( elem.getElementsByTagName("*") ); + } + + // Remove any remaining nodes + while ( elem.firstChild ) { + elem.removeChild( elem.firstChild ); + } + } + + return this; + }, + + clone: function( dataAndEvents, deepDataAndEvents ) { + dataAndEvents = dataAndEvents == null ? false : dataAndEvents; + deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; + + return this.map( function () { + return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); + }); + }, + + html: function( value ) { + return jQuery.access( this, function( value ) { + var elem = this[0] || {}, + i = 0, + l = this.length; + + if ( value === undefined ) { + return elem.nodeType === 1 ? + elem.innerHTML.replace( rinlinejQuery, "" ) : + undefined; + } + + // See if we can take a shortcut and just use innerHTML + if ( typeof value === "string" && !rnoInnerhtml.test( value ) && + ( jQuery.support.htmlSerialize || !rnoshimcache.test( value ) ) && + ( jQuery.support.leadingWhitespace || !rleadingWhitespace.test( value ) ) && + !wrapMap[ ( rtagName.exec( value ) || ["", ""] )[1].toLowerCase() ] ) { + + value = value.replace( rxhtmlTag, "<$1></$2>" ); + + try { + for (; i < l; i++ ) { + // Remove element nodes and prevent memory leaks + elem = this[i] || {}; + if ( elem.nodeType === 1 ) { + jQuery.cleanData( elem.getElementsByTagName( "*" ) ); + elem.innerHTML = value; + } + } + + elem = 0; + + // If using innerHTML throws an exception, use the fallback method + } catch(e) {} + } + + if ( elem ) { + this.empty().append( value ); + } + }, null, value, arguments.length ); + }, + + replaceWith: function( value ) { + if ( !isDisconnected( this[0] ) ) { + // Make sure that the elements are removed from the DOM before they are inserted + // this can help fix replacing a parent with child elements + if ( jQuery.isFunction( value ) ) { + return this.each(function(i) { + var self = jQuery(this), old = self.html(); + self.replaceWith( value.call( this, i, old ) ); + }); + } + + if ( typeof value !== "string" ) { + value = jQuery( value ).detach(); + } + + return this.each(function() { + var next = this.nextSibling, + parent = this.parentNode; + + jQuery( this ).remove(); + + if ( next ) { + jQuery(next).before( value ); + } else { + jQuery(parent).append( value ); + } + }); + } + + return this.length ? + this.pushStack( jQuery(jQuery.isFunction(value) ? value() : value), "replaceWith", value ) : + this; + }, + + detach: function( selector ) { + return this.remove( selector, true ); + }, + + domManip: function( args, table, callback ) { + + // Flatten any nested arrays + args = [].concat.apply( [], args ); + + var results, first, fragment, iNoClone, + i = 0, + value = args[0], + scripts = [], + l = this.length; + + // We can't cloneNode fragments that contain checked, in WebKit + if ( !jQuery.support.checkClone && l > 1 && typeof value === "string" && rchecked.test( value ) ) { + return this.each(function() { + jQuery(this).domManip( args, table, callback ); + }); + } + + if ( jQuery.isFunction(value) ) { + return this.each(function(i) { + var self = jQuery(this); + args[0] = value.call( this, i, table ? self.html() : undefined ); + self.domManip( args, table, callback ); + }); + } + + if ( this[0] ) { + results = jQuery.buildFragment( args, this, scripts ); + fragment = results.fragment; + first = fragment.firstChild; + + if ( fragment.childNodes.length === 1 ) { + fragment = first; + } + + if ( first ) { + table = table && jQuery.nodeName( first, "tr" ); + + // Use the original fragment for the last item instead of the first because it can end up + // being emptied incorrectly in certain situations (#8070). + // Fragments from the fragment cache must always be cloned and never used in place. + for ( iNoClone = results.cacheable || l - 1; i < l; i++ ) { + callback.call( + table && jQuery.nodeName( this[i], "table" ) ? + findOrAppend( this[i], "tbody" ) : + this[i], + i === iNoClone ? + fragment : + jQuery.clone( fragment, true, true ) + ); + } + } + + // Fix #11809: Avoid leaking memory + fragment = first = null; + + if ( scripts.length ) { + jQuery.each( scripts, function( i, elem ) { + if ( elem.src ) { + if ( jQuery.ajax ) { + jQuery.ajax({ + url: elem.src, + type: "GET", + dataType: "script", + async: false, + global: false, + "throws": true + }); + } else { + jQuery.error("no ajax"); + } + } else { + jQuery.globalEval( ( elem.text || elem.textContent || elem.innerHTML || "" ).replace( rcleanScript, "" ) ); + } + + if ( elem.parentNode ) { + elem.parentNode.removeChild( elem ); + } + }); + } + } + + return this; + } +}); + +function findOrAppend( elem, tag ) { + return elem.getElementsByTagName( tag )[0] || elem.appendChild( elem.ownerDocument.createElement( tag ) ); +} + +function cloneCopyEvent( src, dest ) { + + if ( dest.nodeType !== 1 || !jQuery.hasData( src ) ) { + return; + } + + var type, i, l, + oldData = jQuery._data( src ), + curData = jQuery._data( dest, oldData ), + events = oldData.events; + + if ( events ) { + delete curData.handle; + curData.events = {}; + + for ( type in events ) { + for ( i = 0, l = events[ type ].length; i < l; i++ ) { + jQuery.event.add( dest, type, events[ type ][ i ] ); + } + } + } + + // make the cloned public data object a copy from the original + if ( curData.data ) { + curData.data = jQuery.extend( {}, curData.data ); + } +} + +function cloneFixAttributes( src, dest ) { + var nodeName; + + // We do not need to do anything for non-Elements + if ( dest.nodeType !== 1 ) { + return; + } + + // clearAttributes removes the attributes, which we don't want, + // but also removes the attachEvent events, which we *do* want + if ( dest.clearAttributes ) { + dest.clearAttributes(); + } + + // mergeAttributes, in contrast, only merges back on the + // original attributes, not the events + if ( dest.mergeAttributes ) { + dest.mergeAttributes( src ); + } + + nodeName = dest.nodeName.toLowerCase(); + + if ( nodeName === "object" ) { + // IE6-10 improperly clones children of object elements using classid. + // IE10 throws NoModificationAllowedError if parent is null, #12132. + if ( dest.parentNode ) { + dest.outerHTML = src.outerHTML; + } + + // This path appears unavoidable for IE9. When cloning an object + // element in IE9, the outerHTML strategy above is not sufficient. + // If the src has innerHTML and the destination does not, + // copy the src.innerHTML into the dest.innerHTML. #10324 + if ( jQuery.support.html5Clone && (src.innerHTML && !jQuery.trim(dest.innerHTML)) ) { + dest.innerHTML = src.innerHTML; + } + + } else if ( nodeName === "input" && rcheckableType.test( src.type ) ) { + // IE6-8 fails to persist the checked state of a cloned checkbox + // or radio button. Worse, IE6-7 fail to give the cloned element + // a checked appearance if the defaultChecked value isn't also set + + dest.defaultChecked = dest.checked = src.checked; + + // IE6-7 get confused and end up setting the value of a cloned + // checkbox/radio button to an empty string instead of "on" + if ( dest.value !== src.value ) { + dest.value = src.value; + } + + // IE6-8 fails to return the selected option to the default selected + // state when cloning options + } else if ( nodeName === "option" ) { + dest.selected = src.defaultSelected; + + // IE6-8 fails to set the defaultValue to the correct value when + // cloning other types of input fields + } else if ( nodeName === "input" || nodeName === "textarea" ) { + dest.defaultValue = src.defaultValue; + + // IE blanks contents when cloning scripts + } else if ( nodeName === "script" && dest.text !== src.text ) { + dest.text = src.text; + } + + // Event data gets referenced instead of copied if the expando + // gets copied too + dest.removeAttribute( jQuery.expando ); +} + +jQuery.buildFragment = function( args, context, scripts ) { + var fragment, cacheable, cachehit, + first = args[ 0 ]; + + // Set context from what may come in as undefined or a jQuery collection or a node + // Updated to fix #12266 where accessing context[0] could throw an exception in IE9/10 & + // also doubles as fix for #8950 where plain objects caused createDocumentFragment exception + context = context || document; + context = !context.nodeType && context[0] || context; + context = context.ownerDocument || context; + + // Only cache "small" (1/2 KB) HTML strings that are associated with the main document + // Cloning options loses the selected state, so don't cache them + // IE 6 doesn't like it when you put <object> or <embed> elements in a fragment + // Also, WebKit does not clone 'checked' attributes on cloneNode, so don't cache + // Lastly, IE6,7,8 will not correctly reuse cached fragments that were created from unknown elems #10501 + if ( args.length === 1 && typeof first === "string" && first.length < 512 && context === document && + first.charAt(0) === "<" && !rnocache.test( first ) && + (jQuery.support.checkClone || !rchecked.test( first )) && + (jQuery.support.html5Clone || !rnoshimcache.test( first )) ) { + + // Mark cacheable and look for a hit + cacheable = true; + fragment = jQuery.fragments[ first ]; + cachehit = fragment !== undefined; + } + + if ( !fragment ) { + fragment = context.createDocumentFragment(); + jQuery.clean( args, context, fragment, scripts ); + + // Update the cache, but only store false + // unless this is a second parsing of the same content + if ( cacheable ) { + jQuery.fragments[ first ] = cachehit && fragment; + } + } + + return { fragment: fragment, cacheable: cacheable }; +}; + +jQuery.fragments = {}; + +jQuery.each({ + appendTo: "append", + prependTo: "prepend", + insertBefore: "before", + insertAfter: "after", + replaceAll: "replaceWith" +}, function( name, original ) { + jQuery.fn[ name ] = function( selector ) { + var elems, + i = 0, + ret = [], + insert = jQuery( selector ), + l = insert.length, + parent = this.length === 1 && this[0].parentNode; + + if ( (parent == null || parent && parent.nodeType === 11 && parent.childNodes.length === 1) && l === 1 ) { + insert[ original ]( this[0] ); + return this; + } else { + for ( ; i < l; i++ ) { + elems = ( i > 0 ? this.clone(true) : this ).get(); + jQuery( insert[i] )[ original ]( elems ); + ret = ret.concat( elems ); + } + + return this.pushStack( ret, name, insert.selector ); + } + }; +}); + +function getAll( elem ) { + if ( typeof elem.getElementsByTagName !== "undefined" ) { + return elem.getElementsByTagName( "*" ); + + } else if ( typeof elem.querySelectorAll !== "undefined" ) { + return elem.querySelectorAll( "*" ); + + } else { + return []; + } +} + +// Used in clean, fixes the defaultChecked property +function fixDefaultChecked( elem ) { + if ( rcheckableType.test( elem.type ) ) { + elem.defaultChecked = elem.checked; + } +} + +jQuery.extend({ + clone: function( elem, dataAndEvents, deepDataAndEvents ) { + var srcElements, + destElements, + i, + clone; + + if ( jQuery.support.html5Clone || jQuery.isXMLDoc(elem) || !rnoshimcache.test( "<" + elem.nodeName + ">" ) ) { + clone = elem.cloneNode( true ); + + // IE<=8 does not properly clone detached, unknown element nodes + } else { + fragmentDiv.innerHTML = elem.outerHTML; + fragmentDiv.removeChild( clone = fragmentDiv.firstChild ); + } + + if ( (!jQuery.support.noCloneEvent || !jQuery.support.noCloneChecked) && + (elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) { + // IE copies events bound via attachEvent when using cloneNode. + // Calling detachEvent on the clone will also remove the events + // from the original. In order to get around this, we use some + // proprietary methods to clear the events. Thanks to MooTools + // guys for this hotness. + + cloneFixAttributes( elem, clone ); + + // Using Sizzle here is crazy slow, so we use getElementsByTagName instead + srcElements = getAll( elem ); + destElements = getAll( clone ); + + // Weird iteration because IE will replace the length property + // with an element if you are cloning the body and one of the + // elements on the page has a name or id of "length" + for ( i = 0; srcElements[i]; ++i ) { + // Ensure that the destination node is not null; Fixes #9587 + if ( destElements[i] ) { + cloneFixAttributes( srcElements[i], destElements[i] ); + } + } + } + + // Copy the events from the original to the clone + if ( dataAndEvents ) { + cloneCopyEvent( elem, clone ); + + if ( deepDataAndEvents ) { + srcElements = getAll( elem ); + destElements = getAll( clone ); + + for ( i = 0; srcElements[i]; ++i ) { + cloneCopyEvent( srcElements[i], destElements[i] ); + } + } + } + + srcElements = destElements = null; + + // Return the cloned set + return clone; + }, + + clean: function( elems, context, fragment, scripts ) { + var i, j, elem, tag, wrap, depth, div, hasBody, tbody, len, handleScript, jsTags, + safe = context === document && safeFragment, + ret = []; + + // Ensure that context is a document + if ( !context || typeof context.createDocumentFragment === "undefined" ) { + context = document; + } + + // Use the already-created safe fragment if context permits + for ( i = 0; (elem = elems[i]) != null; i++ ) { + if ( typeof elem === "number" ) { + elem += ""; + } + + if ( !elem ) { + continue; + } + + // Convert html string into DOM nodes + if ( typeof elem === "string" ) { + if ( !rhtml.test( elem ) ) { + elem = context.createTextNode( elem ); + } else { + // Ensure a safe container in which to render the html + safe = safe || createSafeFragment( context ); + div = context.createElement("div"); + safe.appendChild( div ); + + // Fix "XHTML"-style tags in all browsers + elem = elem.replace(rxhtmlTag, "<$1></$2>"); + + // Go to html and back, then peel off extra wrappers + tag = ( rtagName.exec( elem ) || ["", ""] )[1].toLowerCase(); + wrap = wrapMap[ tag ] || wrapMap._default; + depth = wrap[0]; + div.innerHTML = wrap[1] + elem + wrap[2]; + + // Move to the right depth + while ( depth-- ) { + div = div.lastChild; + } + + // Remove IE's autoinserted <tbody> from table fragments + if ( !jQuery.support.tbody ) { + + // String was a <table>, *may* have spurious <tbody> + hasBody = rtbody.test(elem); + tbody = tag === "table" && !hasBody ? + div.firstChild && div.firstChild.childNodes : + + // String was a bare <thead> or <tfoot> + wrap[1] === "<table>" && !hasBody ? + div.childNodes : + []; + + for ( j = tbody.length - 1; j >= 0 ; --j ) { + if ( jQuery.nodeName( tbody[ j ], "tbody" ) && !tbody[ j ].childNodes.length ) { + tbody[ j ].parentNode.removeChild( tbody[ j ] ); + } + } + } + + // IE completely kills leading whitespace when innerHTML is used + if ( !jQuery.support.leadingWhitespace && rleadingWhitespace.test( elem ) ) { + div.insertBefore( context.createTextNode( rleadingWhitespace.exec(elem)[0] ), div.firstChild ); + } + + elem = div.childNodes; + + // Take out of fragment container (we need a fresh div each time) + div.parentNode.removeChild( div ); + } + } + + if ( elem.nodeType ) { + ret.push( elem ); + } else { + jQuery.merge( ret, elem ); + } + } + + // Fix #11356: Clear elements from safeFragment + if ( div ) { + elem = div = safe = null; + } + + // Reset defaultChecked for any radios and checkboxes + // about to be appended to the DOM in IE 6/7 (#8060) + if ( !jQuery.support.appendChecked ) { + for ( i = 0; (elem = ret[i]) != null; i++ ) { + if ( jQuery.nodeName( elem, "input" ) ) { + fixDefaultChecked( elem ); + } else if ( typeof elem.getElementsByTagName !== "undefined" ) { + jQuery.grep( elem.getElementsByTagName("input"), fixDefaultChecked ); + } + } + } + + // Append elements to a provided document fragment + if ( fragment ) { + // Special handling of each script element + handleScript = function( elem ) { + // Check if we consider it executable + if ( !elem.type || rscriptType.test( elem.type ) ) { + // Detach the script and store it in the scripts array (if provided) or the fragment + // Return truthy to indicate that it has been handled + return scripts ? + scripts.push( elem.parentNode ? elem.parentNode.removeChild( elem ) : elem ) : + fragment.appendChild( elem ); + } + }; + + for ( i = 0; (elem = ret[i]) != null; i++ ) { + // Check if we're done after handling an executable script + if ( !( jQuery.nodeName( elem, "script" ) && handleScript( elem ) ) ) { + // Append to fragment and handle embedded scripts + fragment.appendChild( elem ); + if ( typeof elem.getElementsByTagName !== "undefined" ) { + // handleScript alters the DOM, so use jQuery.merge to ensure snapshot iteration + jsTags = jQuery.grep( jQuery.merge( [], elem.getElementsByTagName("script") ), handleScript ); + + // Splice the scripts into ret after their former ancestor and advance our index beyond them + ret.splice.apply( ret, [i + 1, 0].concat( jsTags ) ); + i += jsTags.length; + } + } + } + } + + return ret; + }, + + cleanData: function( elems, /* internal */ acceptData ) { + var data, id, elem, type, + i = 0, + internalKey = jQuery.expando, + cache = jQuery.cache, + deleteExpando = jQuery.support.deleteExpando, + special = jQuery.event.special; + + for ( ; (elem = elems[i]) != null; i++ ) { + + if ( acceptData || jQuery.acceptData( elem ) ) { + + id = elem[ internalKey ]; + data = id && cache[ id ]; + + if ( data ) { + if ( data.events ) { + for ( type in data.events ) { + if ( special[ type ] ) { + jQuery.event.remove( elem, type ); + + // This is a shortcut to avoid jQuery.event.remove's overhead + } else { + jQuery.removeEvent( elem, type, data.handle ); + } + } + } + + // Remove cache only if it was not already removed by jQuery.event.remove + if ( cache[ id ] ) { + + delete cache[ id ]; + + // IE does not allow us to delete expando properties from nodes, + // nor does it have a removeAttribute function on Document nodes; + // we must handle all of these cases + if ( deleteExpando ) { + delete elem[ internalKey ]; + + } else if ( elem.removeAttribute ) { + elem.removeAttribute( internalKey ); + + } else { + elem[ internalKey ] = null; + } + + jQuery.deletedIds.push( id ); + } + } + } + } + } +}); +// Limit scope pollution from any deprecated API +(function() { + +var matched, browser; + +// Use of jQuery.browser is frowned upon. +// More details: http://api.jquery.com/jQuery.browser +// jQuery.uaMatch maintained for back-compat +jQuery.uaMatch = function( ua ) { + ua = ua.toLowerCase(); + + var match = /(chrome)[ \/]([\w.]+)/.exec( ua ) || + /(webkit)[ \/]([\w.]+)/.exec( ua ) || + /(opera)(?:.*version|)[ \/]([\w.]+)/.exec( ua ) || + /(msie) ([\w.]+)/.exec( ua ) || + ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec( ua ) || + []; + + return { + browser: match[ 1 ] || "", + version: match[ 2 ] || "0" + }; +}; + +matched = jQuery.uaMatch( navigator.userAgent ); +browser = {}; + +if ( matched.browser ) { + browser[ matched.browser ] = true; + browser.version = matched.version; +} + +// Chrome is Webkit, but Webkit is also Safari. +if ( browser.chrome ) { + browser.webkit = true; +} else if ( browser.webkit ) { + browser.safari = true; +} + +jQuery.browser = browser; + +jQuery.sub = function() { + function jQuerySub( selector, context ) { + return new jQuerySub.fn.init( selector, context ); + } + jQuery.extend( true, jQuerySub, this ); + jQuerySub.superclass = this; + jQuerySub.fn = jQuerySub.prototype = this(); + jQuerySub.fn.constructor = jQuerySub; + jQuerySub.sub = this.sub; + jQuerySub.fn.init = function init( selector, context ) { + if ( context && context instanceof jQuery && !(context instanceof jQuerySub) ) { + context = jQuerySub( context ); + } + + return jQuery.fn.init.call( this, selector, context, rootjQuerySub ); + }; + jQuerySub.fn.init.prototype = jQuerySub.fn; + var rootjQuerySub = jQuerySub(document); + return jQuerySub; +}; + +})(); +var curCSS, iframe, iframeDoc, + ralpha = /alpha\([^)]*\)/i, + ropacity = /opacity=([^)]*)/, + rposition = /^(top|right|bottom|left)$/, + // swappable if display is none or starts with table except "table", "table-cell", or "table-caption" + // see here for display values: https://developer.mozilla.org/en-US/docs/CSS/display + rdisplayswap = /^(none|table(?!-c[ea]).+)/, + rmargin = /^margin/, + rnumsplit = new RegExp( "^(" + core_pnum + ")(.*)$", "i" ), + rnumnonpx = new RegExp( "^(" + core_pnum + ")(?!px)[a-z%]+$", "i" ), + rrelNum = new RegExp( "^([-+])=(" + core_pnum + ")", "i" ), + elemdisplay = { BODY: "block" }, + + cssShow = { position: "absolute", visibility: "hidden", display: "block" }, + cssNormalTransform = { + letterSpacing: 0, + fontWeight: 400 + }, + + cssExpand = [ "Top", "Right", "Bottom", "Left" ], + cssPrefixes = [ "Webkit", "O", "Moz", "ms" ], + + eventsToggle = jQuery.fn.toggle; + +// return a css property mapped to a potentially vendor prefixed property +function vendorPropName( style, name ) { + + // shortcut for names that are not vendor prefixed + if ( name in style ) { + return name; + } + + // check for vendor prefixed names + var capName = name.charAt(0).toUpperCase() + name.slice(1), + origName = name, + i = cssPrefixes.length; + + while ( i-- ) { + name = cssPrefixes[ i ] + capName; + if ( name in style ) { + return name; + } + } + + return origName; +} + +function isHidden( elem, el ) { + elem = el || elem; + return jQuery.css( elem, "display" ) === "none" || !jQuery.contains( elem.ownerDocument, elem ); +} + +function showHide( elements, show ) { + var elem, display, + values = [], + index = 0, + length = elements.length; + + for ( ; index < length; index++ ) { + elem = elements[ index ]; + if ( !elem.style ) { + continue; + } + values[ index ] = jQuery._data( elem, "olddisplay" ); + if ( show ) { + // Reset the inline display of this element to learn if it is + // being hidden by cascaded rules or not + if ( !values[ index ] && elem.style.display === "none" ) { + elem.style.display = ""; + } + + // Set elements which have been overridden with display: none + // in a stylesheet to whatever the default browser style is + // for such an element + if ( elem.style.display === "" && isHidden( elem ) ) { + values[ index ] = jQuery._data( elem, "olddisplay", css_defaultDisplay(elem.nodeName) ); + } + } else { + display = curCSS( elem, "display" ); + + if ( !values[ index ] && display !== "none" ) { + jQuery._data( elem, "olddisplay", display ); + } + } + } + + // Set the display of most of the elements in a second loop + // to avoid the constant reflow + for ( index = 0; index < length; index++ ) { + elem = elements[ index ]; + if ( !elem.style ) { + continue; + } + if ( !show || elem.style.display === "none" || elem.style.display === "" ) { + elem.style.display = show ? values[ index ] || "" : "none"; + } + } + + return elements; +} + +jQuery.fn.extend({ + css: function( name, value ) { + return jQuery.access( this, function( elem, name, value ) { + return value !== undefined ? + jQuery.style( elem, name, value ) : + jQuery.css( elem, name ); + }, name, value, arguments.length > 1 ); + }, + show: function() { + return showHide( this, true ); + }, + hide: function() { + return showHide( this ); + }, + toggle: function( state, fn2 ) { + var bool = typeof state === "boolean"; + + if ( jQuery.isFunction( state ) && jQuery.isFunction( fn2 ) ) { + return eventsToggle.apply( this, arguments ); + } + + return this.each(function() { + if ( bool ? state : isHidden( this ) ) { + jQuery( this ).show(); + } else { + jQuery( this ).hide(); + } + }); + } +}); + +jQuery.extend({ + // Add in style property hooks for overriding the default + // behavior of getting and setting a style property + cssHooks: { + opacity: { + get: function( elem, computed ) { + if ( computed ) { + // We should always get a number back from opacity + var ret = curCSS( elem, "opacity" ); + return ret === "" ? "1" : ret; + + } + } + } + }, + + // Exclude the following css properties to add px + cssNumber: { + "fillOpacity": true, + "fontWeight": true, + "lineHeight": true, + "opacity": true, + "orphans": true, + "widows": true, + "zIndex": true, + "zoom": true + }, + + // Add in properties whose names you wish to fix before + // setting or getting the value + cssProps: { + // normalize float css property + "float": jQuery.support.cssFloat ? "cssFloat" : "styleFloat" + }, + + // Get and set the style property on a DOM Node + style: function( elem, name, value, extra ) { + // Don't set styles on text and comment nodes + if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) { + return; + } + + // Make sure that we're working with the right name + var ret, type, hooks, + origName = jQuery.camelCase( name ), + style = elem.style; + + name = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( style, origName ) ); + + // gets hook for the prefixed version + // followed by the unprefixed version + hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; + + // Check if we're setting a value + if ( value !== undefined ) { + type = typeof value; + + // convert relative number strings (+= or -=) to relative numbers. #7345 + if ( type === "string" && (ret = rrelNum.exec( value )) ) { + value = ( ret[1] + 1 ) * ret[2] + parseFloat( jQuery.css( elem, name ) ); + // Fixes bug #9237 + type = "number"; + } + + // Make sure that NaN and null values aren't set. See: #7116 + if ( value == null || type === "number" && isNaN( value ) ) { + return; + } + + // If a number was passed in, add 'px' to the (except for certain CSS properties) + if ( type === "number" && !jQuery.cssNumber[ origName ] ) { + value += "px"; + } + + // If a hook was provided, use that value, otherwise just set the specified value + if ( !hooks || !("set" in hooks) || (value = hooks.set( elem, value, extra )) !== undefined ) { + // Wrapped to prevent IE from throwing errors when 'invalid' values are provided + // Fixes bug #5509 + try { + style[ name ] = value; + } catch(e) {} + } + + } else { + // If a hook was provided get the non-computed value from there + if ( hooks && "get" in hooks && (ret = hooks.get( elem, false, extra )) !== undefined ) { + return ret; + } + + // Otherwise just get the value from the style object + return style[ name ]; + } + }, + + css: function( elem, name, numeric, extra ) { + var val, num, hooks, + origName = jQuery.camelCase( name ); + + // Make sure that we're working with the right name + name = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( elem.style, origName ) ); + + // gets hook for the prefixed version + // followed by the unprefixed version + hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; + + // If a hook was provided get the computed value from there + if ( hooks && "get" in hooks ) { + val = hooks.get( elem, true, extra ); + } + + // Otherwise, if a way to get the computed value exists, use that + if ( val === undefined ) { + val = curCSS( elem, name ); + } + + //convert "normal" to computed value + if ( val === "normal" && name in cssNormalTransform ) { + val = cssNormalTransform[ name ]; + } + + // Return, converting to number if forced or a qualifier was provided and val looks numeric + if ( numeric || extra !== undefined ) { + num = parseFloat( val ); + return numeric || jQuery.isNumeric( num ) ? num || 0 : val; + } + return val; + }, + + // A method for quickly swapping in/out CSS properties to get correct calculations + swap: function( elem, options, callback ) { + var ret, name, + old = {}; + + // Remember the old values, and insert the new ones + for ( name in options ) { + old[ name ] = elem.style[ name ]; + elem.style[ name ] = options[ name ]; + } + + ret = callback.call( elem ); + + // Revert the old values + for ( name in options ) { + elem.style[ name ] = old[ name ]; + } + + return ret; + } +}); + +// NOTE: To any future maintainer, we've window.getComputedStyle +// because jsdom on node.js will break without it. +if ( window.getComputedStyle ) { + curCSS = function( elem, name ) { + var ret, width, minWidth, maxWidth, + computed = window.getComputedStyle( elem, null ), + style = elem.style; + + if ( computed ) { + + // getPropertyValue is only needed for .css('filter') in IE9, see #12537 + ret = computed.getPropertyValue( name ) || computed[ name ]; + + if ( ret === "" && !jQuery.contains( elem.ownerDocument, elem ) ) { + ret = jQuery.style( elem, name ); + } + + // A tribute to the "awesome hack by Dean Edwards" + // Chrome < 17 and Safari 5.0 uses "computed value" instead of "used value" for margin-right + // Safari 5.1.7 (at least) returns percentage for a larger set of values, but width seems to be reliably pixels + // this is against the CSSOM draft spec: http://dev.w3.org/csswg/cssom/#resolved-values + if ( rnumnonpx.test( ret ) && rmargin.test( name ) ) { + width = style.width; + minWidth = style.minWidth; + maxWidth = style.maxWidth; + + style.minWidth = style.maxWidth = style.width = ret; + ret = computed.width; + + style.width = width; + style.minWidth = minWidth; + style.maxWidth = maxWidth; + } + } + + return ret; + }; +} else if ( document.documentElement.currentStyle ) { + curCSS = function( elem, name ) { + var left, rsLeft, + ret = elem.currentStyle && elem.currentStyle[ name ], + style = elem.style; + + // Avoid setting ret to empty string here + // so we don't default to auto + if ( ret == null && style && style[ name ] ) { + ret = style[ name ]; + } + + // From the awesome hack by Dean Edwards + // http://erik.eae.net/archives/2007/07/27/18.54.15/#comment-102291 + + // If we're not dealing with a regular pixel number + // but a number that has a weird ending, we need to convert it to pixels + // but not position css attributes, as those are proportional to the parent element instead + // and we can't measure the parent instead because it might trigger a "stacking dolls" problem + if ( rnumnonpx.test( ret ) && !rposition.test( name ) ) { + + // Remember the original values + left = style.left; + rsLeft = elem.runtimeStyle && elem.runtimeStyle.left; + + // Put in the new values to get a computed value out + if ( rsLeft ) { + elem.runtimeStyle.left = elem.currentStyle.left; + } + style.left = name === "fontSize" ? "1em" : ret; + ret = style.pixelLeft + "px"; + + // Revert the changed values + style.left = left; + if ( rsLeft ) { + elem.runtimeStyle.left = rsLeft; + } + } + + return ret === "" ? "auto" : ret; + }; +} + +function setPositiveNumber( elem, value, subtract ) { + var matches = rnumsplit.exec( value ); + return matches ? + Math.max( 0, matches[ 1 ] - ( subtract || 0 ) ) + ( matches[ 2 ] || "px" ) : + value; +} + +function augmentWidthOrHeight( elem, name, extra, isBorderBox ) { + var i = extra === ( isBorderBox ? "border" : "content" ) ? + // If we already have the right measurement, avoid augmentation + 4 : + // Otherwise initialize for horizontal or vertical properties + name === "width" ? 1 : 0, + + val = 0; + + for ( ; i < 4; i += 2 ) { + // both box models exclude margin, so add it if we want it + if ( extra === "margin" ) { + // we use jQuery.css instead of curCSS here + // because of the reliableMarginRight CSS hook! + val += jQuery.css( elem, extra + cssExpand[ i ], true ); + } + + // From this point on we use curCSS for maximum performance (relevant in animations) + if ( isBorderBox ) { + // border-box includes padding, so remove it if we want content + if ( extra === "content" ) { + val -= parseFloat( curCSS( elem, "padding" + cssExpand[ i ] ) ) || 0; + } + + // at this point, extra isn't border nor margin, so remove border + if ( extra !== "margin" ) { + val -= parseFloat( curCSS( elem, "border" + cssExpand[ i ] + "Width" ) ) || 0; + } + } else { + // at this point, extra isn't content, so add padding + val += parseFloat( curCSS( elem, "padding" + cssExpand[ i ] ) ) || 0; + + // at this point, extra isn't content nor padding, so add border + if ( extra !== "padding" ) { + val += parseFloat( curCSS( elem, "border" + cssExpand[ i ] + "Width" ) ) || 0; + } + } + } + + return val; +} + +function getWidthOrHeight( elem, name, extra ) { + + // Start with offset property, which is equivalent to the border-box value + var val = name === "width" ? elem.offsetWidth : elem.offsetHeight, + valueIsBorderBox = true, + isBorderBox = jQuery.support.boxSizing && jQuery.css( elem, "boxSizing" ) === "border-box"; + + // some non-html elements return undefined for offsetWidth, so check for null/undefined + // svg - https://bugzilla.mozilla.org/show_bug.cgi?id=649285 + // MathML - https://bugzilla.mozilla.org/show_bug.cgi?id=491668 + if ( val <= 0 || val == null ) { + // Fall back to computed then uncomputed css if necessary + val = curCSS( elem, name ); + if ( val < 0 || val == null ) { + val = elem.style[ name ]; + } + + // Computed unit is not pixels. Stop here and return. + if ( rnumnonpx.test(val) ) { + return val; + } + + // we need the check for style in case a browser which returns unreliable values + // for getComputedStyle silently falls back to the reliable elem.style + valueIsBorderBox = isBorderBox && ( jQuery.support.boxSizingReliable || val === elem.style[ name ] ); + + // Normalize "", auto, and prepare for extra + val = parseFloat( val ) || 0; + } + + // use the active box-sizing model to add/subtract irrelevant styles + return ( val + + augmentWidthOrHeight( + elem, + name, + extra || ( isBorderBox ? "border" : "content" ), + valueIsBorderBox + ) + ) + "px"; +} + + +// Try to determine the default display value of an element +function css_defaultDisplay( nodeName ) { + if ( elemdisplay[ nodeName ] ) { + return elemdisplay[ nodeName ]; + } + + var elem = jQuery( "<" + nodeName + ">" ).appendTo( document.body ), + display = elem.css("display"); + elem.remove(); + + // If the simple way fails, + // get element's real default display by attaching it to a temp iframe + if ( display === "none" || display === "" ) { + // Use the already-created iframe if possible + iframe = document.body.appendChild( + iframe || jQuery.extend( document.createElement("iframe"), { + frameBorder: 0, + width: 0, + height: 0 + }) + ); + + // Create a cacheable copy of the iframe document on first call. + // IE and Opera will allow us to reuse the iframeDoc without re-writing the fake HTML + // document to it; WebKit & Firefox won't allow reusing the iframe document. + if ( !iframeDoc || !iframe.createElement ) { + iframeDoc = ( iframe.contentWindow || iframe.contentDocument ).document; + iframeDoc.write("<!doctype html><html><body>"); + iframeDoc.close(); + } + + elem = iframeDoc.body.appendChild( iframeDoc.createElement(nodeName) ); + + display = curCSS( elem, "display" ); + document.body.removeChild( iframe ); + } + + // Store the correct default display + elemdisplay[ nodeName ] = display; + + return display; +} + +jQuery.each([ "height", "width" ], function( i, name ) { + jQuery.cssHooks[ name ] = { + get: function( elem, computed, extra ) { + if ( computed ) { + // certain elements can have dimension info if we invisibly show them + // however, it must have a current display style that would benefit from this + if ( elem.offsetWidth === 0 && rdisplayswap.test( curCSS( elem, "display" ) ) ) { + return jQuery.swap( elem, cssShow, function() { + return getWidthOrHeight( elem, name, extra ); + }); + } else { + return getWidthOrHeight( elem, name, extra ); + } + } + }, + + set: function( elem, value, extra ) { + return setPositiveNumber( elem, value, extra ? + augmentWidthOrHeight( + elem, + name, + extra, + jQuery.support.boxSizing && jQuery.css( elem, "boxSizing" ) === "border-box" + ) : 0 + ); + } + }; +}); + +if ( !jQuery.support.opacity ) { + jQuery.cssHooks.opacity = { + get: function( elem, computed ) { + // IE uses filters for opacity + return ropacity.test( (computed && elem.currentStyle ? elem.currentStyle.filter : elem.style.filter) || "" ) ? + ( 0.01 * parseFloat( RegExp.$1 ) ) + "" : + computed ? "1" : ""; + }, + + set: function( elem, value ) { + var style = elem.style, + currentStyle = elem.currentStyle, + opacity = jQuery.isNumeric( value ) ? "alpha(opacity=" + value * 100 + ")" : "", + filter = currentStyle && currentStyle.filter || style.filter || ""; + + // IE has trouble with opacity if it does not have layout + // Force it by setting the zoom level + style.zoom = 1; + + // if setting opacity to 1, and no other filters exist - attempt to remove filter attribute #6652 + if ( value >= 1 && jQuery.trim( filter.replace( ralpha, "" ) ) === "" && + style.removeAttribute ) { + + // Setting style.filter to null, "" & " " still leave "filter:" in the cssText + // if "filter:" is present at all, clearType is disabled, we want to avoid this + // style.removeAttribute is IE Only, but so apparently is this code path... + style.removeAttribute( "filter" ); + + // if there there is no filter style applied in a css rule, we are done + if ( currentStyle && !currentStyle.filter ) { + return; + } + } + + // otherwise, set new filter values + style.filter = ralpha.test( filter ) ? + filter.replace( ralpha, opacity ) : + filter + " " + opacity; + } + }; +} + +// These hooks cannot be added until DOM ready because the support test +// for it is not run until after DOM ready +jQuery(function() { + if ( !jQuery.support.reliableMarginRight ) { + jQuery.cssHooks.marginRight = { + get: function( elem, computed ) { + // WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right + // Work around by temporarily setting element display to inline-block + return jQuery.swap( elem, { "display": "inline-block" }, function() { + if ( computed ) { + return curCSS( elem, "marginRight" ); + } + }); + } + }; + } + + // Webkit bug: https://bugs.webkit.org/show_bug.cgi?id=29084 + // getComputedStyle returns percent when specified for top/left/bottom/right + // rather than make the css module depend on the offset module, we just check for it here + if ( !jQuery.support.pixelPosition && jQuery.fn.position ) { + jQuery.each( [ "top", "left" ], function( i, prop ) { + jQuery.cssHooks[ prop ] = { + get: function( elem, computed ) { + if ( computed ) { + var ret = curCSS( elem, prop ); + // if curCSS returns percentage, fallback to offset + return rnumnonpx.test( ret ) ? jQuery( elem ).position()[ prop ] + "px" : ret; + } + } + }; + }); + } + +}); + +if ( jQuery.expr && jQuery.expr.filters ) { + jQuery.expr.filters.hidden = function( elem ) { + return ( elem.offsetWidth === 0 && elem.offsetHeight === 0 ) || (!jQuery.support.reliableHiddenOffsets && ((elem.style && elem.style.display) || curCSS( elem, "display" )) === "none"); + }; + + jQuery.expr.filters.visible = function( elem ) { + return !jQuery.expr.filters.hidden( elem ); + }; +} + +// These hooks are used by animate to expand properties +jQuery.each({ + margin: "", + padding: "", + border: "Width" +}, function( prefix, suffix ) { + jQuery.cssHooks[ prefix + suffix ] = { + expand: function( value ) { + var i, + + // assumes a single number if not a string + parts = typeof value === "string" ? value.split(" ") : [ value ], + expanded = {}; + + for ( i = 0; i < 4; i++ ) { + expanded[ prefix + cssExpand[ i ] + suffix ] = + parts[ i ] || parts[ i - 2 ] || parts[ 0 ]; + } + + return expanded; + } + }; + + if ( !rmargin.test( prefix ) ) { + jQuery.cssHooks[ prefix + suffix ].set = setPositiveNumber; + } +}); +var r20 = /%20/g, + rbracket = /\[\]$/, + rCRLF = /\r?\n/g, + rinput = /^(?:color|date|datetime|datetime-local|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i, + rselectTextarea = /^(?:select|textarea)/i; + +jQuery.fn.extend({ + serialize: function() { + return jQuery.param( this.serializeArray() ); + }, + serializeArray: function() { + return this.map(function(){ + return this.elements ? jQuery.makeArray( this.elements ) : this; + }) + .filter(function(){ + return this.name && !this.disabled && + ( this.checked || rselectTextarea.test( this.nodeName ) || + rinput.test( this.type ) ); + }) + .map(function( i, elem ){ + var val = jQuery( this ).val(); + + return val == null ? + null : + jQuery.isArray( val ) ? + jQuery.map( val, function( val, i ){ + return { name: elem.name, value: val.replace( rCRLF, "\r\n" ) }; + }) : + { name: elem.name, value: val.replace( rCRLF, "\r\n" ) }; + }).get(); + } +}); + +//Serialize an array of form elements or a set of +//key/values into a query string +jQuery.param = function( a, traditional ) { + var prefix, + s = [], + add = function( key, value ) { + // If value is a function, invoke it and return its value + value = jQuery.isFunction( value ) ? value() : ( value == null ? "" : value ); + s[ s.length ] = encodeURIComponent( key ) + "=" + encodeURIComponent( value ); + }; + + // Set traditional to true for jQuery <= 1.3.2 behavior. + if ( traditional === undefined ) { + traditional = jQuery.ajaxSettings && jQuery.ajaxSettings.traditional; + } + + // If an array was passed in, assume that it is an array of form elements. + if ( jQuery.isArray( a ) || ( a.jquery && !jQuery.isPlainObject( a ) ) ) { + // Serialize the form elements + jQuery.each( a, function() { + add( this.name, this.value ); + }); + + } else { + // If traditional, encode the "old" way (the way 1.3.2 or older + // did it), otherwise encode params recursively. + for ( prefix in a ) { + buildParams( prefix, a[ prefix ], traditional, add ); + } + } + + // Return the resulting serialization + return s.join( "&" ).replace( r20, "+" ); +}; + +function buildParams( prefix, obj, traditional, add ) { + var name; + + if ( jQuery.isArray( obj ) ) { + // Serialize array item. + jQuery.each( obj, function( i, v ) { + if ( traditional || rbracket.test( prefix ) ) { + // Treat each array item as a scalar. + add( prefix, v ); + + } else { + // If array item is non-scalar (array or object), encode its + // numeric index to resolve deserialization ambiguity issues. + // Note that rack (as of 1.0.0) can't currently deserialize + // nested arrays properly, and attempting to do so may cause + // a server error. Possible fixes are to modify rack's + // deserialization algorithm or to provide an option or flag + // to force array serialization to be shallow. + buildParams( prefix + "[" + ( typeof v === "object" ? i : "" ) + "]", v, traditional, add ); + } + }); + + } else if ( !traditional && jQuery.type( obj ) === "object" ) { + // Serialize object item. + for ( name in obj ) { + buildParams( prefix + "[" + name + "]", obj[ name ], traditional, add ); + } + + } else { + // Serialize scalar item. + add( prefix, obj ); + } +} +var + // Document location + ajaxLocParts, + ajaxLocation, + + rhash = /#.*$/, + rheaders = /^(.*?):[ \t]*([^\r\n]*)\r?$/mg, // IE leaves an \r character at EOL + // #7653, #8125, #8152: local protocol detection + rlocalProtocol = /^(?:about|app|app\-storage|.+\-extension|file|res|widget):$/, + rnoContent = /^(?:GET|HEAD)$/, + rprotocol = /^\/\//, + rquery = /\?/, + rscript = /<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, + rts = /([?&])_=[^&]*/, + rurl = /^([\w\+\.\-]+:)(?:\/\/([^\/?#:]*)(?::(\d+)|)|)/, + + // Keep a copy of the old load method + _load = jQuery.fn.load, + + /* Prefilters + * 1) They are useful to introduce custom dataTypes (see ajax/jsonp.js for an example) + * 2) These are called: + * - BEFORE asking for a transport + * - AFTER param serialization (s.data is a string if s.processData is true) + * 3) key is the dataType + * 4) the catchall symbol "*" can be used + * 5) execution will start with transport dataType and THEN continue down to "*" if needed + */ + prefilters = {}, + + /* Transports bindings + * 1) key is the dataType + * 2) the catchall symbol "*" can be used + * 3) selection will start with transport dataType and THEN go to "*" if needed + */ + transports = {}, + + // Avoid comment-prolog char sequence (#10098); must appease lint and evade compression + allTypes = ["*/"] + ["*"]; + +// #8138, IE may throw an exception when accessing +// a field from window.location if document.domain has been set +try { + ajaxLocation = location.href; +} catch( e ) { + // Use the href attribute of an A element + // since IE will modify it given document.location + ajaxLocation = document.createElement( "a" ); + ajaxLocation.href = ""; + ajaxLocation = ajaxLocation.href; +} + +// Segment location into parts +ajaxLocParts = rurl.exec( ajaxLocation.toLowerCase() ) || []; + +// Base "constructor" for jQuery.ajaxPrefilter and jQuery.ajaxTransport +function addToPrefiltersOrTransports( structure ) { + + // dataTypeExpression is optional and defaults to "*" + return function( dataTypeExpression, func ) { + + if ( typeof dataTypeExpression !== "string" ) { + func = dataTypeExpression; + dataTypeExpression = "*"; + } + + var dataType, list, placeBefore, + dataTypes = dataTypeExpression.toLowerCase().split( core_rspace ), + i = 0, + length = dataTypes.length; + + if ( jQuery.isFunction( func ) ) { + // For each dataType in the dataTypeExpression + for ( ; i < length; i++ ) { + dataType = dataTypes[ i ]; + // We control if we're asked to add before + // any existing element + placeBefore = /^\+/.test( dataType ); + if ( placeBefore ) { + dataType = dataType.substr( 1 ) || "*"; + } + list = structure[ dataType ] = structure[ dataType ] || []; + // then we add to the structure accordingly + list[ placeBefore ? "unshift" : "push" ]( func ); + } + } + }; +} + +// Base inspection function for prefilters and transports +function inspectPrefiltersOrTransports( structure, options, originalOptions, jqXHR, + dataType /* internal */, inspected /* internal */ ) { + + dataType = dataType || options.dataTypes[ 0 ]; + inspected = inspected || {}; + + inspected[ dataType ] = true; + + var selection, + list = structure[ dataType ], + i = 0, + length = list ? list.length : 0, + executeOnly = ( structure === prefilters ); + + for ( ; i < length && ( executeOnly || !selection ); i++ ) { + selection = list[ i ]( options, originalOptions, jqXHR ); + // If we got redirected to another dataType + // we try there if executing only and not done already + if ( typeof selection === "string" ) { + if ( !executeOnly || inspected[ selection ] ) { + selection = undefined; + } else { + options.dataTypes.unshift( selection ); + selection = inspectPrefiltersOrTransports( + structure, options, originalOptions, jqXHR, selection, inspected ); + } + } + } + // If we're only executing or nothing was selected + // we try the catchall dataType if not done already + if ( ( executeOnly || !selection ) && !inspected[ "*" ] ) { + selection = inspectPrefiltersOrTransports( + structure, options, originalOptions, jqXHR, "*", inspected ); + } + // unnecessary when only executing (prefilters) + // but it'll be ignored by the caller in that case + return selection; +} + +// A special extend for ajax options +// that takes "flat" options (not to be deep extended) +// Fixes #9887 +function ajaxExtend( target, src ) { + var key, deep, + flatOptions = jQuery.ajaxSettings.flatOptions || {}; + for ( key in src ) { + if ( src[ key ] !== undefined ) { + ( flatOptions[ key ] ? target : ( deep || ( deep = {} ) ) )[ key ] = src[ key ]; + } + } + if ( deep ) { + jQuery.extend( true, target, deep ); + } +} + +jQuery.fn.load = function( url, params, callback ) { + if ( typeof url !== "string" && _load ) { + return _load.apply( this, arguments ); + } + + // Don't do a request if no elements are being requested + if ( !this.length ) { + return this; + } + + var selector, type, response, + self = this, + off = url.indexOf(" "); + + if ( off >= 0 ) { + selector = url.slice( off, url.length ); + url = url.slice( 0, off ); + } + + // If it's a function + if ( jQuery.isFunction( params ) ) { + + // We assume that it's the callback + callback = params; + params = undefined; + + // Otherwise, build a param string + } else if ( params && typeof params === "object" ) { + type = "POST"; + } + + // Request the remote document + jQuery.ajax({ + url: url, + + // if "type" variable is undefined, then "GET" method will be used + type: type, + dataType: "html", + data: params, + complete: function( jqXHR, status ) { + if ( callback ) { + self.each( callback, response || [ jqXHR.responseText, status, jqXHR ] ); + } + } + }).done(function( responseText ) { + + // Save response for use in complete callback + response = arguments; + + // See if a selector was specified + self.html( selector ? + + // Create a dummy div to hold the results + jQuery("<div>") + + // inject the contents of the document in, removing the scripts + // to avoid any 'Permission Denied' errors in IE + .append( responseText.replace( rscript, "" ) ) + + // Locate the specified elements + .find( selector ) : + + // If not, just inject the full result + responseText ); + + }); + + return this; +}; + +// Attach a bunch of functions for handling common AJAX events +jQuery.each( "ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split( " " ), function( i, o ){ + jQuery.fn[ o ] = function( f ){ + return this.on( o, f ); + }; +}); + +jQuery.each( [ "get", "post" ], function( i, method ) { + jQuery[ method ] = function( url, data, callback, type ) { + // shift arguments if data argument was omitted + if ( jQuery.isFunction( data ) ) { + type = type || callback; + callback = data; + data = undefined; + } + + return jQuery.ajax({ + type: method, + url: url, + data: data, + success: callback, + dataType: type + }); + }; +}); + +jQuery.extend({ + + getScript: function( url, callback ) { + return jQuery.get( url, undefined, callback, "script" ); + }, + + getJSON: function( url, data, callback ) { + return jQuery.get( url, data, callback, "json" ); + }, + + // Creates a full fledged settings object into target + // with both ajaxSettings and settings fields. + // If target is omitted, writes into ajaxSettings. + ajaxSetup: function( target, settings ) { + if ( settings ) { + // Building a settings object + ajaxExtend( target, jQuery.ajaxSettings ); + } else { + // Extending ajaxSettings + settings = target; + target = jQuery.ajaxSettings; + } + ajaxExtend( target, settings ); + return target; + }, + + ajaxSettings: { + url: ajaxLocation, + isLocal: rlocalProtocol.test( ajaxLocParts[ 1 ] ), + global: true, + type: "GET", + contentType: "application/x-www-form-urlencoded; charset=UTF-8", + processData: true, + async: true, + /* + timeout: 0, + data: null, + dataType: null, + username: null, + password: null, + cache: null, + throws: false, + traditional: false, + headers: {}, + */ + + accepts: { + xml: "application/xml, text/xml", + html: "text/html", + text: "text/plain", + json: "application/json, text/javascript", + "*": allTypes + }, + + contents: { + xml: /xml/, + html: /html/, + json: /json/ + }, + + responseFields: { + xml: "responseXML", + text: "responseText" + }, + + // List of data converters + // 1) key format is "source_type destination_type" (a single space in-between) + // 2) the catchall symbol "*" can be used for source_type + converters: { + + // Convert anything to text + "* text": window.String, + + // Text to html (true = no transformation) + "text html": true, + + // Evaluate text as a json expression + "text json": jQuery.parseJSON, + + // Parse text as xml + "text xml": jQuery.parseXML + }, + + // For options that shouldn't be deep extended: + // you can add your own custom options here if + // and when you create one that shouldn't be + // deep extended (see ajaxExtend) + flatOptions: { + context: true, + url: true + } + }, + + ajaxPrefilter: addToPrefiltersOrTransports( prefilters ), + ajaxTransport: addToPrefiltersOrTransports( transports ), + + // Main method + ajax: function( url, options ) { + + // If url is an object, simulate pre-1.5 signature + if ( typeof url === "object" ) { + options = url; + url = undefined; + } + + // Force options to be an object + options = options || {}; + + var // ifModified key + ifModifiedKey, + // Response headers + responseHeadersString, + responseHeaders, + // transport + transport, + // timeout handle + timeoutTimer, + // Cross-domain detection vars + parts, + // To know if global events are to be dispatched + fireGlobals, + // Loop variable + i, + // Create the final options object + s = jQuery.ajaxSetup( {}, options ), + // Callbacks context + callbackContext = s.context || s, + // Context for global events + // It's the callbackContext if one was provided in the options + // and if it's a DOM node or a jQuery collection + globalEventContext = callbackContext !== s && + ( callbackContext.nodeType || callbackContext instanceof jQuery ) ? + jQuery( callbackContext ) : jQuery.event, + // Deferreds + deferred = jQuery.Deferred(), + completeDeferred = jQuery.Callbacks( "once memory" ), + // Status-dependent callbacks + statusCode = s.statusCode || {}, + // Headers (they are sent all at once) + requestHeaders = {}, + requestHeadersNames = {}, + // The jqXHR state + state = 0, + // Default abort message + strAbort = "canceled", + // Fake xhr + jqXHR = { + + readyState: 0, + + // Caches the header + setRequestHeader: function( name, value ) { + if ( !state ) { + var lname = name.toLowerCase(); + name = requestHeadersNames[ lname ] = requestHeadersNames[ lname ] || name; + requestHeaders[ name ] = value; + } + return this; + }, + + // Raw string + getAllResponseHeaders: function() { + return state === 2 ? responseHeadersString : null; + }, + + // Builds headers hashtable if needed + getResponseHeader: function( key ) { + var match; + if ( state === 2 ) { + if ( !responseHeaders ) { + responseHeaders = {}; + while( ( match = rheaders.exec( responseHeadersString ) ) ) { + responseHeaders[ match[1].toLowerCase() ] = match[ 2 ]; + } + } + match = responseHeaders[ key.toLowerCase() ]; + } + return match === undefined ? null : match; + }, + + // Overrides response content-type header + overrideMimeType: function( type ) { + if ( !state ) { + s.mimeType = type; + } + return this; + }, + + // Cancel the request + abort: function( statusText ) { + statusText = statusText || strAbort; + if ( transport ) { + transport.abort( statusText ); + } + done( 0, statusText ); + return this; + } + }; + + // Callback for when everything is done + // It is defined here because jslint complains if it is declared + // at the end of the function (which would be more logical and readable) + function done( status, nativeStatusText, responses, headers ) { + var isSuccess, success, error, response, modified, + statusText = nativeStatusText; + + // Called once + if ( state === 2 ) { + return; + } + + // State is "done" now + state = 2; + + // Clear timeout if it exists + if ( timeoutTimer ) { + clearTimeout( timeoutTimer ); + } + + // Dereference transport for early garbage collection + // (no matter how long the jqXHR object will be used) + transport = undefined; + + // Cache response headers + responseHeadersString = headers || ""; + + // Set readyState + jqXHR.readyState = status > 0 ? 4 : 0; + + // Get response data + if ( responses ) { + response = ajaxHandleResponses( s, jqXHR, responses ); + } + + // If successful, handle type chaining + if ( status >= 200 && status < 300 || status === 304 ) { + + // Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode. + if ( s.ifModified ) { + + modified = jqXHR.getResponseHeader("Last-Modified"); + if ( modified ) { + jQuery.lastModified[ ifModifiedKey ] = modified; + } + modified = jqXHR.getResponseHeader("Etag"); + if ( modified ) { + jQuery.etag[ ifModifiedKey ] = modified; + } + } + + // If not modified + if ( status === 304 ) { + + statusText = "notmodified"; + isSuccess = true; + + // If we have data + } else { + + isSuccess = ajaxConvert( s, response ); + statusText = isSuccess.state; + success = isSuccess.data; + error = isSuccess.error; + isSuccess = !error; + } + } else { + // We extract error from statusText + // then normalize statusText and status for non-aborts + error = statusText; + if ( !statusText || status ) { + statusText = "error"; + if ( status < 0 ) { + status = 0; + } + } + } + + // Set data for the fake xhr object + jqXHR.status = status; + jqXHR.statusText = ( nativeStatusText || statusText ) + ""; + + // Success/Error + if ( isSuccess ) { + deferred.resolveWith( callbackContext, [ success, statusText, jqXHR ] ); + } else { + deferred.rejectWith( callbackContext, [ jqXHR, statusText, error ] ); + } + + // Status-dependent callbacks + jqXHR.statusCode( statusCode ); + statusCode = undefined; + + if ( fireGlobals ) { + globalEventContext.trigger( "ajax" + ( isSuccess ? "Success" : "Error" ), + [ jqXHR, s, isSuccess ? success : error ] ); + } + + // Complete + completeDeferred.fireWith( callbackContext, [ jqXHR, statusText ] ); + + if ( fireGlobals ) { + globalEventContext.trigger( "ajaxComplete", [ jqXHR, s ] ); + // Handle the global AJAX counter + if ( !( --jQuery.active ) ) { + jQuery.event.trigger( "ajaxStop" ); + } + } + } + + // Attach deferreds + deferred.promise( jqXHR ); + jqXHR.success = jqXHR.done; + jqXHR.error = jqXHR.fail; + jqXHR.complete = completeDeferred.add; + + // Status-dependent callbacks + jqXHR.statusCode = function( map ) { + if ( map ) { + var tmp; + if ( state < 2 ) { + for ( tmp in map ) { + statusCode[ tmp ] = [ statusCode[tmp], map[tmp] ]; + } + } else { + tmp = map[ jqXHR.status ]; + jqXHR.always( tmp ); + } + } + return this; + }; + + // Remove hash character (#7531: and string promotion) + // Add protocol if not provided (#5866: IE7 issue with protocol-less urls) + // We also use the url parameter if available + s.url = ( ( url || s.url ) + "" ).replace( rhash, "" ).replace( rprotocol, ajaxLocParts[ 1 ] + "//" ); + + // Extract dataTypes list + s.dataTypes = jQuery.trim( s.dataType || "*" ).toLowerCase().split( core_rspace ); + + // A cross-domain request is in order when we have a protocol:host:port mismatch + if ( s.crossDomain == null ) { + parts = rurl.exec( s.url.toLowerCase() ); + s.crossDomain = !!( parts && + ( parts[ 1 ] !== ajaxLocParts[ 1 ] || parts[ 2 ] !== ajaxLocParts[ 2 ] || + ( parts[ 3 ] || ( parts[ 1 ] === "http:" ? 80 : 443 ) ) != + ( ajaxLocParts[ 3 ] || ( ajaxLocParts[ 1 ] === "http:" ? 80 : 443 ) ) ) + ); + } + + // Convert data if not already a string + if ( s.data && s.processData && typeof s.data !== "string" ) { + s.data = jQuery.param( s.data, s.traditional ); + } + + // Apply prefilters + inspectPrefiltersOrTransports( prefilters, s, options, jqXHR ); + + // If request was aborted inside a prefilter, stop there + if ( state === 2 ) { + return jqXHR; + } + + // We can fire global events as of now if asked to + fireGlobals = s.global; + + // Uppercase the type + s.type = s.type.toUpperCase(); + + // Determine if request has content + s.hasContent = !rnoContent.test( s.type ); + + // Watch for a new set of requests + if ( fireGlobals && jQuery.active++ === 0 ) { + jQuery.event.trigger( "ajaxStart" ); + } + + // More options handling for requests with no content + if ( !s.hasContent ) { + + // If data is available, append data to url + if ( s.data ) { + s.url += ( rquery.test( s.url ) ? "&" : "?" ) + s.data; + // #9682: remove data so that it's not used in an eventual retry + delete s.data; + } + + // Get ifModifiedKey before adding the anti-cache parameter + ifModifiedKey = s.url; + + // Add anti-cache in url if needed + if ( s.cache === false ) { + + var ts = jQuery.now(), + // try replacing _= if it is there + ret = s.url.replace( rts, "$1_=" + ts ); + + // if nothing was replaced, add timestamp to the end + s.url = ret + ( ( ret === s.url ) ? ( rquery.test( s.url ) ? "&" : "?" ) + "_=" + ts : "" ); + } + } + + // Set the correct header, if data is being sent + if ( s.data && s.hasContent && s.contentType !== false || options.contentType ) { + jqXHR.setRequestHeader( "Content-Type", s.contentType ); + } + + // Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode. + if ( s.ifModified ) { + ifModifiedKey = ifModifiedKey || s.url; + if ( jQuery.lastModified[ ifModifiedKey ] ) { + jqXHR.setRequestHeader( "If-Modified-Since", jQuery.lastModified[ ifModifiedKey ] ); + } + if ( jQuery.etag[ ifModifiedKey ] ) { + jqXHR.setRequestHeader( "If-None-Match", jQuery.etag[ ifModifiedKey ] ); + } + } + + // Set the Accepts header for the server, depending on the dataType + jqXHR.setRequestHeader( + "Accept", + s.dataTypes[ 0 ] && s.accepts[ s.dataTypes[0] ] ? + s.accepts[ s.dataTypes[0] ] + ( s.dataTypes[ 0 ] !== "*" ? ", " + allTypes + "; q=0.01" : "" ) : + s.accepts[ "*" ] + ); + + // Check for headers option + for ( i in s.headers ) { + jqXHR.setRequestHeader( i, s.headers[ i ] ); + } + + // Allow custom headers/mimetypes and early abort + if ( s.beforeSend && ( s.beforeSend.call( callbackContext, jqXHR, s ) === false || state === 2 ) ) { + // Abort if not done already and return + return jqXHR.abort(); + + } + + // aborting is no longer a cancellation + strAbort = "abort"; + + // Install callbacks on deferreds + for ( i in { success: 1, error: 1, complete: 1 } ) { + jqXHR[ i ]( s[ i ] ); + } + + // Get transport + transport = inspectPrefiltersOrTransports( transports, s, options, jqXHR ); + + // If no transport, we auto-abort + if ( !transport ) { + done( -1, "No Transport" ); + } else { + jqXHR.readyState = 1; + // Send global event + if ( fireGlobals ) { + globalEventContext.trigger( "ajaxSend", [ jqXHR, s ] ); + } + // Timeout + if ( s.async && s.timeout > 0 ) { + timeoutTimer = setTimeout( function(){ + jqXHR.abort( "timeout" ); + }, s.timeout ); + } + + try { + state = 1; + transport.send( requestHeaders, done ); + } catch (e) { + // Propagate exception as error if not done + if ( state < 2 ) { + done( -1, e ); + // Simply rethrow otherwise + } else { + throw e; + } + } + } + + return jqXHR; + }, + + // Counter for holding the number of active queries + active: 0, + + // Last-Modified header cache for next request + lastModified: {}, + etag: {} + +}); + +/* Handles responses to an ajax request: + * - sets all responseXXX fields accordingly + * - finds the right dataType (mediates between content-type and expected dataType) + * - returns the corresponding response + */ +function ajaxHandleResponses( s, jqXHR, responses ) { + + var ct, type, finalDataType, firstDataType, + contents = s.contents, + dataTypes = s.dataTypes, + responseFields = s.responseFields; + + // Fill responseXXX fields + for ( type in responseFields ) { + if ( type in responses ) { + jqXHR[ responseFields[type] ] = responses[ type ]; + } + } + + // Remove auto dataType and get content-type in the process + while( dataTypes[ 0 ] === "*" ) { + dataTypes.shift(); + if ( ct === undefined ) { + ct = s.mimeType || jqXHR.getResponseHeader( "content-type" ); + } + } + + // Check if we're dealing with a known content-type + if ( ct ) { + for ( type in contents ) { + if ( contents[ type ] && contents[ type ].test( ct ) ) { + dataTypes.unshift( type ); + break; + } + } + } + + // Check to see if we have a response for the expected dataType + if ( dataTypes[ 0 ] in responses ) { + finalDataType = dataTypes[ 0 ]; + } else { + // Try convertible dataTypes + for ( type in responses ) { + if ( !dataTypes[ 0 ] || s.converters[ type + " " + dataTypes[0] ] ) { + finalDataType = type; + break; + } + if ( !firstDataType ) { + firstDataType = type; + } + } + // Or just use first one + finalDataType = finalDataType || firstDataType; + } + + // If we found a dataType + // We add the dataType to the list if needed + // and return the corresponding response + if ( finalDataType ) { + if ( finalDataType !== dataTypes[ 0 ] ) { + dataTypes.unshift( finalDataType ); + } + return responses[ finalDataType ]; + } +} + +// Chain conversions given the request and the original response +function ajaxConvert( s, response ) { + + var conv, conv2, current, tmp, + // Work with a copy of dataTypes in case we need to modify it for conversion + dataTypes = s.dataTypes.slice(), + prev = dataTypes[ 0 ], + converters = {}, + i = 0; + + // Apply the dataFilter if provided + if ( s.dataFilter ) { + response = s.dataFilter( response, s.dataType ); + } + + // Create converters map with lowercased keys + if ( dataTypes[ 1 ] ) { + for ( conv in s.converters ) { + converters[ conv.toLowerCase() ] = s.converters[ conv ]; + } + } + + // Convert to each sequential dataType, tolerating list modification + for ( ; (current = dataTypes[++i]); ) { + + // There's only work to do if current dataType is non-auto + if ( current !== "*" ) { + + // Convert response if prev dataType is non-auto and differs from current + if ( prev !== "*" && prev !== current ) { + + // Seek a direct converter + conv = converters[ prev + " " + current ] || converters[ "* " + current ]; + + // If none found, seek a pair + if ( !conv ) { + for ( conv2 in converters ) { + + // If conv2 outputs current + tmp = conv2.split(" "); + if ( tmp[ 1 ] === current ) { + + // If prev can be converted to accepted input + conv = converters[ prev + " " + tmp[ 0 ] ] || + converters[ "* " + tmp[ 0 ] ]; + if ( conv ) { + // Condense equivalence converters + if ( conv === true ) { + conv = converters[ conv2 ]; + + // Otherwise, insert the intermediate dataType + } else if ( converters[ conv2 ] !== true ) { + current = tmp[ 0 ]; + dataTypes.splice( i--, 0, current ); + } + + break; + } + } + } + } + + // Apply converter (if not an equivalence) + if ( conv !== true ) { + + // Unless errors are allowed to bubble, catch and return them + if ( conv && s["throws"] ) { + response = conv( response ); + } else { + try { + response = conv( response ); + } catch ( e ) { + return { state: "parsererror", error: conv ? e : "No conversion from " + prev + " to " + current }; + } + } + } + } + + // Update prev for next iteration + prev = current; + } + } + + return { state: "success", data: response }; +} +var oldCallbacks = [], + rquestion = /\?/, + rjsonp = /(=)\?(?=&|$)|\?\?/, + nonce = jQuery.now(); + +// Default jsonp settings +jQuery.ajaxSetup({ + jsonp: "callback", + jsonpCallback: function() { + var callback = oldCallbacks.pop() || ( jQuery.expando + "_" + ( nonce++ ) ); + this[ callback ] = true; + return callback; + } +}); + +// Detect, normalize options and install callbacks for jsonp requests +jQuery.ajaxPrefilter( "json jsonp", function( s, originalSettings, jqXHR ) { + + var callbackName, overwritten, responseContainer, + data = s.data, + url = s.url, + hasCallback = s.jsonp !== false, + replaceInUrl = hasCallback && rjsonp.test( url ), + replaceInData = hasCallback && !replaceInUrl && typeof data === "string" && + !( s.contentType || "" ).indexOf("application/x-www-form-urlencoded") && + rjsonp.test( data ); + + // Handle iff the expected data type is "jsonp" or we have a parameter to set + if ( s.dataTypes[ 0 ] === "jsonp" || replaceInUrl || replaceInData ) { + + // Get callback name, remembering preexisting value associated with it + callbackName = s.jsonpCallback = jQuery.isFunction( s.jsonpCallback ) ? + s.jsonpCallback() : + s.jsonpCallback; + overwritten = window[ callbackName ]; + + // Insert callback into url or form data + if ( replaceInUrl ) { + s.url = url.replace( rjsonp, "$1" + callbackName ); + } else if ( replaceInData ) { + s.data = data.replace( rjsonp, "$1" + callbackName ); + } else if ( hasCallback ) { + s.url += ( rquestion.test( url ) ? "&" : "?" ) + s.jsonp + "=" + callbackName; + } + + // Use data converter to retrieve json after script execution + s.converters["script json"] = function() { + if ( !responseContainer ) { + jQuery.error( callbackName + " was not called" ); + } + return responseContainer[ 0 ]; + }; + + // force json dataType + s.dataTypes[ 0 ] = "json"; + + // Install callback + window[ callbackName ] = function() { + responseContainer = arguments; + }; + + // Clean-up function (fires after converters) + jqXHR.always(function() { + // Restore preexisting value + window[ callbackName ] = overwritten; + + // Save back as free + if ( s[ callbackName ] ) { + // make sure that re-using the options doesn't screw things around + s.jsonpCallback = originalSettings.jsonpCallback; + + // save the callback name for future use + oldCallbacks.push( callbackName ); + } + + // Call if it was a function and we have a response + if ( responseContainer && jQuery.isFunction( overwritten ) ) { + overwritten( responseContainer[ 0 ] ); + } + + responseContainer = overwritten = undefined; + }); + + // Delegate to script + return "script"; + } +}); +// Install script dataType +jQuery.ajaxSetup({ + accepts: { + script: "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript" + }, + contents: { + script: /javascript|ecmascript/ + }, + converters: { + "text script": function( text ) { + jQuery.globalEval( text ); + return text; + } + } +}); + +// Handle cache's special case and global +jQuery.ajaxPrefilter( "script", function( s ) { + if ( s.cache === undefined ) { + s.cache = false; + } + if ( s.crossDomain ) { + s.type = "GET"; + s.global = false; + } +}); + +// Bind script tag hack transport +jQuery.ajaxTransport( "script", function(s) { + + // This transport only deals with cross domain requests + if ( s.crossDomain ) { + + var script, + head = document.head || document.getElementsByTagName( "head" )[0] || document.documentElement; + + return { + + send: function( _, callback ) { + + script = document.createElement( "script" ); + + script.async = "async"; + + if ( s.scriptCharset ) { + script.charset = s.scriptCharset; + } + + script.src = s.url; + + // Attach handlers for all browsers + script.onload = script.onreadystatechange = function( _, isAbort ) { + + if ( isAbort || !script.readyState || /loaded|complete/.test( script.readyState ) ) { + + // Handle memory leak in IE + script.onload = script.onreadystatechange = null; + + // Remove the script + if ( head && script.parentNode ) { + head.removeChild( script ); + } + + // Dereference the script + script = undefined; + + // Callback if not abort + if ( !isAbort ) { + callback( 200, "success" ); + } + } + }; + // Use insertBefore instead of appendChild to circumvent an IE6 bug. + // This arises when a base node is used (#2709 and #4378). + head.insertBefore( script, head.firstChild ); + }, + + abort: function() { + if ( script ) { + script.onload( 0, 1 ); + } + } + }; + } +}); +var xhrCallbacks, + // #5280: Internet Explorer will keep connections alive if we don't abort on unload + xhrOnUnloadAbort = window.ActiveXObject ? function() { + // Abort all pending requests + for ( var key in xhrCallbacks ) { + xhrCallbacks[ key ]( 0, 1 ); + } + } : false, + xhrId = 0; + +// Functions to create xhrs +function createStandardXHR() { + try { + return new window.XMLHttpRequest(); + } catch( e ) {} +} + +function createActiveXHR() { + try { + return new window.ActiveXObject( "Microsoft.XMLHTTP" ); + } catch( e ) {} +} + +// Create the request object +// (This is still attached to ajaxSettings for backward compatibility) +jQuery.ajaxSettings.xhr = window.ActiveXObject ? + /* Microsoft failed to properly + * implement the XMLHttpRequest in IE7 (can't request local files), + * so we use the ActiveXObject when it is available + * Additionally XMLHttpRequest can be disabled in IE7/IE8 so + * we need a fallback. + */ + function() { + return !this.isLocal && createStandardXHR() || createActiveXHR(); + } : + // For all other browsers, use the standard XMLHttpRequest object + createStandardXHR; + +// Determine support properties +(function( xhr ) { + jQuery.extend( jQuery.support, { + ajax: !!xhr, + cors: !!xhr && ( "withCredentials" in xhr ) + }); +})( jQuery.ajaxSettings.xhr() ); + +// Create transport if the browser can provide an xhr +if ( jQuery.support.ajax ) { + + jQuery.ajaxTransport(function( s ) { + // Cross domain only allowed if supported through XMLHttpRequest + if ( !s.crossDomain || jQuery.support.cors ) { + + var callback; + + return { + send: function( headers, complete ) { + + // Get a new xhr + var handle, i, + xhr = s.xhr(); + + // Open the socket + // Passing null username, generates a login popup on Opera (#2865) + if ( s.username ) { + xhr.open( s.type, s.url, s.async, s.username, s.password ); + } else { + xhr.open( s.type, s.url, s.async ); + } + + // Apply custom fields if provided + if ( s.xhrFields ) { + for ( i in s.xhrFields ) { + xhr[ i ] = s.xhrFields[ i ]; + } + } + + // Override mime type if needed + if ( s.mimeType && xhr.overrideMimeType ) { + xhr.overrideMimeType( s.mimeType ); + } + + // X-Requested-With header + // For cross-domain requests, seeing as conditions for a preflight are + // akin to a jigsaw puzzle, we simply never set it to be sure. + // (it can always be set on a per-request basis or even using ajaxSetup) + // For same-domain requests, won't change header if already provided. + if ( !s.crossDomain && !headers["X-Requested-With"] ) { + headers[ "X-Requested-With" ] = "XMLHttpRequest"; + } + + // Need an extra try/catch for cross domain requests in Firefox 3 + try { + for ( i in headers ) { + xhr.setRequestHeader( i, headers[ i ] ); + } + } catch( _ ) {} + + // Do send the request + // This may raise an exception which is actually + // handled in jQuery.ajax (so no try/catch here) + xhr.send( ( s.hasContent && s.data ) || null ); + + // Listener + callback = function( _, isAbort ) { + + var status, + statusText, + responseHeaders, + responses, + xml; + + // Firefox throws exceptions when accessing properties + // of an xhr when a network error occurred + // http://helpful.knobs-dials.com/index.php/Component_returned_failure_code:_0x80040111_(NS_ERROR_NOT_AVAILABLE) + try { + + // Was never called and is aborted or complete + if ( callback && ( isAbort || xhr.readyState === 4 ) ) { + + // Only called once + callback = undefined; + + // Do not keep as active anymore + if ( handle ) { + xhr.onreadystatechange = jQuery.noop; + if ( xhrOnUnloadAbort ) { + delete xhrCallbacks[ handle ]; + } + } + + // If it's an abort + if ( isAbort ) { + // Abort it manually if needed + if ( xhr.readyState !== 4 ) { + xhr.abort(); + } + } else { + status = xhr.status; + responseHeaders = xhr.getAllResponseHeaders(); + responses = {}; + xml = xhr.responseXML; + + // Construct response list + if ( xml && xml.documentElement /* #4958 */ ) { + responses.xml = xml; + } + + // When requesting binary data, IE6-9 will throw an exception + // on any attempt to access responseText (#11426) + try { + responses.text = xhr.responseText; + } catch( e ) { + } + + // Firefox throws an exception when accessing + // statusText for faulty cross-domain requests + try { + statusText = xhr.statusText; + } catch( e ) { + // We normalize with Webkit giving an empty statusText + statusText = ""; + } + + // Filter status for non standard behaviors + + // If the request is local and we have data: assume a success + // (success with no data won't get notified, that's the best we + // can do given current implementations) + if ( !status && s.isLocal && !s.crossDomain ) { + status = responses.text ? 200 : 404; + // IE - #1450: sometimes returns 1223 when it should be 204 + } else if ( status === 1223 ) { + status = 204; + } + } + } + } catch( firefoxAccessException ) { + if ( !isAbort ) { + complete( -1, firefoxAccessException ); + } + } + + // Call complete if needed + if ( responses ) { + complete( status, statusText, responses, responseHeaders ); + } + }; + + if ( !s.async ) { + // if we're in sync mode we fire the callback + callback(); + } else if ( xhr.readyState === 4 ) { + // (IE6 & IE7) if it's in cache and has been + // retrieved directly we need to fire the callback + setTimeout( callback, 0 ); + } else { + handle = ++xhrId; + if ( xhrOnUnloadAbort ) { + // Create the active xhrs callbacks list if needed + // and attach the unload handler + if ( !xhrCallbacks ) { + xhrCallbacks = {}; + jQuery( window ).unload( xhrOnUnloadAbort ); + } + // Add to list of active xhrs callbacks + xhrCallbacks[ handle ] = callback; + } + xhr.onreadystatechange = callback; + } + }, + + abort: function() { + if ( callback ) { + callback(0,1); + } + } + }; + } + }); +} +var fxNow, timerId, + rfxtypes = /^(?:toggle|show|hide)$/, + rfxnum = new RegExp( "^(?:([-+])=|)(" + core_pnum + ")([a-z%]*)$", "i" ), + rrun = /queueHooks$/, + animationPrefilters = [ defaultPrefilter ], + tweeners = { + "*": [function( prop, value ) { + var end, unit, + tween = this.createTween( prop, value ), + parts = rfxnum.exec( value ), + target = tween.cur(), + start = +target || 0, + scale = 1, + maxIterations = 20; + + if ( parts ) { + end = +parts[2]; + unit = parts[3] || ( jQuery.cssNumber[ prop ] ? "" : "px" ); + + // We need to compute starting value + if ( unit !== "px" && start ) { + // Iteratively approximate from a nonzero starting point + // Prefer the current property, because this process will be trivial if it uses the same units + // Fallback to end or a simple constant + start = jQuery.css( tween.elem, prop, true ) || end || 1; + + do { + // If previous iteration zeroed out, double until we get *something* + // Use a string for doubling factor so we don't accidentally see scale as unchanged below + scale = scale || ".5"; + + // Adjust and apply + start = start / scale; + jQuery.style( tween.elem, prop, start + unit ); + + // Update scale, tolerating zero or NaN from tween.cur() + // And breaking the loop if scale is unchanged or perfect, or if we've just had enough + } while ( scale !== (scale = tween.cur() / target) && scale !== 1 && --maxIterations ); + } + + tween.unit = unit; + tween.start = start; + // If a +=/-= token was provided, we're doing a relative animation + tween.end = parts[1] ? start + ( parts[1] + 1 ) * end : end; + } + return tween; + }] + }; + +// Animations created synchronously will run synchronously +function createFxNow() { + setTimeout(function() { + fxNow = undefined; + }, 0 ); + return ( fxNow = jQuery.now() ); +} + +function createTweens( animation, props ) { + jQuery.each( props, function( prop, value ) { + var collection = ( tweeners[ prop ] || [] ).concat( tweeners[ "*" ] ), + index = 0, + length = collection.length; + for ( ; index < length; index++ ) { + if ( collection[ index ].call( animation, prop, value ) ) { + + // we're done with this property + return; + } + } + }); +} + +function Animation( elem, properties, options ) { + var result, + index = 0, + tweenerIndex = 0, + length = animationPrefilters.length, + deferred = jQuery.Deferred().always( function() { + // don't match elem in the :animated selector + delete tick.elem; + }), + tick = function() { + var currentTime = fxNow || createFxNow(), + remaining = Math.max( 0, animation.startTime + animation.duration - currentTime ), + // archaic crash bug won't allow us to use 1 - ( 0.5 || 0 ) (#12497) + temp = remaining / animation.duration || 0, + percent = 1 - temp, + index = 0, + length = animation.tweens.length; + + for ( ; index < length ; index++ ) { + animation.tweens[ index ].run( percent ); + } + + deferred.notifyWith( elem, [ animation, percent, remaining ]); + + if ( percent < 1 && length ) { + return remaining; + } else { + deferred.resolveWith( elem, [ animation ] ); + return false; + } + }, + animation = deferred.promise({ + elem: elem, + props: jQuery.extend( {}, properties ), + opts: jQuery.extend( true, { specialEasing: {} }, options ), + originalProperties: properties, + originalOptions: options, + startTime: fxNow || createFxNow(), + duration: options.duration, + tweens: [], + createTween: function( prop, end, easing ) { + var tween = jQuery.Tween( elem, animation.opts, prop, end, + animation.opts.specialEasing[ prop ] || animation.opts.easing ); + animation.tweens.push( tween ); + return tween; + }, + stop: function( gotoEnd ) { + var index = 0, + // if we are going to the end, we want to run all the tweens + // otherwise we skip this part + length = gotoEnd ? animation.tweens.length : 0; + + for ( ; index < length ; index++ ) { + animation.tweens[ index ].run( 1 ); + } + + // resolve when we played the last frame + // otherwise, reject + if ( gotoEnd ) { + deferred.resolveWith( elem, [ animation, gotoEnd ] ); + } else { + deferred.rejectWith( elem, [ animation, gotoEnd ] ); + } + return this; + } + }), + props = animation.props; + + propFilter( props, animation.opts.specialEasing ); + + for ( ; index < length ; index++ ) { + result = animationPrefilters[ index ].call( animation, elem, props, animation.opts ); + if ( result ) { + return result; + } + } + + createTweens( animation, props ); + + if ( jQuery.isFunction( animation.opts.start ) ) { + animation.opts.start.call( elem, animation ); + } + + jQuery.fx.timer( + jQuery.extend( tick, { + anim: animation, + queue: animation.opts.queue, + elem: elem + }) + ); + + // attach callbacks from options + return animation.progress( animation.opts.progress ) + .done( animation.opts.done, animation.opts.complete ) + .fail( animation.opts.fail ) + .always( animation.opts.always ); +} + +function propFilter( props, specialEasing ) { + var index, name, easing, value, hooks; + + // camelCase, specialEasing and expand cssHook pass + for ( index in props ) { + name = jQuery.camelCase( index ); + easing = specialEasing[ name ]; + value = props[ index ]; + if ( jQuery.isArray( value ) ) { + easing = value[ 1 ]; + value = props[ index ] = value[ 0 ]; + } + + if ( index !== name ) { + props[ name ] = value; + delete props[ index ]; + } + + hooks = jQuery.cssHooks[ name ]; + if ( hooks && "expand" in hooks ) { + value = hooks.expand( value ); + delete props[ name ]; + + // not quite $.extend, this wont overwrite keys already present. + // also - reusing 'index' from above because we have the correct "name" + for ( index in value ) { + if ( !( index in props ) ) { + props[ index ] = value[ index ]; + specialEasing[ index ] = easing; + } + } + } else { + specialEasing[ name ] = easing; + } + } +} + +jQuery.Animation = jQuery.extend( Animation, { + + tweener: function( props, callback ) { + if ( jQuery.isFunction( props ) ) { + callback = props; + props = [ "*" ]; + } else { + props = props.split(" "); + } + + var prop, + index = 0, + length = props.length; + + for ( ; index < length ; index++ ) { + prop = props[ index ]; + tweeners[ prop ] = tweeners[ prop ] || []; + tweeners[ prop ].unshift( callback ); + } + }, + + prefilter: function( callback, prepend ) { + if ( prepend ) { + animationPrefilters.unshift( callback ); + } else { + animationPrefilters.push( callback ); + } + } +}); + +function defaultPrefilter( elem, props, opts ) { + var index, prop, value, length, dataShow, toggle, tween, hooks, oldfire, + anim = this, + style = elem.style, + orig = {}, + handled = [], + hidden = elem.nodeType && isHidden( elem ); + + // handle queue: false promises + if ( !opts.queue ) { + hooks = jQuery._queueHooks( elem, "fx" ); + if ( hooks.unqueued == null ) { + hooks.unqueued = 0; + oldfire = hooks.empty.fire; + hooks.empty.fire = function() { + if ( !hooks.unqueued ) { + oldfire(); + } + }; + } + hooks.unqueued++; + + anim.always(function() { + // doing this makes sure that the complete handler will be called + // before this completes + anim.always(function() { + hooks.unqueued--; + if ( !jQuery.queue( elem, "fx" ).length ) { + hooks.empty.fire(); + } + }); + }); + } + + // height/width overflow pass + if ( elem.nodeType === 1 && ( "height" in props || "width" in props ) ) { + // Make sure that nothing sneaks out + // Record all 3 overflow attributes because IE does not + // change the overflow attribute when overflowX and + // overflowY are set to the same value + opts.overflow = [ style.overflow, style.overflowX, style.overflowY ]; + + // Set display property to inline-block for height/width + // animations on inline elements that are having width/height animated + if ( jQuery.css( elem, "display" ) === "inline" && + jQuery.css( elem, "float" ) === "none" ) { + + // inline-level elements accept inline-block; + // block-level elements need to be inline with layout + if ( !jQuery.support.inlineBlockNeedsLayout || css_defaultDisplay( elem.nodeName ) === "inline" ) { + style.display = "inline-block"; + + } else { + style.zoom = 1; + } + } + } + + if ( opts.overflow ) { + style.overflow = "hidden"; + if ( !jQuery.support.shrinkWrapBlocks ) { + anim.done(function() { + style.overflow = opts.overflow[ 0 ]; + style.overflowX = opts.overflow[ 1 ]; + style.overflowY = opts.overflow[ 2 ]; + }); + } + } + + + // show/hide pass + for ( index in props ) { + value = props[ index ]; + if ( rfxtypes.exec( value ) ) { + delete props[ index ]; + toggle = toggle || value === "toggle"; + if ( value === ( hidden ? "hide" : "show" ) ) { + continue; + } + handled.push( index ); + } + } + + length = handled.length; + if ( length ) { + dataShow = jQuery._data( elem, "fxshow" ) || jQuery._data( elem, "fxshow", {} ); + if ( "hidden" in dataShow ) { + hidden = dataShow.hidden; + } + + // store state if its toggle - enables .stop().toggle() to "reverse" + if ( toggle ) { + dataShow.hidden = !hidden; + } + if ( hidden ) { + jQuery( elem ).show(); + } else { + anim.done(function() { + jQuery( elem ).hide(); + }); + } + anim.done(function() { + var prop; + jQuery.removeData( elem, "fxshow", true ); + for ( prop in orig ) { + jQuery.style( elem, prop, orig[ prop ] ); + } + }); + for ( index = 0 ; index < length ; index++ ) { + prop = handled[ index ]; + tween = anim.createTween( prop, hidden ? dataShow[ prop ] : 0 ); + orig[ prop ] = dataShow[ prop ] || jQuery.style( elem, prop ); + + if ( !( prop in dataShow ) ) { + dataShow[ prop ] = tween.start; + if ( hidden ) { + tween.end = tween.start; + tween.start = prop === "width" || prop === "height" ? 1 : 0; + } + } + } + } +} + +function Tween( elem, options, prop, end, easing ) { + return new Tween.prototype.init( elem, options, prop, end, easing ); +} +jQuery.Tween = Tween; + +Tween.prototype = { + constructor: Tween, + init: function( elem, options, prop, end, easing, unit ) { + this.elem = elem; + this.prop = prop; + this.easing = easing || "swing"; + this.options = options; + this.start = this.now = this.cur(); + this.end = end; + this.unit = unit || ( jQuery.cssNumber[ prop ] ? "" : "px" ); + }, + cur: function() { + var hooks = Tween.propHooks[ this.prop ]; + + return hooks && hooks.get ? + hooks.get( this ) : + Tween.propHooks._default.get( this ); + }, + run: function( percent ) { + var eased, + hooks = Tween.propHooks[ this.prop ]; + + if ( this.options.duration ) { + this.pos = eased = jQuery.easing[ this.easing ]( + percent, this.options.duration * percent, 0, 1, this.options.duration + ); + } else { + this.pos = eased = percent; + } + this.now = ( this.end - this.start ) * eased + this.start; + + if ( this.options.step ) { + this.options.step.call( this.elem, this.now, this ); + } + + if ( hooks && hooks.set ) { + hooks.set( this ); + } else { + Tween.propHooks._default.set( this ); + } + return this; + } +}; + +Tween.prototype.init.prototype = Tween.prototype; + +Tween.propHooks = { + _default: { + get: function( tween ) { + var result; + + if ( tween.elem[ tween.prop ] != null && + (!tween.elem.style || tween.elem.style[ tween.prop ] == null) ) { + return tween.elem[ tween.prop ]; + } + + // passing any value as a 4th parameter to .css will automatically + // attempt a parseFloat and fallback to a string if the parse fails + // so, simple values such as "10px" are parsed to Float. + // complex values such as "rotate(1rad)" are returned as is. + result = jQuery.css( tween.elem, tween.prop, false, "" ); + // Empty strings, null, undefined and "auto" are converted to 0. + return !result || result === "auto" ? 0 : result; + }, + set: function( tween ) { + // use step hook for back compat - use cssHook if its there - use .style if its + // available and use plain properties where available + if ( jQuery.fx.step[ tween.prop ] ) { + jQuery.fx.step[ tween.prop ]( tween ); + } else if ( tween.elem.style && ( tween.elem.style[ jQuery.cssProps[ tween.prop ] ] != null || jQuery.cssHooks[ tween.prop ] ) ) { + jQuery.style( tween.elem, tween.prop, tween.now + tween.unit ); + } else { + tween.elem[ tween.prop ] = tween.now; + } + } + } +}; + +// Remove in 2.0 - this supports IE8's panic based approach +// to setting things on disconnected nodes + +Tween.propHooks.scrollTop = Tween.propHooks.scrollLeft = { + set: function( tween ) { + if ( tween.elem.nodeType && tween.elem.parentNode ) { + tween.elem[ tween.prop ] = tween.now; + } + } +}; + +jQuery.each([ "toggle", "show", "hide" ], function( i, name ) { + var cssFn = jQuery.fn[ name ]; + jQuery.fn[ name ] = function( speed, easing, callback ) { + return speed == null || typeof speed === "boolean" || + // special check for .toggle( handler, handler, ... ) + ( !i && jQuery.isFunction( speed ) && jQuery.isFunction( easing ) ) ? + cssFn.apply( this, arguments ) : + this.animate( genFx( name, true ), speed, easing, callback ); + }; +}); + +jQuery.fn.extend({ + fadeTo: function( speed, to, easing, callback ) { + + // show any hidden elements after setting opacity to 0 + return this.filter( isHidden ).css( "opacity", 0 ).show() + + // animate to the value specified + .end().animate({ opacity: to }, speed, easing, callback ); + }, + animate: function( prop, speed, easing, callback ) { + var empty = jQuery.isEmptyObject( prop ), + optall = jQuery.speed( speed, easing, callback ), + doAnimation = function() { + // Operate on a copy of prop so per-property easing won't be lost + var anim = Animation( this, jQuery.extend( {}, prop ), optall ); + + // Empty animations resolve immediately + if ( empty ) { + anim.stop( true ); + } + }; + + return empty || optall.queue === false ? + this.each( doAnimation ) : + this.queue( optall.queue, doAnimation ); + }, + stop: function( type, clearQueue, gotoEnd ) { + var stopQueue = function( hooks ) { + var stop = hooks.stop; + delete hooks.stop; + stop( gotoEnd ); + }; + + if ( typeof type !== "string" ) { + gotoEnd = clearQueue; + clearQueue = type; + type = undefined; + } + if ( clearQueue && type !== false ) { + this.queue( type || "fx", [] ); + } + + return this.each(function() { + var dequeue = true, + index = type != null && type + "queueHooks", + timers = jQuery.timers, + data = jQuery._data( this ); + + if ( index ) { + if ( data[ index ] && data[ index ].stop ) { + stopQueue( data[ index ] ); + } + } else { + for ( index in data ) { + if ( data[ index ] && data[ index ].stop && rrun.test( index ) ) { + stopQueue( data[ index ] ); + } + } + } + + for ( index = timers.length; index--; ) { + if ( timers[ index ].elem === this && (type == null || timers[ index ].queue === type) ) { + timers[ index ].anim.stop( gotoEnd ); + dequeue = false; + timers.splice( index, 1 ); + } + } + + // start the next in the queue if the last step wasn't forced + // timers currently will call their complete callbacks, which will dequeue + // but only if they were gotoEnd + if ( dequeue || !gotoEnd ) { + jQuery.dequeue( this, type ); + } + }); + } +}); + +// Generate parameters to create a standard animation +function genFx( type, includeWidth ) { + var which, + attrs = { height: type }, + i = 0; + + // if we include width, step value is 1 to do all cssExpand values, + // if we don't include width, step value is 2 to skip over Left and Right + includeWidth = includeWidth? 1 : 0; + for( ; i < 4 ; i += 2 - includeWidth ) { + which = cssExpand[ i ]; + attrs[ "margin" + which ] = attrs[ "padding" + which ] = type; + } + + if ( includeWidth ) { + attrs.opacity = attrs.width = type; + } + + return attrs; +} + +// Generate shortcuts for custom animations +jQuery.each({ + slideDown: genFx("show"), + slideUp: genFx("hide"), + slideToggle: genFx("toggle"), + fadeIn: { opacity: "show" }, + fadeOut: { opacity: "hide" }, + fadeToggle: { opacity: "toggle" } +}, function( name, props ) { + jQuery.fn[ name ] = function( speed, easing, callback ) { + return this.animate( props, speed, easing, callback ); + }; +}); + +jQuery.speed = function( speed, easing, fn ) { + var opt = speed && typeof speed === "object" ? jQuery.extend( {}, speed ) : { + complete: fn || !fn && easing || + jQuery.isFunction( speed ) && speed, + duration: speed, + easing: fn && easing || easing && !jQuery.isFunction( easing ) && easing + }; + + opt.duration = jQuery.fx.off ? 0 : typeof opt.duration === "number" ? opt.duration : + opt.duration in jQuery.fx.speeds ? jQuery.fx.speeds[ opt.duration ] : jQuery.fx.speeds._default; + + // normalize opt.queue - true/undefined/null -> "fx" + if ( opt.queue == null || opt.queue === true ) { + opt.queue = "fx"; + } + + // Queueing + opt.old = opt.complete; + + opt.complete = function() { + if ( jQuery.isFunction( opt.old ) ) { + opt.old.call( this ); + } + + if ( opt.queue ) { + jQuery.dequeue( this, opt.queue ); + } + }; + + return opt; +}; + +jQuery.easing = { + linear: function( p ) { + return p; + }, + swing: function( p ) { + return 0.5 - Math.cos( p*Math.PI ) / 2; + } +}; + +jQuery.timers = []; +jQuery.fx = Tween.prototype.init; +jQuery.fx.tick = function() { + var timer, + timers = jQuery.timers, + i = 0; + + fxNow = jQuery.now(); + + for ( ; i < timers.length; i++ ) { + timer = timers[ i ]; + // Checks the timer has not already been removed + if ( !timer() && timers[ i ] === timer ) { + timers.splice( i--, 1 ); + } + } + + if ( !timers.length ) { + jQuery.fx.stop(); + } + fxNow = undefined; +}; + +jQuery.fx.timer = function( timer ) { + if ( timer() && jQuery.timers.push( timer ) && !timerId ) { + timerId = setInterval( jQuery.fx.tick, jQuery.fx.interval ); + } +}; + +jQuery.fx.interval = 13; + +jQuery.fx.stop = function() { + clearInterval( timerId ); + timerId = null; +}; + +jQuery.fx.speeds = { + slow: 600, + fast: 200, + // Default speed + _default: 400 +}; + +// Back Compat <1.8 extension point +jQuery.fx.step = {}; + +if ( jQuery.expr && jQuery.expr.filters ) { + jQuery.expr.filters.animated = function( elem ) { + return jQuery.grep(jQuery.timers, function( fn ) { + return elem === fn.elem; + }).length; + }; +} +var rroot = /^(?:body|html)$/i; + +jQuery.fn.offset = function( options ) { + if ( arguments.length ) { + return options === undefined ? + this : + this.each(function( i ) { + jQuery.offset.setOffset( this, options, i ); + }); + } + + var docElem, body, win, clientTop, clientLeft, scrollTop, scrollLeft, + box = { top: 0, left: 0 }, + elem = this[ 0 ], + doc = elem && elem.ownerDocument; + + if ( !doc ) { + return; + } + + if ( (body = doc.body) === elem ) { + return jQuery.offset.bodyOffset( elem ); + } + + docElem = doc.documentElement; + + // Make sure it's not a disconnected DOM node + if ( !jQuery.contains( docElem, elem ) ) { + return box; + } + + // If we don't have gBCR, just use 0,0 rather than error + // BlackBerry 5, iOS 3 (original iPhone) + if ( typeof elem.getBoundingClientRect !== "undefined" ) { + box = elem.getBoundingClientRect(); + } + win = getWindow( doc ); + clientTop = docElem.clientTop || body.clientTop || 0; + clientLeft = docElem.clientLeft || body.clientLeft || 0; + scrollTop = win.pageYOffset || docElem.scrollTop; + scrollLeft = win.pageXOffset || docElem.scrollLeft; + return { + top: box.top + scrollTop - clientTop, + left: box.left + scrollLeft - clientLeft + }; +}; + +jQuery.offset = { + + bodyOffset: function( body ) { + var top = body.offsetTop, + left = body.offsetLeft; + + if ( jQuery.support.doesNotIncludeMarginInBodyOffset ) { + top += parseFloat( jQuery.css(body, "marginTop") ) || 0; + left += parseFloat( jQuery.css(body, "marginLeft") ) || 0; + } + + return { top: top, left: left }; + }, + + setOffset: function( elem, options, i ) { + var position = jQuery.css( elem, "position" ); + + // set position first, in-case top/left are set even on static elem + if ( position === "static" ) { + elem.style.position = "relative"; + } + + var curElem = jQuery( elem ), + curOffset = curElem.offset(), + curCSSTop = jQuery.css( elem, "top" ), + curCSSLeft = jQuery.css( elem, "left" ), + calculatePosition = ( position === "absolute" || position === "fixed" ) && jQuery.inArray("auto", [curCSSTop, curCSSLeft]) > -1, + props = {}, curPosition = {}, curTop, curLeft; + + // need to be able to calculate position if either top or left is auto and position is either absolute or fixed + if ( calculatePosition ) { + curPosition = curElem.position(); + curTop = curPosition.top; + curLeft = curPosition.left; + } else { + curTop = parseFloat( curCSSTop ) || 0; + curLeft = parseFloat( curCSSLeft ) || 0; + } + + if ( jQuery.isFunction( options ) ) { + options = options.call( elem, i, curOffset ); + } + + if ( options.top != null ) { + props.top = ( options.top - curOffset.top ) + curTop; + } + if ( options.left != null ) { + props.left = ( options.left - curOffset.left ) + curLeft; + } + + if ( "using" in options ) { + options.using.call( elem, props ); + } else { + curElem.css( props ); + } + } +}; + + +jQuery.fn.extend({ + + position: function() { + if ( !this[0] ) { + return; + } + + var elem = this[0], + + // Get *real* offsetParent + offsetParent = this.offsetParent(), + + // Get correct offsets + offset = this.offset(), + parentOffset = rroot.test(offsetParent[0].nodeName) ? { top: 0, left: 0 } : offsetParent.offset(); + + // Subtract element margins + // note: when an element has margin: auto the offsetLeft and marginLeft + // are the same in Safari causing offset.left to incorrectly be 0 + offset.top -= parseFloat( jQuery.css(elem, "marginTop") ) || 0; + offset.left -= parseFloat( jQuery.css(elem, "marginLeft") ) || 0; + + // Add offsetParent borders + parentOffset.top += parseFloat( jQuery.css(offsetParent[0], "borderTopWidth") ) || 0; + parentOffset.left += parseFloat( jQuery.css(offsetParent[0], "borderLeftWidth") ) || 0; + + // Subtract the two offsets + return { + top: offset.top - parentOffset.top, + left: offset.left - parentOffset.left + }; + }, + + offsetParent: function() { + return this.map(function() { + var offsetParent = this.offsetParent || document.body; + while ( offsetParent && (!rroot.test(offsetParent.nodeName) && jQuery.css(offsetParent, "position") === "static") ) { + offsetParent = offsetParent.offsetParent; + } + return offsetParent || document.body; + }); + } +}); + + +// Create scrollLeft and scrollTop methods +jQuery.each( {scrollLeft: "pageXOffset", scrollTop: "pageYOffset"}, function( method, prop ) { + var top = /Y/.test( prop ); + + jQuery.fn[ method ] = function( val ) { + return jQuery.access( this, function( elem, method, val ) { + var win = getWindow( elem ); + + if ( val === undefined ) { + return win ? (prop in win) ? win[ prop ] : + win.document.documentElement[ method ] : + elem[ method ]; + } + + if ( win ) { + win.scrollTo( + !top ? val : jQuery( win ).scrollLeft(), + top ? val : jQuery( win ).scrollTop() + ); + + } else { + elem[ method ] = val; + } + }, method, val, arguments.length, null ); + }; +}); + +function getWindow( elem ) { + return jQuery.isWindow( elem ) ? + elem : + elem.nodeType === 9 ? + elem.defaultView || elem.parentWindow : + false; +} +// Create innerHeight, innerWidth, height, width, outerHeight and outerWidth methods +jQuery.each( { Height: "height", Width: "width" }, function( name, type ) { + jQuery.each( { padding: "inner" + name, content: type, "": "outer" + name }, function( defaultExtra, funcName ) { + // margin is only for outerHeight, outerWidth + jQuery.fn[ funcName ] = function( margin, value ) { + var chainable = arguments.length && ( defaultExtra || typeof margin !== "boolean" ), + extra = defaultExtra || ( margin === true || value === true ? "margin" : "border" ); + + return jQuery.access( this, function( elem, type, value ) { + var doc; + + if ( jQuery.isWindow( elem ) ) { + // As of 5/8/2012 this will yield incorrect results for Mobile Safari, but there + // isn't a whole lot we can do. See pull request at this URL for discussion: + // https://github.com/jquery/jquery/pull/764 + return elem.document.documentElement[ "client" + name ]; + } + + // Get document width or height + if ( elem.nodeType === 9 ) { + doc = elem.documentElement; + + // Either scroll[Width/Height] or offset[Width/Height] or client[Width/Height], whichever is greatest + // unfortunately, this causes bug #3838 in IE6/8 only, but there is currently no good, small way to fix it. + return Math.max( + elem.body[ "scroll" + name ], doc[ "scroll" + name ], + elem.body[ "offset" + name ], doc[ "offset" + name ], + doc[ "client" + name ] + ); + } + + return value === undefined ? + // Get width or height on the element, requesting but not forcing parseFloat + jQuery.css( elem, type, value, extra ) : + + // Set width or height on the element + jQuery.style( elem, type, value, extra ); + }, type, chainable ? margin : undefined, chainable, null ); + }; + }); +}); +// Expose jQuery to the global object +window.jQuery = window.$ = jQuery; + +// Expose jQuery as an AMD module, but only for AMD loaders that +// understand the issues with loading multiple versions of jQuery +// in a page that all might call define(). The loader will indicate +// they have special allowances for multiple jQuery versions by +// specifying define.amd.jQuery = true. Register as a named module, +// since jQuery can be concatenated with other files that may use define, +// but not use a proper concatenation script that understands anonymous +// AMD modules. A named AMD is safest and most robust way to register. +// Lowercase jquery is used because AMD module names are derived from +// file names, and jQuery is normally delivered in a lowercase file name. +// Do this after creating the global so that if an AMD module wants to call +// noConflict to hide this version of jQuery, it will work. +if ( typeof define === "function" && define.amd && define.amd.jQuery ) { + define( "jquery", [], function () { return jQuery; } ); +} + +})( window ); diff --git a/qa/workunits/erasure-code/plot.js b/qa/workunits/erasure-code/plot.js new file mode 100644 index 000000000..bd2bba5bb --- /dev/null +++ b/qa/workunits/erasure-code/plot.js @@ -0,0 +1,82 @@ +$(function() { + encode = []; + if (typeof encode_vandermonde_isa != 'undefined') { + encode.push({ + data: encode_vandermonde_isa, + label: "ISA, Vandermonde", + points: { show: true }, + lines: { show: true }, + }); + } + if (typeof encode_vandermonde_jerasure != 'undefined') { + encode.push({ + data: encode_vandermonde_jerasure, + label: "Jerasure Generic, Vandermonde", + points: { show: true }, + lines: { show: true }, + }); + } + if (typeof encode_cauchy_isa != 'undefined') { + encode.push({ + data: encode_cauchy_isa, + label: "ISA, Cauchy", + points: { show: true }, + lines: { show: true }, + }); + } + if (typeof encode_cauchy_jerasure != 'undefined') { + encode.push({ + data: encode_cauchy_jerasure, + label: "Jerasure, Cauchy", + points: { show: true }, + lines: { show: true }, + }); + } + $.plot("#encode", encode, { + xaxis: { + mode: "categories", + tickLength: 0 + }, + }); + + decode = []; + if (typeof decode_vandermonde_isa != 'undefined') { + decode.push({ + data: decode_vandermonde_isa, + label: "ISA, Vandermonde", + points: { show: true }, + lines: { show: true }, + }); + } + if (typeof decode_vandermonde_jerasure != 'undefined') { + decode.push({ + data: decode_vandermonde_jerasure, + label: "Jerasure Generic, Vandermonde", + points: { show: true }, + lines: { show: true }, + }); + } + if (typeof decode_cauchy_isa != 'undefined') { + decode.push({ + data: decode_cauchy_isa, + label: "ISA, Cauchy", + points: { show: true }, + lines: { show: true }, + }); + } + if (typeof decode_cauchy_jerasure != 'undefined') { + decode.push({ + data: decode_cauchy_jerasure, + label: "Jerasure, Cauchy", + points: { show: true }, + lines: { show: true }, + }); + } + $.plot("#decode", decode, { + xaxis: { + mode: "categories", + tickLength: 0 + }, + }); + +}); diff --git a/qa/workunits/false.sh b/qa/workunits/false.sh new file mode 100644 index 000000000..8a961b329 --- /dev/null +++ b/qa/workunits/false.sh @@ -0,0 +1,3 @@ +#!/bin/sh -ex + +false
\ No newline at end of file diff --git a/qa/workunits/fs/.gitignore b/qa/workunits/fs/.gitignore new file mode 100644 index 000000000..f7f7a0614 --- /dev/null +++ b/qa/workunits/fs/.gitignore @@ -0,0 +1 @@ +test_o_trunc diff --git a/qa/workunits/fs/Makefile b/qa/workunits/fs/Makefile new file mode 100644 index 000000000..c9934254d --- /dev/null +++ b/qa/workunits/fs/Makefile @@ -0,0 +1,11 @@ +CFLAGS = -Wall -Wextra -D_GNU_SOURCE + +TARGETS = test_o_trunc + +.c: + $(CC) $(CFLAGS) $@.c -o $@ + +all: $(TARGETS) + +clean: + rm $(TARGETS) diff --git a/qa/workunits/fs/cephfs_mirror_ha_gen.sh b/qa/workunits/fs/cephfs_mirror_ha_gen.sh new file mode 100755 index 000000000..35ee9d4c7 --- /dev/null +++ b/qa/workunits/fs/cephfs_mirror_ha_gen.sh @@ -0,0 +1,69 @@ +#!/bin/bash -ex +# +# cephfs_mirror_ha_gen.sh - generate workload to synchronize +# + +. $(dirname $0)/cephfs_mirror_helpers.sh + +cleanup() +{ + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + for j in `seq 1 $NR_SNAPSHOTS` + do + snap_name=$repo_name/.snap/snap_$j + if test -d $snap_name; then + rmdir $snap_name + fi + done + done + exit 1 +} +trap cleanup EXIT + +configure_peer() +{ + ceph mgr module enable mirroring + ceph fs snapshot mirror enable $PRIMARY_FS + ceph fs snapshot mirror peer_add $PRIMARY_FS client.mirror_remote@ceph $BACKUP_FS + + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + ceph fs snapshot mirror add $PRIMARY_FS "$MIRROR_SUBDIR/$repo_name" + done +} + +create_snaps() +{ + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + for j in `seq 1 $NR_SNAPSHOTS` + do + snap_name=$repo_name/.snap/snap_$j + r=$(( $RANDOM % 100 + 5 )) + arr=($repo_name "reset" "--hard" "HEAD~$r") + exec_git_cmd "${arr[@]}" + mkdir $snap_name + store_checksum $snap_name + done + done +} + +unset CEPH_CLI_TEST_DUP_COMMAND + +echo "running generator on prmary file system..." + +# setup git repos to be used as data set +setup_repos + +# turn on mirroring, add peers... +configure_peer + +# snapshots on primary +create_snaps + +# do not cleanup when exiting on success.. +trap - EXIT diff --git a/qa/workunits/fs/cephfs_mirror_ha_verify.sh b/qa/workunits/fs/cephfs_mirror_ha_verify.sh new file mode 100755 index 000000000..8d8b3859c --- /dev/null +++ b/qa/workunits/fs/cephfs_mirror_ha_verify.sh @@ -0,0 +1,40 @@ +#!/bin/bash -ex +# +# cephfs_mirror_ha_verify.sh - verify synchronized snapshots +# + +. $(dirname $0)/cephfs_mirror_helpers.sh + +echo "running verifier on secondary file system..." + +for i in `seq 1 $NR_DIRECTORIES` +do + repo_name="${REPO_PATH_PFX}_$i" + for j in `seq 1 $NR_SNAPSHOTS` + do + for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 64 128 128 + do + sleep $s + snap_name=$repo_name/.snap/snap_$j + if test -d $repo_name; then + echo "checking snapshot [$snap_name] in $repo_name" + if test -d $snap_name; then + echo "generating hash for $snap_name" + cksum='' + calc_checksum $snap_name cksum + ret=$(compare_checksum $cksum $snap_name) + if [ $ret -ne 0 ]; then + echo "checksum failed $snap_name ($cksum)" + return $ret + else + echo "checksum matched $snap_name ($cksum)" + break + fi + fi + fi + done + echo "couldn't complete verification for: $snap_name" + done +done + +echo "verify done!" diff --git a/qa/workunits/fs/cephfs_mirror_helpers.sh b/qa/workunits/fs/cephfs_mirror_helpers.sh new file mode 100644 index 000000000..69f1c6f3d --- /dev/null +++ b/qa/workunits/fs/cephfs_mirror_helpers.sh @@ -0,0 +1,66 @@ +PRIMARY_FS='dc' +BACKUP_FS='dc-backup' + +REPO=ceph-qa-suite +REPO_DIR=ceph_repo +REPO_PATH_PFX="$REPO_DIR/$REPO" + +NR_DIRECTORIES=4 +NR_SNAPSHOTS=4 +MIRROR_SUBDIR='/mirror' + +calc_checksum() +{ + local path=$1 + local -n ref=$2 + ref=`find -L $path -type f -exec md5sum {} + | awk '{ print $1 }' | md5sum | awk '{ print $1 }'` +} + +store_checksum() +{ + local path=$1 + local cksum='' #something invalid + local fhash=`echo -n $path | md5sum | awk '{ print $1 }'` + calc_checksum $path cksum + echo -n $cksum > "/tmp/primary-$fhash" +} + +compare_checksum() +{ + local ret=0 + local cksum=$1 + local path=$2 + local fhash=`echo -n $path | md5sum | awk '{ print $1 }'` + local cksum_ondisk=`cat /tmp/primary-$fhash` + if [ $cksum != $cksum_ondisk ]; then + echo "$cksum <> $cksum_ondisk" + ret=1 + fi + echo $ret +} + +exec_git_cmd() +{ + local arg=("$@") + local repo_name=${arg[0]} + local cmd=${arg[@]:1} + git --git-dir "$repo_name/.git" $cmd +} + +clone_repo() +{ + local repo_name=$1 + git clone --branch giant "http://github.com/ceph/$REPO" $repo_name +} + +setup_repos() +{ + mkdir "$REPO_DIR" + + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + mkdir $repo_name + clone_repo $repo_name + done +} diff --git a/qa/workunits/fs/damage/test-first-damage.sh b/qa/workunits/fs/damage/test-first-damage.sh new file mode 100755 index 000000000..57447b957 --- /dev/null +++ b/qa/workunits/fs/damage/test-first-damage.sh @@ -0,0 +1,194 @@ +#!/bin/bash + +set -ex + +FIRST_DAMAGE="first-damage.py" +FS=cephfs +METADATA_POOL=cephfs_meta +MOUNT=~/mnt/mnt.0 +PYTHON=python3 + +function usage { + printf '%s: [--fs=<fs_name>] [--metadata-pool=<pool>] [--first-damage=</path/to/first-damage.py>]\n' + exit 1 +} + + +function create { + ceph config set mds mds_bal_fragment_dirs 0 + mkdir dir + DIR_INODE=$(stat -c '%i' dir) + touch dir/a + touch dir/"a space" + touch -- $(printf 'dir/\xff') + mkdir dir/.snap/1 + mkdir dir/.snap/2 + # two snaps + rm dir/a + mkdir dir/.snap/3 + # not present in HEAD + touch dir/a + mkdir dir/.snap/4 + # one snap + rm dir/a + touch dir/a + mkdir dir/.snap/5 + # unlink then create + rm dir/a + touch dir/a + # unlink then create, HEAD not snapped + ls dir/.snap/*/ + mkdir big + BIG_DIR_INODE=$(stat -c '%i' big) + for i in `seq 1 15000`; do + touch $(printf 'big/%08d' $i) + done +} + +function flush { + ceph tell mds."$FS":0 flush journal +} + +function damage { + local IS=$(printf '%llx.%08llx' "$DIR_INODE" 0) + local LS=$(ceph tell mds."$FS":0 dump snaps | jq .last_created) + + local T=$(mktemp -p /tmp) + + # nuke snap 1 version of "a" + rados --pool="$METADATA_POOL" getomapval "$IS" a_$(printf %x $((LS-4))) "$T" + printf '\xff\xff\xff\xf0' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat + rados --pool="$METADATA_POOL" setomapval "$IS" a_$(printf %x $((LS-4))) --input-file="$T" + + # nuke snap 4 version of "a" + rados --pool="$METADATA_POOL" getomapval "$IS" a_$(printf %x $((LS-1))) "$T" + printf '\xff\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat + rados --pool="$METADATA_POOL" setomapval "$IS" a_$(printf %x $((LS-1))) --input-file="$T" + + # screw up HEAD + rados --pool="$METADATA_POOL" getomapval "$IS" a_head "$T" + printf '\xfe\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat + rados --pool="$METADATA_POOL" setomapval "$IS" a_head --input-file="$T" + + # screw up HEAD on what dentry in big + IS=$(printf '%llx.%08llx' "$BIG_DIR_INODE" 0) + rados --pool="$METADATA_POOL" getomapval "$IS" 00009999_head "$T" + printf '\xfe\xff\xff\xff' | dd of="$T" count=4 bs=1 conv=notrunc,nocreat + rados --pool="$METADATA_POOL" setomapval "$IS" 00009999_head --input-file="$T" + + rm -f "$T" +} + +function recover { + flush + ceph fs fail "$FS" + sleep 5 + cephfs-journal-tool --rank="$FS":0 event recover_dentries summary + cephfs-journal-tool --rank="$FS":0 journal reset + "$PYTHON" $FIRST_DAMAGE --debug /tmp/debug1 --memo /tmp/memo1 "$METADATA_POOL" + "$PYTHON" $FIRST_DAMAGE --debug /tmp/debug2 --memo /tmp/memo2 --repair-nosnap "$METADATA_POOL" + "$PYTHON" $FIRST_DAMAGE --debug /tmp/debug3 --memo /tmp/memo3 --remove "$METADATA_POOL" + ceph fs set "$FS" joinable true +} + +function check { + stat dir || exit 1 + stat dir/a || exit 1 + for i in `seq 1 5`; do + stat dir/.snap/$i || exit 2 + done + stat dir/.snap/2/a || exit 3 + stat dir/.snap/5/a || exit 4 + if stat dir/.snap/1/a; then + echo should be gone + exit 5 + fi + if stat dir/.snap/3/a; then + echo should not ever exist + exit 6 + fi + if stat dir/.snap/4/a; then + echo should be gone + exit 7 + fi +} + +function cleanup { + rmdir dir/.snap/* + find dir + rm -rf dir +} + +function mount { + sudo --preserve-env=CEPH_CONF bin/mount.ceph :/ "$MOUNT" -o name=admin,noshare + df -h "$MOUNT" +} + +function main { + eval set -- $(getopt --name "$0" --options '' --longoptions 'help,fs:,metadata-pool:,first-damage:,mount:,python:' -- "$@") + + while [ "$#" -gt 0 ]; do + echo "$*" + echo "$1" + case "$1" in + -h|--help) + usage + ;; + --fs) + FS="$2" + shift 2 + ;; + --metadata-pool) + METADATA_POOL="$2" + shift 2 + ;; + --mount) + MOUNT="$2" + shift 2 + ;; + --first-damage) + FIRST_DAMAGE="$2" + shift 2 + ;; + --python) + PYTHON="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + usage + ;; + esac + done + + mount + + pushd "$MOUNT" + create + popd + + sudo umount -f "$MOUNT" + + # flush dentries/inodes to omap + flush + + damage + + recover + + sleep 5 # for mds to join + + mount + + pushd "$MOUNT" + check + cleanup + popd + + sudo umount -f "$MOUNT" +} + +main "$@" diff --git a/qa/workunits/fs/fscrypt.sh b/qa/workunits/fs/fscrypt.sh new file mode 100755 index 000000000..ca856a62e --- /dev/null +++ b/qa/workunits/fs/fscrypt.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +set -xe + +mydir=`dirname $0` + +if [ $# -ne 2 ] +then + echo "2 parameters are required!\n" + echo "Usage:" + echo " fscrypt.sh <type> <testdir>" + echo " type: should be any of 'none', 'unlocked' or 'locked'" + echo " testdir: the test direcotry name" + exit 1 +fi + +fscrypt=$1 +testcase=$2 +testdir=fscrypt_test_${fscrypt}_${testcase} +mkdir $testdir + +XFSPROGS_DIR='xfprogs-dev-dir' +XFSTESTS_DIR='xfstest-dev-dir' +export XFS_IO_PROG="$(type -P xfs_io)" + +# Setup the xfstests env +setup_xfstests_env() +{ + git clone https://git.ceph.com/xfstests-dev.git $XFSTESTS_DIR --depth 1 + pushd $XFSTESTS_DIR + . common/encrypt + popd +} + +install_deps() +{ + local system_value=$(sudo lsb_release -is | awk '{print tolower($0)}') + case $system_value in + "centos" | "centosstream" | "fedora") + sudo yum install -y inih-devel userspace-rcu-devel \ + libblkid-devel gettext libedit-devel \ + libattr-devel device-mapper-devel libicu-devel + ;; + "ubuntu" | "debian") + sudo apt-get install -y libinih-dev liburcu-dev \ + libblkid-dev gettext libedit-dev libattr1-dev \ + libdevmapper-dev libicu-dev pkg-config + ;; + *) + echo "Unsupported distro $system_value" + exit 1 + ;; + esac +} + +# Install xfsprogs-dev from source to support "add_enckey" for xfs_io +install_xfsprogs() +{ + local install_xfsprogs=0 + + xfs_io -c "help add_enckey" | grep -q 'not found' && install_xfsprogs=1 + + if [ $install_xfsprogs -eq 1 ]; then + install_deps + + git clone https://git.ceph.com/xfsprogs-dev.git $XFSPROGS_DIR --depth 1 + pushd $XFSPROGS_DIR + make + sudo make install + popd + fi +} + +clean_up() +{ + rm -rf $XFSPROGS_DIR + rm -rf $XFSTESTS_DIR + rm -rf $testdir +} + +# For now will test the V2 encryption policy only as the +# V1 encryption policy is deprecated + +install_xfsprogs +setup_xfstests_env + +# Generate a fixed keying identifier +raw_key=$(_generate_raw_encryption_key) +keyid=$(_add_enckey $testdir "$raw_key" | awk '{print $NF}') + +case ${fscrypt} in + "none") + # do nothing for the test directory and will test it + # as one non-encrypted directory. + pushd $testdir + ${mydir}/../suites/${testcase}.sh + popd + clean_up + ;; + "unlocked") + # set encrypt policy with the key provided and then + # the test directory will be encrypted & unlocked + _set_encpolicy $testdir $keyid + pushd $testdir + ${mydir}/../suites/${testcase}.sh + popd + clean_up + ;; + "locked") + # remove the key, then the test directory will be locked + # and any modification will be denied by requiring the key + _rm_enckey $testdir $keyid + clean_up + ;; + *) + clean_up + echo "Unknown parameter $1" + exit 1 +esac diff --git a/qa/workunits/fs/full/subvolume_clone.sh b/qa/workunits/fs/full/subvolume_clone.sh new file mode 100755 index 000000000..a11131215 --- /dev/null +++ b/qa/workunits/fs/full/subvolume_clone.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -ex + +# This testcase tests the 'ceph fs subvolume snapshot clone' when the osd is full. +# The clone fails with 'MetadataMgrException: -28 (error in write)' and +# truncates the config file of corresponding subvolume while updating the config file. +# Hence the subsequent subvolume commands on the clone fails with +# 'MetadataMgrException: -2 (section 'GLOBAL' does not exist)' traceback. + +# The osd is of the size 1GB. The full-ratios are set so that osd is treated full +# at around 600MB. The subvolume is created and 100MB is written. +# The subvolume is snapshotted and cloned ten times. Since the clone delay is set to 15 seconds, +# all the clones reach pending state for sure. Among ten clones, only few succeed and rest fails +# with ENOSPACE. + +# At this stage, ".meta" config file of the failed clones are checked if it's truncated. +# and clone status command is checked for traceback. + +# Note that the failed clones would be in retry loop and it's state would be 'pending' or 'in-progress'. +# It's state is not updated to 'failed' as the config update to gets ENOSPACE too. + +set -e +ignore_failure() { + if "$@"; then return 0; else return 0; fi +} + +expect_failure() { + if "$@"; then return 1; else return 0; fi +} + +NUM_CLONES=10 + +ceph fs subvolume create cephfs sub_0 +subvol_path_0=$(ceph fs subvolume getpath cephfs sub_0 2>/dev/null) + +# For debugging +echo "Before ratios are set" +df $CEPH_MNT +ceph osd df + +ceph osd set-full-ratio 0.6 +ceph osd set-nearfull-ratio 0.50 +ceph osd set-backfillfull-ratio 0.55 + +# For debugging +echo "After ratios are set" +df -h +ceph osd df + +for i in {1..100};do sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path_0/1MB_file-$i status=progress bs=1M count=1 conv=fdatasync;done + +# For debugging +echo "After subvolumes are written" +df -h $CEPH_MNT +ceph osd df + +# snapshot +ceph fs subvolume snapshot create cephfs sub_0 snap_0 + +# Set clone snapshot delay +ceph config set mgr mgr/volumes/snapshot_clone_delay 15 + +# Schedule few clones, some would fail with no space +for i in $(eval echo {1..$NUM_CLONES});do ceph fs subvolume snapshot clone cephfs sub_0 snap_0 clone_$i;done + +# Wait for osd is full +timeout=90 +while [ $timeout -gt 0 ] +do + health=$(ceph health detail) + [[ $health = *"OSD_FULL"* ]] && echo "OSD is full" && break + echo "Wating for osd to be full: $timeout" + sleep 1 + let "timeout-=1" +done + +# For debugging +echo "After osd is full" +df -h $CEPH_MNT +ceph osd df + +# Check clone status, this should not crash +for i in $(eval echo {1..$NUM_CLONES}) +do + ignore_failure ceph fs clone status cephfs clone_$i >/tmp/out_${PID}_file 2>/tmp/error_${PID}_file + cat /tmp/error_${PID}_file + if grep "complete" /tmp/out_${PID}_file; then + echo "The clone_$i is completed" + else + #in-progress/pending clones, No traceback should be found in stderr + echo clone_$i in PENDING/IN-PROGRESS + expect_failure sudo grep "Traceback" /tmp/error_${PID}_file + #config file should not be truncated and GLOBAL section should be found + sudo grep "GLOBAL" $CEPH_MNT/volumes/_nogroup/clone_$i/.meta + fi +done + +# Hard cleanup +ignore_failure sudo rm -rf $CEPH_MNT/_index/clone/* +ignore_failure sudo rm -rf $CEPH_MNT/volumes/_nogroup/clone_* +ignore_failure sudo rmdir $CEPH_MNT/volumes/_nogroup/sub_0/.snap/snap_0 +ignore_failure sudo rm -rf $CEPH_MNT/volumes/_nogroup/sub_0 + +#Set the ratios back for other full tests to run +ceph osd set-full-ratio 0.95 +ceph osd set-nearfull-ratio 0.95 +ceph osd set-backfillfull-ratio 0.95 + +#After test +echo "After test" +df -h $CEPH_MNT +ceph osd df + +echo OK diff --git a/qa/workunits/fs/full/subvolume_rm.sh b/qa/workunits/fs/full/subvolume_rm.sh new file mode 100755 index 000000000..a464e30f5 --- /dev/null +++ b/qa/workunits/fs/full/subvolume_rm.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -ex + +# This testcase tests the scenario of the 'ceph fs subvolume rm' mgr command +# when the osd is full. The command used to hang. The osd is of the size 1GB. +# The subvolume is created and 500MB file is written. The full-ratios are +# set below 500MB such that the osd is treated as full. Now the subvolume is +# is removed. This should be successful with the introduction of FULL +# capabilities which the mgr holds. + +set -e +expect_failure() { + if "$@"; then return 1; else return 0; fi +} + +ceph fs subvolume create cephfs sub_0 +subvol_path=$(ceph fs subvolume getpath cephfs sub_0 2>/dev/null) + +#For debugging +echo "Before write" +df -h +ceph osd df + +sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path/500MB_file-1 status=progress bs=1M count=500 + +ceph osd set-full-ratio 0.2 +ceph osd set-nearfull-ratio 0.16 +ceph osd set-backfillfull-ratio 0.18 + +timeout=30 +while [ $timeout -gt 0 ] +do + health=$(ceph health detail) + [[ $health = *"OSD_FULL"* ]] && echo "OSD is full" && break + echo "Wating for osd to be full: $timeout" + sleep 1 + let "timeout-=1" +done + +#For debugging +echo "After ratio set" +df -h +ceph osd df + +#Delete subvolume +ceph fs subvolume rm cephfs sub_0 + +#Validate subvolume is deleted +expect_failure ceph fs subvolume info cephfs sub_0 + +#Wait for subvolume to delete data +trashdir=$CEPH_MNT/volumes/_deleting +timeout=30 +while [ $timeout -gt 0 ] +do + [ -z "$(sudo ls -A $trashdir)" ] && echo "Trash directory $trashdir is empty" && break + echo "Wating for trash dir to be empty: $timeout" + sleep 1 + let "timeout-=1" +done + +#Set the ratios back for other full tests to run +ceph osd set-full-ratio 0.95 +ceph osd set-nearfull-ratio 0.95 +ceph osd set-backfillfull-ratio 0.95 + +#After test +echo "After test" +df -h +ceph osd df + +echo OK diff --git a/qa/workunits/fs/full/subvolume_snapshot_rm.sh b/qa/workunits/fs/full/subvolume_snapshot_rm.sh new file mode 100755 index 000000000..f6d0add9f --- /dev/null +++ b/qa/workunits/fs/full/subvolume_snapshot_rm.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -ex + +# This testcase tests the 'ceph fs subvolume snapshot rm' when the osd is full. +# The snapshot rm fails with 'MetadataMgrException: -28 (error in write)' and +# truncates the config file of corresponding subvolume. Hence the subsequent +# snapshot rm of the same snapshot fails with 'MetadataMgrException: -2 (section 'GLOBAL' does not exist)' +# traceback. + +# The osd is of the size 1GB. The subvolume is created and 800MB file is written. +# Then full-ratios are set below 500MB such that the osd is treated as full. +# The subvolume snapshot is taken which succeeds as no extra space is required +# for snapshot. Now, the removal of the snapshot fails with ENOSPACE as it +# fails to remove the snapshot metadata set. The snapshot removal fails +# but should not traceback and truncate the config file. + +set -e +expect_failure() { + if "$@"; then return 1; else return 0; fi +} + +ignore_failure() { + if "$@"; then return 0; else return 0; fi +} + +ceph fs subvolume create cephfs sub_0 +subvol_path=$(ceph fs subvolume getpath cephfs sub_0 2>/dev/null) + +#For debugging +echo "Before write" +df $CEPH_MNT +ceph osd df + +# Write 800MB file and set full ratio to around 200MB +ignore_failure sudo dd if=/dev/urandom of=$CEPH_MNT$subvol_path/800MB_file-1 status=progress bs=1M count=800 conv=fdatasync + +ceph osd set-full-ratio 0.2 +ceph osd set-nearfull-ratio 0.16 +ceph osd set-backfillfull-ratio 0.18 + +timeout=30 +while [ $timeout -gt 0 ] +do + health=$(ceph health detail) + [[ $health = *"OSD_FULL"* ]] && echo "OSD is full" && break + echo "Wating for osd to be full: $timeout" + sleep 1 + let "timeout-=1" +done + +#Take snapshot +ceph fs subvolume snapshot create cephfs sub_0 snap_0 + +#Remove snapshot fails but should not throw traceback +expect_failure ceph fs subvolume snapshot rm cephfs sub_0 snap_0 2>/tmp/error_${PID}_file +cat /tmp/error_${PID}_file + +# No traceback should be found +expect_failure grep "Traceback" /tmp/error_${PID}_file + +# Validate config file is not truncated and GLOBAL section exists +sudo grep "GLOBAL" $CEPH_MNT/volumes/_nogroup/sub_0/.meta + +#For debugging +echo "After write" +df $CEPH_MNT +ceph osd df + +# Snapshot removal with force option should succeed +ceph fs subvolume snapshot rm cephfs sub_0 snap_0 --force + +#Cleanup from backend +ignore_failure sudo rm -f /tmp/error_${PID}_file +ignore_failure sudo rm -rf $CEPH_MNT/volumes/_nogroup/sub_0 + +#Set the ratios back for other full tests to run +ceph osd set-full-ratio 0.95 +ceph osd set-nearfull-ratio 0.95 +ceph osd set-backfillfull-ratio 0.95 + +#After test +echo "After test" +df -h $CEPH_MNT +ceph osd df + +echo OK diff --git a/qa/workunits/fs/maxentries/maxentries.sh b/qa/workunits/fs/maxentries/maxentries.sh new file mode 100755 index 000000000..d48fd956e --- /dev/null +++ b/qa/workunits/fs/maxentries/maxentries.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +set -ex + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +function make_files() +{ + set +x + temp_dir=`mktemp -d mkfile_test_XXXXXX` + for i in $(seq 1 $1) + do + echo -n | dd of="${temp_dir}/file_$i" conv=fsync || return 1 + sync "${temp_dir}" || return 1 + done + set -x + return 0 +} + +function make_dirs() +{ + set +x + temp_dir=`mktemp -d mkdir_test_XXXXXX` + for i in $(seq 1 $1) + do + mkdir -p ${temp_dir}/dir_${i} || return 1 + sync "${temp_dir}" || return 1 + done + set -x + return 0 +} + +function make_nodes() +{ + set +x + temp_dir=`mktemp -d mknod_test_XXXXXX` + for i in $(seq 1 $1) + do + mknod ${temp_dir}/fifo_${i} p || return 1 + sync "${temp_dir}" || return 1 + done + set -x + return 0 +} + +function rename_files() +{ + set +x + temp_dir=`mktemp -d rename_test_XXXXXX` + mkdir -p ${temp_dir}/rename + + for i in $(seq 1 $1) + do + touch ${temp_dir}/file_${i} || return 1 + + mv ${temp_dir}/file_${i} ${temp_dir}/rename/ || return 1 + sync "${temp_dir}" || return 1 + done + set -x + return 0 +} + +function make_symlinks() +{ + set +x + temp_dir=`mktemp -d symlink_test_XXXXXX` + mkdir -p ${temp_dir}/symlink + + touch ${temp_dir}/file + + for i in $(seq 1 $1) + do + ln -s ../file ${temp_dir}/symlink/sym_${i} || return 1 + sync "${temp_dir}" || return 1 + done + set -x + return 0 +} + +function make_links() +{ + set +x + temp_dir=`mktemp -d link_test_XXXXXX` + mkdir -p ${temp_dir}/link + + touch ${temp_dir}/file + + for i in $(seq 1 $1) + do + ln ${temp_dir}/file ${temp_dir}/link/link_${i} || return 1 + sync "${temp_dir}" || return 1 + done + set -x + return 0 +} + +function cleanup() +{ + rm -rf * +} + +test_dir="max_entries" +mkdir -p $test_dir +pushd $test_dir + +dir_max_entries=100 +ceph config set mds mds_dir_max_entries $dir_max_entries + +ok_dir_max_entries=$dir_max_entries +fail_dir_max_entries=$((dir_max_entries+1)) + +# make files test +make_files $ok_dir_max_entries +expect_false make_files $fail_dir_max_entries + +# make dirs test +make_dirs $ok_dir_max_entries +expect_false make_dirs $fail_dir_max_entries + +# make nodes test +make_nodes $ok_dir_max_entries +expect_false make_nodes $fail_dir_max_entries + +# rename files test +rename_files $ok_dir_max_entries +expect_false rename_files $fail_dir_max_entries + +# symlink files test +make_symlinks $ok_dir_max_entries +expect_false make_symlinks $fail_dir_max_entries + +# link files test +make_links $ok_dir_max_entries +expect_false make_links $fail_dir_max_entries + +# no limit (e.g., default value) +dir_max_entries=0 +ceph config set mds mds_dir_max_entries $dir_max_entries + +make_files 500 +make_dirs 500 +make_nodes 500 +rename_files 500 +make_symlinks 500 +make_links 500 + +cleanup + +popd # $test_dir + +echo OK diff --git a/qa/workunits/fs/misc/acl.sh b/qa/workunits/fs/misc/acl.sh new file mode 100755 index 000000000..198b05671 --- /dev/null +++ b/qa/workunits/fs/misc/acl.sh @@ -0,0 +1,50 @@ +#!/bin/sh -x + +set -e +mkdir -p testdir +cd testdir + +set +e +setfacl -d -m u:nobody:rw . +if test $? != 0; then + echo "Filesystem does not support ACL" + exit 0 +fi + +expect_failure() { + if "$@"; then return 1; else return 0; fi +} + +set -e +c=0 +while [ $c -lt 100 ] +do + c=`expr $c + 1` + # inherited ACL from parent directory's default ACL + mkdir d1 + c1=`getfacl d1 | grep -c "nobody:rw"` + echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null + c2=`getfacl d1 | grep -c "nobody:rw"` + rmdir d1 + if [ $c1 -ne 2 ] || [ $c2 -ne 2 ] + then + echo "ERROR: incorrect ACLs" + exit 1 + fi +done + +mkdir d1 + +# The ACL xattr only contains ACL header. ACL should be removed +# in this case. +setfattr -n system.posix_acl_access -v 0x02000000 d1 +setfattr -n system.posix_acl_default -v 0x02000000 . + +expect_failure getfattr -n system.posix_acl_access d1 +expect_failure getfattr -n system.posix_acl_default . + + +rmdir d1 +cd .. +rmdir testdir +echo OK diff --git a/qa/workunits/fs/misc/chmod.sh b/qa/workunits/fs/misc/chmod.sh new file mode 100755 index 000000000..de66776f1 --- /dev/null +++ b/qa/workunits/fs/misc/chmod.sh @@ -0,0 +1,60 @@ +#!/bin/sh -x + +set -e + +check_perms() { + + file=$1 + r=$(ls -la ${file}) + if test $? != 0; then + echo "ERROR: File listing/stat failed" + exit 1 + fi + + perms=$2 + if test "${perms}" != $(echo ${r} | awk '{print $1}') && \ + test "${perms}." != $(echo ${r} | awk '{print $1}') && \ + test "${perms}+" != $(echo ${r} | awk '{print $1}'); then + echo "ERROR: Permissions should be ${perms}" + exit 1 + fi +} + +file=test_chmod.$$ + +echo "foo" > ${file} +if test $? != 0; then + echo "ERROR: Failed to create file ${file}" + exit 1 +fi + +chmod 400 ${file} +if test $? != 0; then + echo "ERROR: Failed to change mode of ${file}" + exit 1 +fi + +check_perms ${file} "-r--------" + +set +e +echo "bar" >> ${file} +if test $? = 0; then + echo "ERROR: Write to read-only file should Fail" + exit 1 +fi + +set -e +chmod 600 ${file} +echo "bar" >> ${file} +if test $? != 0; then + echo "ERROR: Write to writeable file failed" + exit 1 +fi + +check_perms ${file} "-rw-------" + +echo "foo" >> ${file} +if test $? != 0; then + echo "ERROR: Failed to write to file" + exit 1 +fi diff --git a/qa/workunits/fs/misc/dac_override.sh b/qa/workunits/fs/misc/dac_override.sh new file mode 100755 index 000000000..dfb1a9091 --- /dev/null +++ b/qa/workunits/fs/misc/dac_override.sh @@ -0,0 +1,19 @@ +#!/bin/sh -x + +expect_failure() { + if "$@"; then return 1; else return 0; fi +} + +set -e + +mkdir -p testdir +file=test_chmod.$$ + +echo "foo" > testdir/${file} +sudo chmod 600 testdir + +# only root can read +expect_failure cat testdir/${file} + +# directory read/write DAC override for root should allow read +sudo cat testdir/${file} diff --git a/qa/workunits/fs/misc/direct_io.py b/qa/workunits/fs/misc/direct_io.py new file mode 100755 index 000000000..f7d59d95a --- /dev/null +++ b/qa/workunits/fs/misc/direct_io.py @@ -0,0 +1,42 @@ +#!/usr/bin/python3 + +import mmap +import os +import subprocess + +def main(): + path = "testfile" + fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_TRUNC | os.O_DIRECT, 0o644) + + ino = os.fstat(fd).st_ino + obj_name = "{ino:x}.00000000".format(ino=ino) + pool_name = os.getxattr(path, "ceph.file.layout.pool") + + buf = mmap.mmap(-1, 1) + buf.write(b'1') + os.write(fd, buf) + + proc = subprocess.Popen(['rados', '-p', pool_name, 'get', obj_name, 'tmpfile']) + proc.wait() + + with open('tmpfile', 'rb') as tmpf: + out = tmpf.read(1) + if out != b'1': + raise RuntimeError("data were not written to object store directly") + + with open('tmpfile', 'wb') as tmpf: + tmpf.write(b'2') + + proc = subprocess.Popen(['rados', '-p', pool_name, 'put', obj_name, 'tmpfile']) + proc.wait() + + os.lseek(fd, 0, os.SEEK_SET) + out = os.read(fd, 1) + if out != b'2': + raise RuntimeError("data were not directly read from object store") + + os.close(fd) + print('ok') + + +main() diff --git a/qa/workunits/fs/misc/dirfrag.sh b/qa/workunits/fs/misc/dirfrag.sh new file mode 100755 index 000000000..eea0ec3bc --- /dev/null +++ b/qa/workunits/fs/misc/dirfrag.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -e + +DEPTH=5 +COUNT=10000 + +kill_jobs() { + jobs -p | xargs kill +} +trap kill_jobs INT + +create_files() { + for i in `seq 1 $COUNT` + do + touch file$i + done +} + +delete_files() { + for i in `ls -f` + do + if [[ ${i}a = file*a ]] + then + rm -f $i + fi + done +} + +rm -rf testdir +mkdir testdir +cd testdir + +echo "creating folder hierarchy" +for i in `seq 1 $DEPTH`; do + mkdir dir$i + cd dir$i + create_files & +done +wait + +echo "created hierarchy, now cleaning up" + +for i in `seq 1 $DEPTH`; do + delete_files & + cd .. +done +wait + +echo "cleaned up hierarchy" +cd .. +rm -rf testdir diff --git a/qa/workunits/fs/misc/filelock_deadlock.py b/qa/workunits/fs/misc/filelock_deadlock.py new file mode 100755 index 000000000..398902f6c --- /dev/null +++ b/qa/workunits/fs/misc/filelock_deadlock.py @@ -0,0 +1,72 @@ +#!/usr/bin/python3 + +import errno +import fcntl +import os +import signal +import struct +import time + + +def handler(signum, frame): + pass + + +def lock_two(f1, f2): + lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 10, 0, 0) + fcntl.fcntl(f1, fcntl.F_SETLKW, lockdata) + time.sleep(10) + + # don't wait forever + signal.signal(signal.SIGALRM, handler) + signal.alarm(10) + exitcode = 0 + try: + fcntl.fcntl(f2, fcntl.F_SETLKW, lockdata) + except IOError as e: + if e.errno == errno.EDEADLK: + exitcode = 1 + elif e.errno == errno.EINTR: + exitcode = 2 + else: + exitcode = 3 + os._exit(exitcode) + + +def main(): + pid1 = os.fork() + if pid1 == 0: + f1 = open("testfile1", 'w') + f2 = open("testfile2", 'w') + lock_two(f1, f2) + + pid2 = os.fork() + if pid2 == 0: + f1 = open("testfile2", 'w') + f2 = open("testfile3", 'w') + lock_two(f1, f2) + + pid3 = os.fork() + if pid3 == 0: + f1 = open("testfile3", 'w') + f2 = open("testfile1", 'w') + lock_two(f1, f2) + + deadlk_count = 0 + i = 0 + while i < 3: + pid, status = os.wait() + exitcode = status >> 8 + if exitcode == 1: + deadlk_count += 1 + elif exitcode != 0: + raise RuntimeError("unexpect exit code of child") + i += 1 + + if deadlk_count != 1: + raise RuntimeError("unexpect count of EDEADLK") + + print('ok') + + +main() diff --git a/qa/workunits/fs/misc/filelock_interrupt.py b/qa/workunits/fs/misc/filelock_interrupt.py new file mode 100755 index 000000000..b261d74fb --- /dev/null +++ b/qa/workunits/fs/misc/filelock_interrupt.py @@ -0,0 +1,94 @@ +#!/usr/bin/python3 + +from contextlib import contextmanager +import errno +import fcntl +import signal +import struct + +@contextmanager +def timeout(seconds): + def timeout_handler(signum, frame): + raise InterruptedError + + orig_handler = signal.signal(signal.SIGALRM, timeout_handler) + try: + signal.alarm(seconds) + yield + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, orig_handler) + + +""" +introduced by Linux 3.15 +""" +setattr(fcntl, "F_OFD_GETLK", 36) +setattr(fcntl, "F_OFD_SETLK", 37) +setattr(fcntl, "F_OFD_SETLKW", 38) + + +def main(): + f1 = open("testfile", 'w') + f2 = open("testfile", 'w') + + fcntl.flock(f1, fcntl.LOCK_SH | fcntl.LOCK_NB) + + """ + is flock interruptible? + """ + with timeout(5): + try: + fcntl.flock(f2, fcntl.LOCK_EX) + except InterruptedError: + pass + else: + raise RuntimeError("expect flock to block") + + fcntl.flock(f1, fcntl.LOCK_UN) + + lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 10, 0, 0) + try: + fcntl.fcntl(f1, fcntl.F_OFD_SETLK, lockdata) + except IOError as e: + if e.errno != errno.EINVAL: + raise + else: + print('kernel does not support fcntl.F_OFD_SETLK') + return + + lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 10, 10, 0, 0) + fcntl.fcntl(f2, fcntl.F_OFD_SETLK, lockdata) + + """ + is posix lock interruptible? + """ + with timeout(5): + try: + lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0) + fcntl.fcntl(f2, fcntl.F_OFD_SETLKW, lockdata) + except InterruptedError: + pass + else: + raise RuntimeError("expect posix lock to block") + + """ + file handler 2 should still hold lock on 10~10 + """ + try: + lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 10, 10, 0, 0) + fcntl.fcntl(f1, fcntl.F_OFD_SETLK, lockdata) + except IOError as e: + if e.errno == errno.EAGAIN: + pass + else: + raise RuntimeError("expect file handler 2 to hold lock on 10~10") + + lockdata = struct.pack('hhllhh', fcntl.F_UNLCK, 0, 0, 0, 0, 0) + fcntl.fcntl(f1, fcntl.F_OFD_SETLK, lockdata) + fcntl.fcntl(f2, fcntl.F_OFD_SETLK, lockdata) + + print('ok') + + +main() diff --git a/qa/workunits/fs/misc/i_complete_vs_rename.sh b/qa/workunits/fs/misc/i_complete_vs_rename.sh new file mode 100755 index 000000000..a9b98271d --- /dev/null +++ b/qa/workunits/fs/misc/i_complete_vs_rename.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +set -e + +mkdir x +cd x +touch a +touch b +touch c +touch d +ls +chmod 777 . +stat e || true +touch f +touch g + +# over existing file +echo attempting rename over existing file... +touch ../xx +mv ../xx f +ls | grep f || false +echo rename over existing file is okay + +# over negative dentry +echo attempting rename over negative dentry... +touch ../xx +mv ../xx e +ls | grep e || false +echo rename over negative dentry is ok + +echo OK diff --git a/qa/workunits/fs/misc/layout_vxattrs.sh b/qa/workunits/fs/misc/layout_vxattrs.sh new file mode 100755 index 000000000..811336273 --- /dev/null +++ b/qa/workunits/fs/misc/layout_vxattrs.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +set -ex + +# detect data pool +datapool= +dir=. +while true ; do + echo $dir + datapool=$(getfattr -n ceph.dir.layout.pool $dir --only-values) && break + dir=$dir/.. +done + +# file +rm -f file file2 +touch file file2 + +getfattr -n ceph.file.layout file +getfattr -n ceph.file.layout file | grep -q object_size= +getfattr -n ceph.file.layout file | grep -q stripe_count= +getfattr -n ceph.file.layout file | grep -q stripe_unit= +getfattr -n ceph.file.layout file | grep -q pool= +getfattr -n ceph.file.layout.pool file +getfattr -n ceph.file.layout.pool_namespace file +getfattr -n ceph.file.layout.stripe_unit file +getfattr -n ceph.file.layout.stripe_count file +getfattr -n ceph.file.layout.object_size file + +getfattr -n ceph.file.layout.bogus file 2>&1 | grep -q 'No such attribute' +getfattr -n ceph.dir.layout file 2>&1 | grep -q 'No such attribute' + +setfattr -n ceph.file.layout.stripe_unit -v 1048576 file2 +setfattr -n ceph.file.layout.stripe_count -v 8 file2 +setfattr -n ceph.file.layout.object_size -v 10485760 file2 + +setfattr -n ceph.file.layout.pool -v $datapool file2 +getfattr -n ceph.file.layout.pool file2 | grep -q $datapool +setfattr -n ceph.file.layout.pool_namespace -v foons file2 +getfattr -n ceph.file.layout.pool_namespace file2 | grep -q foons +setfattr -x ceph.file.layout.pool_namespace file2 +getfattr -n ceph.file.layout.pool_namespace file2 | grep -q -v foons + +getfattr -n ceph.file.layout.stripe_unit file2 | grep -q 1048576 +getfattr -n ceph.file.layout.stripe_count file2 | grep -q 8 +getfattr -n ceph.file.layout.object_size file2 | grep -q 10485760 + +setfattr -n ceph.file.layout -v "stripe_unit=4194304 stripe_count=16 object_size=41943040 pool=$datapool pool_namespace=foons" file2 +getfattr -n ceph.file.layout.stripe_unit file2 | grep -q 4194304 +getfattr -n ceph.file.layout.stripe_count file2 | grep -q 16 +getfattr -n ceph.file.layout.object_size file2 | grep -q 41943040 +getfattr -n ceph.file.layout.pool file2 | grep -q $datapool +getfattr -n ceph.file.layout.pool_namespace file2 | grep -q foons + +setfattr -n ceph.file.layout -v "stripe_unit=1048576" file2 +getfattr -n ceph.file.layout.stripe_unit file2 | grep -q 1048576 +getfattr -n ceph.file.layout.stripe_count file2 | grep -q 16 +getfattr -n ceph.file.layout.object_size file2 | grep -q 41943040 +getfattr -n ceph.file.layout.pool file2 | grep -q $datapool +getfattr -n ceph.file.layout.pool_namespace file2 | grep -q foons + +setfattr -n ceph.file.layout -v "stripe_unit=2097152 stripe_count=4 object_size=2097152 pool=$datapool pool_namespace=barns" file2 +getfattr -n ceph.file.layout.stripe_unit file2 | grep -q 2097152 +getfattr -n ceph.file.layout.stripe_count file2 | grep -q 4 +getfattr -n ceph.file.layout.object_size file2 | grep -q 2097152 +getfattr -n ceph.file.layout.pool file2 | grep -q $datapool +getfattr -n ceph.file.layout.pool_namespace file2 | grep -q barns + +# dir +rm -f dir/file || true +rmdir dir || true +mkdir -p dir + +getfattr -d -m - dir | grep -q ceph.dir.layout && exit 1 || true +getfattr -d -m - dir | grep -q ceph.file.layout && exit 1 || true +getfattr -n ceph.dir.layout dir && exit 1 || true + +setfattr -n ceph.dir.layout.stripe_unit -v 1048576 dir +setfattr -n ceph.dir.layout.stripe_count -v 8 dir +setfattr -n ceph.dir.layout.object_size -v 10485760 dir +setfattr -n ceph.dir.layout.pool -v $datapool dir +setfattr -n ceph.dir.layout.pool_namespace -v dirns dir + +getfattr -n ceph.dir.layout dir +getfattr -n ceph.dir.layout dir | grep -q object_size=10485760 +getfattr -n ceph.dir.layout dir | grep -q stripe_count=8 +getfattr -n ceph.dir.layout dir | grep -q stripe_unit=1048576 +getfattr -n ceph.dir.layout dir | grep -q pool=$datapool +getfattr -n ceph.dir.layout dir | grep -q pool_namespace=dirns +getfattr -n ceph.dir.layout.pool dir | grep -q $datapool +getfattr -n ceph.dir.layout.stripe_unit dir | grep -q 1048576 +getfattr -n ceph.dir.layout.stripe_count dir | grep -q 8 +getfattr -n ceph.dir.layout.object_size dir | grep -q 10485760 +getfattr -n ceph.dir.layout.pool_namespace dir | grep -q dirns + + +setfattr -n ceph.file.layout -v "stripe_count=16" file2 +getfattr -n ceph.file.layout.stripe_count file2 | grep -q 16 +setfattr -n ceph.file.layout -v "object_size=10485760 stripe_count=8 stripe_unit=1048576 pool=$datapool pool_namespace=dirns" file2 +getfattr -n ceph.file.layout.stripe_count file2 | grep -q 8 + +touch dir/file +getfattr -n ceph.file.layout.pool dir/file | grep -q $datapool +getfattr -n ceph.file.layout.stripe_unit dir/file | grep -q 1048576 +getfattr -n ceph.file.layout.stripe_count dir/file | grep -q 8 +getfattr -n ceph.file.layout.object_size dir/file | grep -q 10485760 +getfattr -n ceph.file.layout.pool_namespace dir/file | grep -q dirns + +setfattr -x ceph.dir.layout.pool_namespace dir +getfattr -n ceph.dir.layout dir | grep -q -v pool_namespace=dirns + +setfattr -x ceph.dir.layout dir +getfattr -n ceph.dir.layout dir 2>&1 | grep -q 'No such attribute' + +echo OK + diff --git a/qa/workunits/fs/misc/mkpool_layout_vxattrs.sh b/qa/workunits/fs/misc/mkpool_layout_vxattrs.sh new file mode 100755 index 000000000..6b2fecbc0 --- /dev/null +++ b/qa/workunits/fs/misc/mkpool_layout_vxattrs.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e + +touch foo.$$ +ceph osd pool create foo.$$ 8 +ceph fs add_data_pool cephfs foo.$$ +setfattr -n ceph.file.layout.pool -v foo.$$ foo.$$ + +# cleanup +rm foo.$$ +ceph fs rm_data_pool cephfs foo.$$ +ceph osd pool rm foo.$$ foo.$$ --yes-i-really-really-mean-it + +echo OK diff --git a/qa/workunits/fs/misc/multiple_rsync.sh b/qa/workunits/fs/misc/multiple_rsync.sh new file mode 100755 index 000000000..4397c1e7f --- /dev/null +++ b/qa/workunits/fs/misc/multiple_rsync.sh @@ -0,0 +1,25 @@ +#!/bin/sh -ex + + +# Populate with some arbitrary files from the local system. Take +# a copy to protect against false fails from system updates during test. +export PAYLOAD=/tmp/multiple_rsync_payload.$$ +sudo cp -r /usr/lib/ $PAYLOAD + +set -e + +sudo rsync -av $PAYLOAD payload.1 +sudo rsync -av $PAYLOAD payload.2 + +# this shouldn't transfer any additional files +echo we should get 4 here if no additional files are transferred +sudo rsync -auv $PAYLOAD payload.1 | tee /tmp/$$ +hexdump -C /tmp/$$ +wc -l /tmp/$$ | grep 4 +sudo rsync -auv $PAYLOAD payload.2 | tee /tmp/$$ +hexdump -C /tmp/$$ +wc -l /tmp/$$ | grep 4 +echo OK + +rm /tmp/$$ +sudo rm -rf $PAYLOAD diff --git a/qa/workunits/fs/misc/rstats.sh b/qa/workunits/fs/misc/rstats.sh new file mode 100755 index 000000000..e6b3eddf2 --- /dev/null +++ b/qa/workunits/fs/misc/rstats.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +set -x + +timeout=30 +old_value="" +new_value="" + +wait_until_changed() { + name=$1 + wait=0 + while [ $wait -lt $timeout ]; do + new_value=`getfattr --only-value -n ceph.dir.$name .` + [ $new_value == $old_value ] || return 0 + sleep 1 + wait=$(($wait + 1)) + done + return 1 +} + +check_rctime() { + old_sec=$(echo $old_value | cut -d. -f1) + old_nsec=$(echo $old_value | cut -d. -f2) + new_sec=$(echo $new_value | cut -d. -f1) + new_nsec=$(echo $new_value | cut -d. -f2) + [ "$old_sec" -lt "$new_sec" ] && return 0 + [ "$old_sec" -gt "$new_sec" ] && return 1 + [ "$old_nsec" -lt "$new_nsec" ] && return 0 + return 1 +} + +# sync(3) does not make ceph-fuse flush dirty caps, because fuse kernel module +# does not notify ceph-fuse about it. Use fsync(3) instead. +fsync_path() { + cmd="import os; fd=os.open(\"$1\", os.O_RDONLY); os.fsync(fd); os.close(fd)" + python3 -c "$cmd" +} + +set -e + +mkdir -p rstats_testdir/d1/d2 +cd rstats_testdir + +# rfiles +old_value=`getfattr --only-value -n ceph.dir.rfiles .` +[ $old_value == 0 ] || false +touch d1/d2/f1 +wait_until_changed rfiles +[ $new_value == $(($old_value + 1)) ] || false + +# rsubdirs +old_value=`getfattr --only-value -n ceph.dir.rsubdirs .` +[ $old_value == 3 ] || false +mkdir d1/d2/d3 +wait_until_changed rsubdirs +[ $new_value == $(($old_value + 1)) ] || false + +# rbytes +old_value=`getfattr --only-value -n ceph.dir.rbytes .` +[ $old_value == 0 ] || false +echo hello > d1/d2/f2 +fsync_path d1/d2/f2 +wait_until_changed rbytes +[ $new_value == $(($old_value + 6)) ] || false + +#rctime +old_value=`getfattr --only-value -n ceph.dir.rctime .` +touch d1/d2/d3 # touch existing file +fsync_path d1/d2/d3 +wait_until_changed rctime +check_rctime + +old_value=`getfattr --only-value -n ceph.dir.rctime .` +touch d1/d2/f3 # create new file +wait_until_changed rctime +check_rctime + +cd .. +rm -rf rstats_testdir +echo OK diff --git a/qa/workunits/fs/misc/trivial_sync.sh b/qa/workunits/fs/misc/trivial_sync.sh new file mode 100755 index 000000000..7c8c4e2b4 --- /dev/null +++ b/qa/workunits/fs/misc/trivial_sync.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -e + +mkdir foo +echo foo > bar +sync diff --git a/qa/workunits/fs/misc/xattrs.sh b/qa/workunits/fs/misc/xattrs.sh new file mode 100755 index 000000000..fcd94d22c --- /dev/null +++ b/qa/workunits/fs/misc/xattrs.sh @@ -0,0 +1,14 @@ +#!/bin/sh -x + +set -e + +touch file + +setfattr -n user.foo -v foo file +setfattr -n user.bar -v bar file +setfattr -n user.empty file +getfattr -d file | grep foo +getfattr -d file | grep bar +getfattr -d file | grep empty + +echo OK. diff --git a/qa/workunits/fs/multiclient_sync_read_eof.py b/qa/workunits/fs/multiclient_sync_read_eof.py new file mode 100755 index 000000000..15ecbb825 --- /dev/null +++ b/qa/workunits/fs/multiclient_sync_read_eof.py @@ -0,0 +1,42 @@ +#!/usr/bin/python3 + +import argparse +import os + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('mnt1') + parser.add_argument('mnt2') + parser.add_argument('fn') + args = parser.parse_args() + + open(os.path.join(args.mnt1, args.fn), 'w') + f1 = open(os.path.join(args.mnt1, args.fn), 'r+') + f2 = open(os.path.join(args.mnt2, args.fn), 'r+') + + f1.write('foo') + f1.flush() + a = f2.read(3) + print('got "%s"' % a) + assert a == 'foo' + f2.write('bar') + f2.flush() + a = f1.read(3) + print('got "%s"' % a) + assert a == 'bar' + + ## test short reads + f1.write('short') + f1.flush() + a = f2.read(100) + print('got "%s"' % a) + assert a == 'short' + f2.write('longer') + f2.flush() + a = f1.read(1000) + print('got "%s"' % a) + assert a == 'longer' + + print('ok') + +main() diff --git a/qa/workunits/fs/norstats/kernel_untar_tar.sh b/qa/workunits/fs/norstats/kernel_untar_tar.sh new file mode 100755 index 000000000..6a175dcd9 --- /dev/null +++ b/qa/workunits/fs/norstats/kernel_untar_tar.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# check if there is file changed while being archived + +set -e + +KERNEL=linux-4.0.5 + +wget -q http://download.ceph.com/qa/$KERNEL.tar.xz + +mkdir untar_tar +cd untar_tar + +tar Jxvf ../$KERNEL.tar.xz $KERNEL/Documentation/ +tar cf doc.tar $KERNEL + +tar xf doc.tar +sync +tar c $KERNEL >/dev/null + +rm -rf $KERNEL + +tar xf doc.tar +sync +tar c $KERNEL >/dev/null + +echo Ok diff --git a/qa/workunits/fs/quota/quota.sh b/qa/workunits/fs/quota/quota.sh new file mode 100755 index 000000000..1315be6d8 --- /dev/null +++ b/qa/workunits/fs/quota/quota.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +set -ex + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +function write_file() +{ + set +x + for ((i=1;i<=$2;i++)) + do + dd if=/dev/zero of=$1 bs=1M count=1 conv=notrunc oflag=append 2>/dev/null >/dev/null + if [ $? != 0 ]; then + echo Try to write $(($i * 1048576)) + set -x + return 1 + fi + sleep 0.05 + done + set -x + return 0 +} + +mkdir quota-test +cd quota-test + +# bytes +setfattr . -n ceph.quota.max_bytes -v 100000000 # 100m +expect_false write_file big 1000 # 1g +expect_false write_file second 10 +setfattr . -n ceph.quota.max_bytes -v 0 +dd if=/dev/zero of=third bs=1M count=10 +dd if=/dev/zero of=big2 bs=1M count=100 + + +rm -rf * + +# files +setfattr . -n ceph.quota.max_files -v 5 +mkdir ok +touch ok/1 +touch ok/2 +touch 3 +expect_false touch shouldbefail # 5 files will include the "." +expect_false touch ok/shouldbefail # 5 files will include the "." +setfattr . -n ceph.quota.max_files -v 0 +touch shouldbecreated +touch shouldbecreated2 + + +rm -rf * + +# mix +mkdir bytes bytes/files + +setfattr bytes -n ceph.quota.max_bytes -v 10000000 #10m +setfattr bytes/files -n ceph.quota.max_files -v 5 +dd if=/dev/zero of=bytes/files/1 bs=1M count=4 +dd if=/dev/zero of=bytes/files/2 bs=1M count=4 +expect_false write_file bytes/files/3 1000 +expect_false write_file bytes/files/4 1000 +expect_false write_file bytes/files/5 1000 +stat --printf="%n %s\n" bytes/files/1 #4M +stat --printf="%n %s\n" bytes/files/2 #4M +stat --printf="%n %s\n" bytes/files/3 #bigger than 2M +stat --printf="%n %s\n" bytes/files/4 #should be zero +expect_false stat bytes/files/5 #shouldn't be exist + + + + +rm -rf * + +#mv +mkdir files limit +truncate files/file -s 10G +setfattr limit -n ceph.quota.max_bytes -v 1000000 #1m +expect_false mv files limit/ + + + +rm -rf * + +#limit by ancestor + +mkdir -p ancestor/p1/p2/parent/p3 +setfattr ancestor -n ceph.quota.max_bytes -v 1000000 +setfattr ancestor/p1/p2/parent -n ceph.quota.max_bytes -v 1000000000 #1g +expect_false write_file ancestor/p1/p2/parent/p3/file1 900 #900m +stat --printf="%n %s\n" ancestor/p1/p2/parent/p3/file1 + + +#get/set attribute + +setfattr -n ceph.quota.max_bytes -v 0 . +setfattr -n ceph.quota.max_bytes -v 1 . +setfattr -n ceph.quota.max_bytes -v 9223372036854775807 . +expect_false setfattr -n ceph.quota.max_bytes -v 9223372036854775808 . +expect_false setfattr -n ceph.quota.max_bytes -v -1 . +expect_false setfattr -n ceph.quota.max_bytes -v -9223372036854775808 . +expect_false setfattr -n ceph.quota.max_bytes -v -9223372036854775809 . + +setfattr -n ceph.quota.max_files -v 0 . +setfattr -n ceph.quota.max_files -v 1 . +setfattr -n ceph.quota.max_files -v 9223372036854775807 . +expect_false setfattr -n ceph.quota.max_files -v 9223372036854775808 . +expect_false setfattr -n ceph.quota.max_files -v -1 . +expect_false setfattr -n ceph.quota.max_files -v -9223372036854775808 . +expect_false setfattr -n ceph.quota.max_files -v -9223372036854775809 . + +setfattr -n ceph.quota -v "max_bytes=0 max_files=0" . +setfattr -n ceph.quota -v "max_bytes=1 max_files=0" . +setfattr -n ceph.quota -v "max_bytes=0 max_files=1" . +setfattr -n ceph.quota -v "max_bytes=1 max_files=1" . +expect_false setfattr -n ceph.quota -v "max_bytes=-1 max_files=0" . +expect_false setfattr -n ceph.quota -v "max_bytes=0 max_files=-1" . +expect_false setfattr -n ceph.quota -v "max_bytes=-1 max_files=-1" . + +#addme + +cd .. +rm -rf quota-test + +echo OK diff --git a/qa/workunits/fs/snap-hierarchy.sh b/qa/workunits/fs/snap-hierarchy.sh new file mode 100755 index 000000000..67f0e014b --- /dev/null +++ b/qa/workunits/fs/snap-hierarchy.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +set -ex + +if [ -d "$1" ]; then + mkdir -p -- "$1" && cd "$1" +fi + +[ "$VERIFY" != verify ] && mkdir 1 +[ "$VERIFY" != verify ] && mkdir 1/.snap/first +stat 1/.snap/first +[ "$VERIFY" != verify ] && mkdir 1/2 +stat 1/.snap/first/2 && exit 1 +[ "$VERIFY" != verify ] && mkdir 1/2/.snap/second +stat 1/2/.snap/second +[ "$VERIFY" != verify ] && touch 1/foo +stat 1/.snap/first/foo && exit 1 +[ "$VERIFY" != verify ] && mkdir 1/.snap/third +stat 1/.snap/third/foo || exit 1 +[ "$VERIFY" != verify ] && mkdir 1/2/3 +[ "$VERIFY" != verify ] && mkdir 1/2/.snap/fourth +stat 1/2/.snap/fourth/3 + +exit 0 diff --git a/qa/workunits/fs/snaps/snap-rm-diff.sh b/qa/workunits/fs/snaps/snap-rm-diff.sh new file mode 100755 index 000000000..30ffa9113 --- /dev/null +++ b/qa/workunits/fs/snaps/snap-rm-diff.sh @@ -0,0 +1,10 @@ +#!/bin/sh -ex + +wget -q http://download.ceph.com/qa/linux-2.6.33.tar.bz2 +mkdir foo +cp linux* foo +mkdir foo/.snap/barsnap +rm foo/linux* +diff -q foo/.snap/barsnap/linux* linux* && echo "passed: files are identical" +rmdir foo/.snap/barsnap +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-1.sh b/qa/workunits/fs/snaps/snaptest-1.sh new file mode 100755 index 000000000..431e83387 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-1.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -ex + +echo 1 > file1 +echo 2 > file2 +echo 3 > file3 +[ -e file4 ] && rm file4 +mkdir .snap/snap1 +echo 4 > file4 +now=`ls` +then=`ls .snap/snap1` +rmdir .snap/snap1 +if [ "$now" = "$then" ]; then + echo live and snap contents are identical? + false +fi + +# do it again +echo 1 > file1 +echo 2 > file2 +echo 3 > file3 +mkdir .snap/snap1 +echo 4 > file4 +rmdir .snap/snap1 + +rm file? + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-2.sh b/qa/workunits/fs/snaps/snaptest-2.sh new file mode 100755 index 000000000..11fe9316a --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-2.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +echo "Create dir 100 to 199 ..." +for i in $(seq 100 199); do + echo " create dir $i" + mkdir "$i" + for y in $(seq 10 20); do + echo "This is a test file before any snapshot was taken." >"$i/$y" + done +done + +echo "Take first snapshot .snap/test1" +mkdir .snap/test1 + +echo "Create dir 200 to 299 ..." +for i in $(seq 200 299); do + echo " create dir $i" + mkdir $i + for y in $(seq 20 29); do + echo "This is a test file. Created after .snap/test1" >"$i/$y" + done +done + +echo "Create a snapshot in every first level dir ..." +for dir in $(ls); do + echo " create $dir/.snap/snap-subdir-test" + mkdir "$dir/.snap/snap-subdir-test" + for y in $(seq 30 39); do + echo " create $dir/$y file after the snapshot" + echo "This is a test file. Created after $dir/.snap/snap-subdir-test" >"$dir/$y" + done +done + +echo "Take second snapshot .snap/test2" +mkdir .snap/test2 + +echo "Copy content of .snap/test1 to copyofsnap1 ..." +mkdir copyofsnap1 +cp -Rv .snap/test1 copyofsnap1/ + + +echo "Take third snapshot .snap/test3" +mkdir .snap/test3 + +echo "Delete the snapshots..." + +find ./ -type d -print | \ + xargs -I% -n1 find %/.snap -mindepth 1 -maxdepth 1 \ + \( ! -name "_*" \) -print 2>/dev/null + +find ./ -type d -print | \ + xargs -I% -n1 find %/.snap -mindepth 1 -maxdepth 1 \ + \( ! -name "_*" \) -print 2>/dev/null | \ + xargs -n1 rmdir + +echo "Delete all the files and directories ..." +rm -Rfv ./* + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-authwb.sh b/qa/workunits/fs/snaps/snaptest-authwb.sh new file mode 100755 index 000000000..965ee8512 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-authwb.sh @@ -0,0 +1,12 @@ +#!/bin/sh -x + +set -e + +touch foo +chmod +x foo +mkdir .snap/s +find .snap/s/foo -executable | grep foo +rmdir .snap/s +rm foo + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-capwb.sh b/qa/workunits/fs/snaps/snaptest-capwb.sh new file mode 100755 index 000000000..d26f324b6 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-capwb.sh @@ -0,0 +1,33 @@ +#!/bin/sh -x + +set -e + +mkdir foo + +# make sure mds handles it when the client does not send flushsnap +echo x > foo/x +sync +mkdir foo/.snap/ss +ln foo/x foo/xx +cat foo/.snap/ss/x +rmdir foo/.snap/ss + +# +echo a > foo/a +echo b > foo/b +mkdir foo/.snap/s +r=`cat foo/.snap/s/a` +[ -z "$r" ] && echo "a appears empty in snapshot" && false + +ln foo/b foo/b2 +cat foo/.snap/s/b + +echo "this used to hang:" +echo more >> foo/b2 +echo "oh, it didn't hang! good job." +cat foo/b +rmdir foo/.snap/s + +rm -r foo + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-dir-rename.sh b/qa/workunits/fs/snaps/snaptest-dir-rename.sh new file mode 100755 index 000000000..3bbd9a11e --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-dir-rename.sh @@ -0,0 +1,17 @@ +#!/bin/sh -x + +set -e + +# +# make sure we keep an existing dn's seq +# + +mkdir a +mkdir .snap/bar +mkdir a/.snap/foo +rmdir a/.snap/foo +rmdir a +stat .snap/bar/a +rmdir .snap/bar + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-double-null.sh b/qa/workunits/fs/snaps/snaptest-double-null.sh new file mode 100755 index 000000000..cdf32e4f0 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-double-null.sh @@ -0,0 +1,23 @@ +#!/bin/sh -x + +set -e + +# multiple intervening snapshots with no modifications, and thus no +# snapflush client_caps messages. make sure the mds can handle this. + +for f in `seq 1 20` ; do + +mkdir a +cat > a/foo & +mkdir a/.snap/one +mkdir a/.snap/two +chmod 777 a/foo +sync # this might crash the mds +ps +rmdir a/.snap/* +rm a/foo +rmdir a + +done + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-estale.sh b/qa/workunits/fs/snaps/snaptest-estale.sh new file mode 100755 index 000000000..a4fb94368 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-estale.sh @@ -0,0 +1,13 @@ +#!/bin/sh -x + +mkdir .snap/foo + +echo "We want ENOENT, not ESTALE, here." +for f in `seq 1 100` +do + stat .snap/foo/$f 2>&1 | grep 'No such file' +done + +rmdir .snap/foo + +echo "OK" diff --git a/qa/workunits/fs/snaps/snaptest-git-ceph.sh b/qa/workunits/fs/snaps/snaptest-git-ceph.sh new file mode 100755 index 000000000..12c1f0fdc --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-git-ceph.sh @@ -0,0 +1,52 @@ +#!/bin/sh -x + +set -e + +# try it again if the clone is slow and the second time +retried=false +trap -- 'retry' EXIT +retry() { + rm -rf ceph + # double the timeout value + timeout 3600 git clone https://git.ceph.com/ceph.git +} +rm -rf ceph +timeout 1800 git clone https://git.ceph.com/ceph.git +trap - EXIT +cd ceph + +versions=`seq 1 90` + +for v in $versions +do + if [ $v -eq 48 ]; then + continue + fi + ver="v0.$v" + echo $ver + git reset --hard $ver + mkdir .snap/$ver +done + +for v in $versions +do + if [ $v -eq 48 ]; then + continue + fi + ver="v0.$v" + echo checking $ver + cd .snap/$ver + git diff --exit-code + cd ../.. +done + +for v in $versions +do + if [ $v -eq 48 ]; then + continue + fi + ver="v0.$v" + rmdir .snap/$ver +done + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-hardlink.sh b/qa/workunits/fs/snaps/snaptest-hardlink.sh new file mode 100755 index 000000000..90f3583b1 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-hardlink.sh @@ -0,0 +1,25 @@ +#!/bin/sh -x + +set -e + +mkdir 1 2 +echo asdf >1/file1 +echo asdf >1/file2 + +ln 1/file1 2/file1 +ln 1/file2 2/file2 + +mkdir 2/.snap/s1 + +echo qwer >1/file1 +grep asdf 2/.snap/s1/file1 + +rm -f 1/file2 +grep asdf 2/.snap/s1/file2 +rm -f 2/file2 +grep asdf 2/.snap/s1/file2 + +rmdir 2/.snap/s1 +rm -rf 1 2 + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-intodir.sh b/qa/workunits/fs/snaps/snaptest-intodir.sh new file mode 100755 index 000000000..d6a220f73 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-intodir.sh @@ -0,0 +1,22 @@ +#!/bin/sh -ex + +# this tests fix for #1399 +mkdir foo +mkdir foo/.snap/one +touch bar +mv bar foo +sync +# should not crash :) + +mkdir baz +mkdir baz/.snap/two +mv baz foo +sync +# should not crash :) + +# clean up. +rmdir foo/baz/.snap/two +rmdir foo/.snap/one +rm -r foo + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh b/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh new file mode 100755 index 000000000..5ebc852cf --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh @@ -0,0 +1,42 @@ +#!/bin/sh -x + +set -e + +echo asdf > a +mkdir .snap/1 +chmod 777 a +mkdir .snap/2 +echo qwer > a +mkdir .snap/3 +chmod 666 a +mkdir .snap/4 +echo zxcv > a +mkdir .snap/5 + +ls -al .snap/?/a + +grep asdf .snap/1/a +stat .snap/1/a | grep 'Size: 5' + +grep asdf .snap/2/a +stat .snap/2/a | grep 'Size: 5' +stat .snap/2/a | grep -- '-rwxrwxrwx' + +grep qwer .snap/3/a +stat .snap/3/a | grep 'Size: 5' +stat .snap/3/a | grep -- '-rwxrwxrwx' + +grep qwer .snap/4/a +stat .snap/4/a | grep 'Size: 5' +stat .snap/4/a | grep -- '-rw-rw-rw-' + +grep zxcv .snap/5/a +stat .snap/5/a | grep 'Size: 5' +stat .snap/5/a | grep -- '-rw-rw-rw-' + +rmdir .snap/[12345] + +echo "OK" + + + diff --git a/qa/workunits/fs/snaps/snaptest-name-limits.sh b/qa/workunits/fs/snaps/snaptest-name-limits.sh new file mode 100755 index 000000000..f40d0231e --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-name-limits.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# This tests snapshot names limits: names have to be < 240 chars +# + +function cleanup () +{ + rmdir d1/.snap/* + rm -rf d1 +} + +function fail () +{ + echo $@ + cleanup + exit 1 +} + +mkdir d1 + +longname=$(printf "%.241d" 2) +mkdir d1/.snap/$longname 2> /dev/null +[ -d d1/.snap/$longname ] && fail "Invalid snapshot exists: $longname" + +cleanup + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-parents.sh b/qa/workunits/fs/snaps/snaptest-parents.sh new file mode 100755 index 000000000..7ab1ba7cf --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-parents.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +set -e + +echo "making directory tree and files" +mkdir -p 1/a/b/c/ +echo "i'm file1" > 1/a/file1 +echo "i'm file2" > 1/a/b/file2 +echo "i'm file3" > 1/a/b/c/file3 +echo "snapshotting" +mkdir 1/.snap/foosnap1 +mkdir 2 +echo "moving tree" +mv 1/a 2 +echo "checking snapshot contains tree..." +dir1=`find 1/.snap/foosnap1 | wc -w` +dir2=`find 2/ | wc -w` +#diff $dir1 $dir2 && echo "Success!" +test $dir1==$dir2 && echo "Success!" +echo "adding folder and file to tree..." +mkdir 2/a/b/c/d +echo "i'm file 4!" > 2/a/b/c/d/file4 +echo "snapshotting tree 2" +mkdir 2/.snap/barsnap2 +echo "comparing snapshots" +dir1=`find 1/.snap/foosnap1/ -maxdepth 2 | wc -w` +dir2=`find 2/.snap/barsnap2/ -maxdepth 2 | wc -w` +#diff $dir1 $dir2 && echo "Success!" +test $dir1==$dir2 && echo "Success!" +echo "moving subtree to first folder" +mv 2/a/b/c 1 +echo "comparing snapshots and new tree" +dir1=`find 1/ | wc -w` +dir2=`find 2/.snap/barsnap2/a/b/c | wc -w` +#diff $dir1 $dir2 && echo "Success!" +test $dir1==$dir2 && echo "Success!" +rmdir 1/.snap/* +rmdir 2/.snap/* +echo "OK" diff --git a/qa/workunits/fs/snaps/snaptest-realm-split.sh b/qa/workunits/fs/snaps/snaptest-realm-split.sh new file mode 100755 index 000000000..300cca21d --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-realm-split.sh @@ -0,0 +1,31 @@ +#!/bin/sh -x + +set -e + +mkdir -p 1/a +exec 3<> 1/a/file1 + +echo -n a >&3 + +mkdir 1/.snap/s1 + +echo -n b >&3 + +mkdir 2 +# create new snaprealm at dir a, file1's cap should be attached to the new snaprealm +mv 1/a 2 + +mkdir 2/.snap/s2 + +echo -n c >&3 + +exec 3>&- + +grep '^a$' 1/.snap/s1/a/file1 +grep '^ab$' 2/.snap/s2/a/file1 +grep '^abc$' 2/a/file1 + +rmdir 1/.snap/s1 +rmdir 2/.snap/s2 +rm -rf 1 2 +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-snap-rename.sh b/qa/workunits/fs/snaps/snaptest-snap-rename.sh new file mode 100755 index 000000000..aa7325b92 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-snap-rename.sh @@ -0,0 +1,33 @@ +#!/bin/sh -x + +expect_failure() { + if "$@"; then return 1; else return 0; fi +} +set -e + +mkdir -p d1/d2 +mkdir -p d1/d3 +mkdir d1/.snap/foo +mkdir d1/d2/.snap/foo +mkdir d1/d3/.snap/foo +mkdir d1/d3/.snap/bar +mv d1/d2/.snap/foo d1/d2/.snap/bar +# snapshot name can't start with _ +expect_failure mv d1/d2/.snap/bar d1/d2/.snap/_bar +# can't rename parent snapshot +expect_failure mv d1/d2/.snap/_foo_* d1/d2/.snap/foo +expect_failure mv d1/d2/.snap/_foo_* d1/d2/.snap/_foo_1 +# can't rename snapshot to different directroy +expect_failure mv d1/d2/.snap/bar d1/.snap/ +# can't overwrite existing snapshot +expect_failure python3 -c "import os; os.rename('d1/d3/.snap/foo', 'd1/d3/.snap/bar')" +# can't move snaphost out of snapdir +expect_failure python3 -c "import os; os.rename('d1/.snap/foo', 'd1/foo')" + +rmdir d1/.snap/foo +rmdir d1/d2/.snap/bar +rmdir d1/d3/.snap/foo +rmdir d1/d3/.snap/bar +rm -rf d1 + +echo OK diff --git a/qa/workunits/fs/snaps/snaptest-snap-rm-cmp.sh b/qa/workunits/fs/snaps/snaptest-snap-rm-cmp.sh new file mode 100755 index 000000000..88a0e8ae5 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-snap-rm-cmp.sh @@ -0,0 +1,24 @@ +#!/bin/sh -x + +set -e + +file=linux-2.6.33.tar.bz2 +wget -q http://download.ceph.com/qa/$file + +real=`md5sum $file | awk '{print $1}'` + +for f in `seq 1 20` +do + echo $f + cp $file a + mkdir .snap/s + rm a + cp .snap/s/a /tmp/a + cur=`md5sum /tmp/a | awk '{print $1}'` + if [ "$cur" != "$real" ]; then + echo "FAIL: bad match, /tmp/a $cur != real $real" + false + fi + rmdir .snap/s +done +rm $file diff --git a/qa/workunits/fs/snaps/snaptest-upchildrealms.sh b/qa/workunits/fs/snaps/snaptest-upchildrealms.sh new file mode 100755 index 000000000..4e531a966 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-upchildrealms.sh @@ -0,0 +1,28 @@ +#!/bin/sh -x + +set -e + +# +# verify that a snap update on a parent realm will induce +# snap cap writeback for inodes child realms +# + +mkdir a +mkdir a/b +mkdir a/.snap/a1 +mkdir a/b/.snap/b1 +echo asdf > a/b/foo +mkdir a/.snap/a2 +# client _should_ have just queued a capsnap for writeback +ln a/b/foo a/b/bar # make the server cow the inode + +echo "this should not hang..." +cat a/b/.snap/_a2_*/foo +echo "good, it did not hang." + +rmdir a/b/.snap/b1 +rmdir a/.snap/a1 +rmdir a/.snap/a2 +rm -r a + +echo "OK" diff --git a/qa/workunits/fs/snaps/snaptest-xattrwb.sh b/qa/workunits/fs/snaps/snaptest-xattrwb.sh new file mode 100755 index 000000000..e503aed77 --- /dev/null +++ b/qa/workunits/fs/snaps/snaptest-xattrwb.sh @@ -0,0 +1,29 @@ +#!/bin/sh -x + +set -e + +echo "testing simple xattr wb" +touch x +setfattr -n user.foo x +mkdir .snap/s1 +getfattr -n user.foo .snap/s1/x | grep user.foo +rm x +rmdir .snap/s1 + +echo "testing wb with pre-wb server cow" +mkdir a +mkdir a/b +mkdir a/b/c +# b now has As but not Ax +setfattr -n user.foo a/b +mkdir a/.snap/s +mkdir a/b/cc +# b now has been cowed on the server, but we still have dirty xattr caps +getfattr -n user.foo a/b # there they are... +getfattr -n user.foo a/.snap/s/b | grep user.foo # should be there, too! + +# ok, clean up +rmdir a/.snap/s +rm -r a + +echo OK diff --git a/qa/workunits/fs/snaps/untar_snap_rm.sh b/qa/workunits/fs/snaps/untar_snap_rm.sh new file mode 100755 index 000000000..8a8412e66 --- /dev/null +++ b/qa/workunits/fs/snaps/untar_snap_rm.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +do_tarball() { + wget http://download.ceph.com/qa/$1 + tar xvf$2 $1 + mkdir .snap/k + sync + rm -rv $3 + cp -av .snap/k . + rmdir .snap/k + rm -rv k + rm $1 +} + +do_tarball coreutils_8.5.orig.tar.gz z coreutils-8.5 +do_tarball linux-2.6.33.tar.bz2 j linux-2.6.33 diff --git a/qa/workunits/fs/test_o_trunc.c b/qa/workunits/fs/test_o_trunc.c new file mode 100644 index 000000000..1ce19e4bb --- /dev/null +++ b/qa/workunits/fs/test_o_trunc.c @@ -0,0 +1,45 @@ +#include <stdio.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <string.h> +#include <stdlib.h> + +int main(int argc, char *argv[]) +{ + char obuf[32], ibuf[1024]; + int n, max = 0; + + if (argc > 2) + max = atoi(argv[2]); + if (!max) + max = 600; + + memset(obuf, 0xff, sizeof(obuf)); + + for (n = 1; n <= max; ++n) { + int fd, ret; + fd = open(argv[1], O_RDWR | O_CREAT | O_TRUNC, 0644); + printf("%d/%d: open fd = %d\n", n, max, fd); + + ret = write(fd, obuf, sizeof(obuf)); + printf("write ret = %d\n", ret); + + sleep(1); + + ret = write(fd, obuf, sizeof(obuf)); + printf("write ret = %d\n", ret); + + ret = pread(fd, ibuf, sizeof(ibuf), 0); + printf("pread ret = %d\n", ret); + + if (memcmp(obuf, ibuf, sizeof(obuf))) { + printf("mismatch\n"); + close(fd); + break; + } + close(fd); + } + return 0; +} diff --git a/qa/workunits/fs/test_o_trunc.sh b/qa/workunits/fs/test_o_trunc.sh new file mode 100755 index 000000000..90a72600d --- /dev/null +++ b/qa/workunits/fs/test_o_trunc.sh @@ -0,0 +1,7 @@ +#!/bin/sh -ex + +mydir=`dirname $0` +$mydir/test_o_trunc trunc.foo 600 + +echo OK + diff --git a/qa/workunits/fs/test_python.sh b/qa/workunits/fs/test_python.sh new file mode 100755 index 000000000..6e39b95a4 --- /dev/null +++ b/qa/workunits/fs/test_python.sh @@ -0,0 +1,6 @@ +#!/bin/sh -ex + +# Running as root because the filesystem root directory will be +# owned by uid 0, and that's where we're writing. +sudo python3 -m pytest -v $(dirname $0)/../../../src/test/pybind/test_cephfs.py +exit 0 diff --git a/qa/workunits/hadoop/repl.sh b/qa/workunits/hadoop/repl.sh new file mode 100755 index 000000000..84f6150ab --- /dev/null +++ b/qa/workunits/hadoop/repl.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -e +set -x + +# bail if $TESTDIR is not set as this test will fail in that scenario +[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; } + +# if HADOOP_PREFIX is not set, use default +[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/hadoop; } + +# create pools with different replication factors +for repl in 2 3 7 8 9; do + name=hadoop.$repl + ceph osd pool create $name 8 8 + ceph osd pool set $name size $repl + + id=`ceph osd dump | sed -n "s/^pool \([0-9]*\) '$name'.*/\1/p"` + ceph fs add_data_pool cephfs $id +done + +# create a file in each of the pools +for repl in 2 3 7 8 9; do + name=hadoop.$repl + $HADOOP_PREFIX/bin/hadoop fs -rm -f /$name.dat + dd if=/dev/zero bs=1048576 count=1 | \ + $HADOOP_PREFIX/bin/hadoop fs -Dceph.data.pools="$name" \ + -put - /$name.dat +done + +# check that hadoop reports replication matching +# that of the pool the file was written into +for repl in 2 3 7 8 9; do + name=hadoop.$repl + repl2=$($HADOOP_PREFIX/bin/hadoop fs -ls /$name.dat | awk '{print $2}') + if [ $repl -ne $repl2 ]; then + echo "replication factors didn't match!" + exit 1 + fi +done + +exit 0 diff --git a/qa/workunits/hadoop/terasort.sh b/qa/workunits/hadoop/terasort.sh new file mode 100755 index 000000000..3d6988a21 --- /dev/null +++ b/qa/workunits/hadoop/terasort.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +set -e +set -x + +INPUT=/terasort-input +OUTPUT=/terasort-output +REPORT=/tersort-report + +num_records=100000 +[ ! -z $NUM_RECORDS ] && num_records=$NUM_RECORDS + +# bail if $TESTDIR is not set as this test will fail in that scenario +[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; } + +# if HADOOP_PREFIX is not set, use default +[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/hadoop; } + +# Nuke hadoop directories +$HADOOP_PREFIX/bin/hadoop fs -rm -r $INPUT $OUTPUT $REPORT || true + +# Generate terasort data +# +#-Ddfs.blocksize=512M \ +#-Dio.file.buffer.size=131072 \ +#-Dmapreduce.map.java.opts=-Xmx1536m \ +#-Dmapreduce.map.memory.mb=2048 \ +#-Dmapreduce.task.io.sort.mb=256 \ +#-Dyarn.app.mapreduce.am.resource.mb=1024 \ +#-Dmapred.map.tasks=64 \ +$HADOOP_PREFIX/bin/hadoop jar \ + $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \ + teragen \ + -Dmapred.map.tasks=9 \ + $num_records \ + $INPUT + +# Run the sort job +# +#-Ddfs.blocksize=512M \ +#-Dio.file.buffer.size=131072 \ +#-Dmapreduce.map.java.opts=-Xmx1536m \ +#-Dmapreduce.map.memory.mb=2048 \ +#-Dmapreduce.map.output.compress=true \ +#-Dmapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.Lz4Codec \ +#-Dmapreduce.reduce.java.opts=-Xmx1536m \ +#-Dmapreduce.reduce.memory.mb=2048 \ +#-Dmapreduce.task.io.sort.factor=100 \ +#-Dmapreduce.task.io.sort.mb=768 \ +#-Dyarn.app.mapreduce.am.resource.mb=1024 \ +#-Dmapred.reduce.tasks=100 \ +#-Dmapreduce.terasort.output.replication=1 \ +$HADOOP_PREFIX/bin/hadoop jar \ + $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \ + terasort \ + -Dmapred.reduce.tasks=10 \ + $INPUT $OUTPUT + +# Validate the sorted data +# +#-Ddfs.blocksize=512M \ +#-Dio.file.buffer.size=131072 \ +#-Dmapreduce.map.java.opts=-Xmx1536m \ +#-Dmapreduce.map.memory.mb=2048 \ +#-Dmapreduce.reduce.java.opts=-Xmx1536m \ +#-Dmapreduce.reduce.memory.mb=2048 \ +#-Dmapreduce.task.io.sort.mb=256 \ +#-Dyarn.app.mapreduce.am.resource.mb=1024 \ +#-Dmapred.reduce.tasks=1 \ +$HADOOP_PREFIX/bin/hadoop jar \ + $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \ + teravalidate \ + -Dmapred.reduce.tasks=1 \ + $OUTPUT $REPORT + +exit 0 diff --git a/qa/workunits/hadoop/wordcount.sh b/qa/workunits/hadoop/wordcount.sh new file mode 100755 index 000000000..616b08af2 --- /dev/null +++ b/qa/workunits/hadoop/wordcount.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -e +set -x + +WC_INPUT=/wc_input +WC_OUTPUT=/wc_output +DATA_INPUT=$(mktemp -d) + +echo "starting hadoop-wordcount test" + +# bail if $TESTDIR is not set as this test will fail in that scenario +[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; } + +# if HADOOP_PREFIX is not set, use default +[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/hadoop; } + +# Nuke hadoop directories +$HADOOP_PREFIX/bin/hadoop fs -rm -r $WC_INPUT $WC_OUTPUT || true + +# Fetch and import testing data set +curl http://download.ceph.com/qa/hadoop_input_files.tar | tar xf - -C $DATA_INPUT +$HADOOP_PREFIX/bin/hadoop fs -copyFromLocal $DATA_INPUT $WC_INPUT +rm -rf $DATA_INPUT + +# Run the job +$HADOOP_PREFIX/bin/hadoop jar \ + $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \ + wordcount $WC_INPUT $WC_OUTPUT + +# Cleanup +$HADOOP_PREFIX/bin/hadoop fs -rm -r $WC_INPUT $WC_OUTPUT || true + +echo "completed hadoop-wordcount test" +exit 0 diff --git a/qa/workunits/kernel_untar_build.sh b/qa/workunits/kernel_untar_build.sh new file mode 100755 index 000000000..9b60f065c --- /dev/null +++ b/qa/workunits/kernel_untar_build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +wget -O linux.tar.gz http://download.ceph.com/qa/linux-5.4.tar.gz + +mkdir t +cd t +tar xzf ../linux.tar.gz +cd linux* +make defconfig +make -j`grep -c processor /proc/cpuinfo` +cd .. +if ! rm -rv linux* ; then + echo "uh oh rm -r failed, it left behind:" + find . + exit 1 +fi +cd .. +rm -rv t linux* diff --git a/qa/workunits/libcephfs/test.sh b/qa/workunits/libcephfs/test.sh new file mode 100755 index 000000000..c53fe893c --- /dev/null +++ b/qa/workunits/libcephfs/test.sh @@ -0,0 +1,10 @@ +#!/bin/sh -e + +ceph_test_libcephfs +ceph_test_libcephfs_access +ceph_test_libcephfs_reclaim +ceph_test_libcephfs_lazyio +ceph_test_libcephfs_newops +ceph_test_libcephfs_suidsgid + +exit 0 diff --git a/qa/workunits/mgr/test_localpool.sh b/qa/workunits/mgr/test_localpool.sh new file mode 100755 index 000000000..40a749e8d --- /dev/null +++ b/qa/workunits/mgr/test_localpool.sh @@ -0,0 +1,21 @@ +#!/bin/sh -ex + +ceph config set mgr mgr/localpool/subtree host +ceph config set mgr mgr/localpool/failure_domain osd +ceph mgr module enable localpool + +while ! ceph osd pool ls | grep '^by-host-' +do + sleep 5 +done + +ceph mgr module disable localpool +for p in `ceph osd pool ls | grep '^by-host-'` +do + ceph osd pool rm $p $p --yes-i-really-really-mean-it +done + +ceph config rm mgr mgr/localpool/subtree +ceph config rm mgr mgr/localpool/failure_domain + +echo OK diff --git a/qa/workunits/mgr/test_per_module_finisher.sh b/qa/workunits/mgr/test_per_module_finisher.sh new file mode 100755 index 000000000..dc66bce23 --- /dev/null +++ b/qa/workunits/mgr/test_per_module_finisher.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -ex + +# This testcase tests the per module finisher stats for enabled modules +# using check counter (qa/tasks/check_counter.py). + +# 'balancer' commands +ceph balancer pool ls + +# 'crash' commands +ceph crash ls +ceph crash ls-new + +# 'device' commands +ceph device query-daemon-health-metrics mon.a + +# 'iostat' command +ceph iostat & +pid=$! +sleep 3 +kill -SIGTERM $pid + +# 'pg_autoscaler' command +ceph osd pool autoscale-status + +# 'progress' command +ceph progress +ceph progress json + +# 'status' commands +ceph fs status +ceph osd status + +# 'telemetry' commands +ceph telemetry status +ceph telemetry diff + +echo OK diff --git a/qa/workunits/mon/auth_caps.sh b/qa/workunits/mon/auth_caps.sh new file mode 100755 index 000000000..1f59ae1f7 --- /dev/null +++ b/qa/workunits/mon/auth_caps.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +set -e +set -x +declare -A keymap + +combinations="r w x rw rx wx rwx" + +for i in ${combinations}; do + k="foo_$i" + k=`ceph auth get-or-create-key client.$i mon "allow $i"` || exit 1 + keymap["$i"]=$k +done + +# add special caps +keymap["all"]=`ceph auth get-or-create-key client.all mon 'allow *'` || exit 1 + +tmp=`mktemp` +ceph auth export > $tmp + +trap "rm $tmp" INT ERR EXIT QUIT 0 + +expect() { + + set +e + + local expected_ret=$1 + local ret + + shift + cmd=$@ + + eval $cmd + ret=$? + + set -e + + if [[ $ret -ne $expected_ret ]]; then + echo "ERROR: running \'$cmd\': expected $expected_ret got $ret" + return 1 + fi + + return 0 +} + +read_ops() { + local caps=$1 + local has_read=1 has_exec=1 + local ret + local args + + ( echo $caps | grep 'r' ) || has_read=0 + ( echo $caps | grep 'x' ) || has_exec=0 + + if [[ "$caps" == "all" ]]; then + has_read=1 + has_exec=1 + fi + + ret=13 + if [[ $has_read -gt 0 && $has_exec -gt 0 ]]; then + ret=0 + fi + + args="--id $caps --key ${keymap[$caps]}" + + expect $ret ceph auth get client.admin $args + expect $ret ceph auth get-key client.admin $args + expect $ret ceph auth export $args + expect $ret ceph auth export client.admin $args + expect $ret ceph auth ls $args + expect $ret ceph auth print-key client.admin $args + expect $ret ceph auth print_key client.admin $args +} + +write_ops() { + + local caps=$1 + local has_read=1 has_write=1 has_exec=1 + local ret + local args + + ( echo $caps | grep 'r' ) || has_read=0 + ( echo $caps | grep 'w' ) || has_write=0 + ( echo $caps | grep 'x' ) || has_exec=0 + + if [[ "$caps" == "all" ]]; then + has_read=1 + has_write=1 + has_exec=1 + fi + + ret=13 + if [[ $has_read -gt 0 && $has_write -gt 0 && $has_exec -gt 0 ]]; then + ret=0 + fi + + args="--id $caps --key ${keymap[$caps]}" + + expect $ret ceph auth add client.foo $args + expect $ret "ceph auth caps client.foo mon 'allow *' $args" + expect $ret ceph auth get-or-create client.admin $args + expect $ret ceph auth get-or-create-key client.admin $args + expect $ret ceph auth get-or-create-key client.baz $args + expect $ret ceph auth del client.foo $args + expect $ret ceph auth del client.baz $args + expect $ret ceph auth import -i $tmp $args +} + +echo "running combinations: ${!keymap[@]}" + +subcmd=$1 + +for i in ${!keymap[@]}; do + echo "caps: $i" + if [[ -z "$subcmd" || "$subcmd" == "read" || "$subcmd" == "all" ]]; then + read_ops $i + fi + + if [[ -z "$subcmd" || "$subcmd" == "write" || "$subcmd" == "all" ]]; then + write_ops $i + fi +done + +# cleanup +for i in ${combinations} all; do + ceph auth del client.$i || exit 1 +done + +echo "OK" diff --git a/qa/workunits/mon/auth_key_rotation.sh b/qa/workunits/mon/auth_key_rotation.sh new file mode 100755 index 000000000..1a53bab6d --- /dev/null +++ b/qa/workunits/mon/auth_key_rotation.sh @@ -0,0 +1,58 @@ +#!/usr/bin/bash -ex + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + + +ceph auth export +ceph auth rm client.rot + +ceph auth get-or-create client.rot mon 'allow rwx' +ceph auth export client.rot | grep key +ceph auth export client.rot | expect_false grep pending.key + +ceph auth get-or-create-pending client.rot +ceph auth export client.rot | grep key +ceph auth export client.rot | grep pending.key + +ceph auth clear-pending client.rot +ceph auth export client.rot | expect_false grep pending.key + +ceph auth get-or-create-pending client.rot +ceph auth export client.rot | grep key +ceph auth export client.rot | grep pending.key +K=$(ceph auth export client.rot | grep 'key = ' | head -n 1 | awk '{print $3}') +PK=$(ceph auth export client.rot | grep pending.key | awk '{print $4}') +echo "K is $K" +echo "PK is $PK" +ceph -n client.rot --key $K -s + +ceph auth commit-pending client.rot +ceph auth export client.rot | expect_false grep pending.key +ceph auth export client.rot | grep key | grep $PK + +ceph auth get-or-create-pending client.rot +ceph auth export client.rot | grep key +ceph auth export client.rot | grep pending.key +K=$(ceph auth export client.rot | grep 'key = ' | head -n 1 | awk '{print $3}') +PK=$(ceph auth export client.rot | grep pending.key | awk '{print $4}') +echo "2, K is $K" +echo "2, PK is $PK" + +ceph auth export client.rot + +while ceph -n client.rot --key $K -s ; do + ceph auth export client.rot + ceph -n client.rot --key $PK -s + sleep 1 +done + +ceph auth export client.rot | expect_false grep pending.key +ceph auth export client.rot | grep key | grep $PK + +ceph -n client.rot --key $PK -s + +echo ok diff --git a/qa/workunits/mon/caps.py b/qa/workunits/mon/caps.py new file mode 100644 index 000000000..26c0cd14c --- /dev/null +++ b/qa/workunits/mon/caps.py @@ -0,0 +1,359 @@ +#!/usr/bin/python3 + +from __future__ import print_function + +import subprocess +import shlex +import errno +import sys +import os +import io +import re + +from ceph_argparse import * # noqa + +keyring_base = '/tmp/cephtest-caps.keyring' + +class UnexpectedReturn(Exception): + def __init__(self, cmd, ret, expected, msg): + if isinstance(cmd, list): + self.cmd = ' '.join(cmd) + else: + assert isinstance(cmd, str), 'cmd needs to be either a list or a str' + self.cmd = cmd + self.cmd = str(self.cmd) + self.ret = int(ret) + self.expected = int(expected) + self.msg = str(msg) + + def __str__(self): + return repr('{c}: expected return {e}, got {r} ({o})'.format( + c=self.cmd, e=self.expected, r=self.ret, o=self.msg)) + +def call(cmd): + if isinstance(cmd, list): + args = cmd + elif isinstance(cmd, str): + args = shlex.split(cmd) + else: + assert False, 'cmd is not a string/unicode nor a list!' + + print('call: {0}'.format(args)) + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ret = proc.wait() + + return (ret, proc) + +def expect(cmd, expected_ret): + + try: + (r, p) = call(cmd) + except ValueError as e: + print('unable to run {c}: {err}'.format(c=repr(cmd), err=e.message), + file=sys.stderr) + return errno.EINVAL + + assert r == p.returncode, \ + 'wth? r was supposed to match returncode!' + + if r != expected_ret: + raise UnexpectedReturn(repr(cmd), r, expected_ret, str(p.stderr.read())) + + return p + +def expect_to_file(cmd, expected_ret, out_file): + + # Let the exception be propagated to the caller + p = expect(cmd, expected_ret) + assert p.returncode == expected_ret, \ + 'expected result doesn\'t match and no exception was thrown!' + + with io.open(out_file, 'ab') as file: + file.write(p.stdout.read()) + + return p + +class Command: + def __init__(self, cid, j): + self.cid = cid[3:] + self.perms = j['perm'] + self.module = j['module'] + + self.sig = '' + self.args = [] + for s in j['sig']: + if not isinstance(s, dict): + assert isinstance(s, str), \ + 'malformatted signature cid {0}: {1}\n{2}'.format(cid,s,j) + if len(self.sig) > 0: + self.sig += ' ' + self.sig += s + else: + self.args.append(s) + + def __str__(self): + return repr('command {0}: {1} (requires \'{2}\')'.format(self.cid,\ + self.sig, self.perms)) + + +def destroy_keyring(path): + if not os.path.exists(path): + raise Exception('oops! cannot remove inexistent keyring {0}'.format(path)) + + # grab all client entities from the keyring + entities = [m.group(1) for m in [re.match(r'\[client\.(.*)\]', l) + for l in [str(line.strip()) + for line in io.open(path,'r')]] if m is not None] + + # clean up and make sure each entity is gone + for e in entities: + expect('ceph auth del client.{0}'.format(e), 0) + expect('ceph auth get client.{0}'.format(e), errno.ENOENT) + + # remove keyring + os.unlink(path) + + return True + +def test_basic_auth(): + # make sure we can successfully add/del entities, change their caps + # and import/export keyrings. + + expect('ceph auth add client.basicauth', 0) + expect('ceph auth caps client.basicauth mon \'allow *\'', 0) + # entity exists and caps do not match + expect('ceph auth add client.basicauth', errno.EINVAL) + # this command attempts to change an existing state and will fail + expect('ceph auth add client.basicauth mon \'allow w\'', errno.EINVAL) + expect('ceph auth get-or-create client.basicauth', 0) + expect('ceph auth get-key client.basicauth', 0) + expect('ceph auth get-or-create client.basicauth2', 0) + # cleanup + expect('ceph auth del client.basicauth', 0) + expect('ceph auth del client.basicauth2', 0) + + return True + +def gen_module_keyring(module): + module_caps = [ + ('all', '{t} \'allow service {s} rwx\'', 0), + ('none', '', errno.EACCES), + ('wrong', '{t} \'allow service foobar rwx\'', errno.EACCES), + ('right', '{t} \'allow service {s} {p}\'', 0), + ('no-execute', '{t} \'allow service {s} x\'', errno.EACCES) + ] + + keyring = '{0}.service-{1}'.format(keyring_base,module) + for perms in 'r rw x'.split(): + for (n,p,r) in module_caps: + c = p.format(t='mon', s=module, p=perms) + expect_to_file( + 'ceph auth get-or-create client.{cn}-{cp} {caps}'.format( + cn=n,cp=perms,caps=c), 0, keyring) + + return keyring + + +def test_all(): + + + perms = { + 'good': { + 'broad':[ + ('rwx', 'allow *'), + ('r', 'allow r'), + ('rw', 'allow rw'), + ('x', 'allow x'), + ], + 'service':[ + ('rwx', 'allow service {s} rwx'), + ('r', 'allow service {s} r'), + ('rw', 'allow service {s} rw'), + ('x', 'allow service {s} x'), + ], + 'command':[ + ('rwx', 'allow command "{c}"'), + ], + 'command-with':[ + ('rwx', 'allow command "{c}" with {kv}') + ], + 'command-with-prefix':[ + ('rwx', 'allow command "{c}" with {key} prefix {val}') + ] + }, + 'bad': { + 'broad':[ + ('none', ''), + ], + 'service':[ + ('none1', 'allow service foo rwx'), + ('none2', 'allow service foo r'), + ('none3', 'allow service foo rw'), + ('none4', 'allow service foo x'), + ], + 'command':[ + ('none', 'allow command foo'), + ], + 'command-with':[ + ('none', 'allow command "{c}" with foo=bar'), + ], + 'command-with-prefix':[ + ('none', 'allow command "{c}" with foo prefix bar'), + ], + } + } + + cmds = { + '':[ + { + 'cmd':('status', '', 'r') + }, + { + 'pre':'heap start_profiler', + 'cmd':('heap', 'heapcmd=stats', 'rw'), + 'post':'heap stop_profiler' + } + ], + 'auth':[ + { + 'pre':'', + 'cmd':('auth ls', '', 'r'), + 'post':'' + }, + { + 'pre':'auth get-or-create client.foo mon \'allow *\'', + 'cmd':('auth caps', 'entity="client.foo"', 'rw'), + 'post':'auth del client.foo' + } + ], + 'pg':[ + { + 'cmd':('pg getmap', '', 'r'), + }, + ], + 'mds':[ + { + 'cmd':('mds getmap', '', 'r'), + }, + ], + 'mon':[ + { + 'cmd':('mon getmap', '', 'r') + }, + { + 'cmd':('mon remove', 'name=a', 'rw') + } + ], + 'osd':[ + { + 'cmd':('osd getmap', '', 'r'), + }, + { + 'cmd':('osd pause', '', 'rw'), + 'post':'osd unpause' + }, + { + 'cmd':('osd crush dump', '', 'r') + }, + ], + 'config-key':[ + { + 'pre':'config-key set foo bar', + 'cmd':('config-key get', 'key=foo', 'r') + }, + { + 'pre':'config-key set foo bar', + 'cmd':('config-key del', 'key=foo', 'rw') + } + ] + } + + for (module,cmd_lst) in cmds.items(): + k = keyring_base + '.' + module + for cmd in cmd_lst: + + (cmd_cmd, cmd_args, cmd_perm) = cmd['cmd'] + cmd_args_key = '' + cmd_args_val = '' + if len(cmd_args) > 0: + (cmd_args_key, cmd_args_val) = cmd_args.split('=') + + print('generating keyring for {m}/{c}'.format(m=module,c=cmd_cmd)) + # gen keyring + for (good_or_bad,kind_map) in perms.items(): + for (kind,lst) in kind_map.items(): + for (perm, cap) in lst: + cap_formatted = cap.format( + s=module, + c=cmd_cmd, + kv=cmd_args, + key=cmd_args_key, + val=cmd_args_val) + + if len(cap_formatted) == 0: + run_cap = '' + else: + run_cap = 'mon \'{fc}\''.format(fc=cap_formatted) + + cname = 'client.{gb}-{kind}-{p}'.format( + gb=good_or_bad,kind=kind,p=perm) + expect_to_file( + 'ceph auth get-or-create {n} {c}'.format( + n=cname,c=run_cap), 0, k) + # keyring generated + print('testing {m}/{c}'.format(m=module,c=cmd_cmd)) + + # test + for good_bad in perms.keys(): + for (kind,lst) in perms[good_bad].items(): + for (perm,_) in lst: + cname = 'client.{gb}-{k}-{p}'.format(gb=good_bad,k=kind,p=perm) + + if good_bad == 'good': + expect_ret = 0 + else: + expect_ret = errno.EACCES + + if ( cmd_perm not in perm ): + expect_ret = errno.EACCES + if 'with' in kind and len(cmd_args) == 0: + expect_ret = errno.EACCES + if 'service' in kind and len(module) == 0: + expect_ret = errno.EACCES + + if 'pre' in cmd and len(cmd['pre']) > 0: + expect('ceph {0}'.format(cmd['pre']), 0) + expect('ceph -n {cn} -k {k} {c} {arg_val}'.format( + cn=cname,k=k,c=cmd_cmd,arg_val=cmd_args_val), expect_ret) + if 'post' in cmd and len(cmd['post']) > 0: + expect('ceph {0}'.format(cmd['post']), 0) + # finish testing + destroy_keyring(k) + + + return True + + +def test_misc(): + + k = keyring_base + '.misc' + expect_to_file( + 'ceph auth get-or-create client.caps mon \'allow command "auth caps"' \ + ' with entity="client.caps"\'', 0, k) + expect('ceph -n client.caps -k {kf} quorum_status'.format(kf=k), errno.EACCES) + expect('ceph -n client.caps -k {kf} auth caps client.caps mon \'allow *\''.format(kf=k), 0) + expect('ceph -n client.caps -k {kf} quorum_status'.format(kf=k), 0) + destroy_keyring(k) + +def main(): + + test_basic_auth() + test_all() + test_misc() + + print('OK') + + return 0 + +if __name__ == '__main__': + main() diff --git a/qa/workunits/mon/caps.sh b/qa/workunits/mon/caps.sh new file mode 100755 index 000000000..eae5d8665 --- /dev/null +++ b/qa/workunits/mon/caps.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +set -x + +tmp=/tmp/cephtest-mon-caps-madness + +exit_on_error=1 + +[[ ! -z $TEST_EXIT_ON_ERROR ]] && exit_on_error=$TEST_EXIT_ON_ERROR + +if [ `uname` = FreeBSD ]; then + ETIMEDOUT=60 +else + ETIMEDOUT=110 +fi + +expect() +{ + cmd=$1 + expected_ret=$2 + + echo $cmd + eval $cmd >&/dev/null + ret=$? + + if [[ $ret -ne $expected_ret ]]; then + echo "Error: Expected return $expected_ret, got $ret" + [[ $exit_on_error -eq 1 ]] && exit 1 + return 1 + fi + + return 0 +} + +expect "ceph auth get-or-create client.bazar > $tmp.bazar.keyring" 0 +expect "ceph -k $tmp.bazar.keyring --user bazar quorum_status" 13 +ceph auth del client.bazar + +c="'allow command \"auth ls\", allow command quorum_status'" +expect "ceph auth get-or-create client.foo mon $c > $tmp.foo.keyring" 0 +expect "ceph -k $tmp.foo.keyring --user foo quorum_status" 0 +expect "ceph -k $tmp.foo.keyring --user foo auth ls" 0 +expect "ceph -k $tmp.foo.keyring --user foo auth export" 13 +expect "ceph -k $tmp.foo.keyring --user foo auth del client.bazar" 13 +expect "ceph -k $tmp.foo.keyring --user foo osd dump" 13 + +# monitor drops the subscribe message from client if it does not have enough caps +# for read from mon. in that case, the client will be waiting for mgrmap in vain, +# if it is instructed to send a command to mgr. "pg dump" is served by mgr. so, +# we need to set a timeout for testing this scenario. +# +# leave plenty of time here because the mons might be thrashing. +export CEPH_ARGS='--rados-mon-op-timeout=300' +expect "ceph -k $tmp.foo.keyring --user foo pg dump" $ETIMEDOUT +export CEPH_ARGS='' + +ceph auth del client.foo +expect "ceph -k $tmp.foo.keyring --user foo quorum_status" 13 + +c="'allow command service with prefix=list, allow command quorum_status'" +expect "ceph auth get-or-create client.bar mon $c > $tmp.bar.keyring" 0 +expect "ceph -k $tmp.bar.keyring --user bar quorum_status" 0 +expect "ceph -k $tmp.bar.keyring --user bar auth ls" 13 +expect "ceph -k $tmp.bar.keyring --user bar auth export" 13 +expect "ceph -k $tmp.bar.keyring --user bar auth del client.foo" 13 +expect "ceph -k $tmp.bar.keyring --user bar osd dump" 13 + +# again, we'll need to timeout. +export CEPH_ARGS='--rados-mon-op-timeout=300' +expect "ceph -k $tmp.bar.keyring --user bar pg dump" $ETIMEDOUT +export CEPH_ARGS='' + +ceph auth del client.bar +expect "ceph -k $tmp.bar.keyring --user bar quorum_status" 13 + +rm $tmp.bazar.keyring $tmp.foo.keyring $tmp.bar.keyring + +# invalid caps health warning +cat <<EOF | ceph auth import -i - +[client.bad] + caps mon = this is wrong + caps osd = does not parse + caps mds = also does not parse +EOF +ceph health | grep AUTH_BAD_CAP +ceph health detail | grep client.bad +ceph auth rm client.bad +expect "ceph auth health | grep AUTH_BAD_CAP" 1 + +echo OK diff --git a/qa/workunits/mon/config.sh b/qa/workunits/mon/config.sh new file mode 100755 index 000000000..1b00201ae --- /dev/null +++ b/qa/workunits/mon/config.sh @@ -0,0 +1,136 @@ +#!/bin/bash -ex + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +ceph config dump + +# value validation +ceph config set mon.a debug_asok 22 +ceph config set mon.a debug_asok 22/33 +ceph config get mon.a debug_asok | grep 22 +ceph config set mon.a debug_asok 1/2 +expect_false ceph config set mon.a debug_asok foo +expect_false ceph config set mon.a debug_asok -10 +ceph config rm mon.a debug_asok + +ceph config set global log_graylog_port 123 +expect_false ceph config set global log_graylog_port asdf +ceph config rm global log_graylog_port + +ceph config set mon mon_cluster_log_to_stderr true +ceph config get mon.a mon_cluster_log_to_stderr | grep true +ceph config set mon mon_cluster_log_to_stderr 2 +ceph config get mon.a mon_cluster_log_to_stderr | grep true +ceph config set mon mon_cluster_log_to_stderr 1 +ceph config get mon.a mon_cluster_log_to_stderr | grep true +ceph config set mon mon_cluster_log_to_stderr false +ceph config get mon.a mon_cluster_log_to_stderr | grep false +ceph config set mon mon_cluster_log_to_stderr 0 +ceph config get mon.a mon_cluster_log_to_stderr | grep false +expect_false ceph config set mon mon_cluster_log_to_stderr fiddle +expect_false ceph config set mon mon_cluster_log_to_stderr '' +ceph config rm mon mon_cluster_log_to_stderr + +expect_false ceph config set mon.a osd_pool_default_type foo +ceph config set mon.a osd_pool_default_type replicated +ceph config rm mon.a osd_pool_default_type + +# scoping +ceph config set global debug_asok 33 +ceph config get mon.a debug_asok | grep 33 +ceph config set mon debug_asok 11 +ceph config get mon.a debug_asok | grep 11 +ceph config set mon.a debug_asok 22 +ceph config get mon.a debug_asok | grep 22 +ceph config rm mon.a debug_asok +ceph config get mon.a debug_asok | grep 11 +ceph config rm mon debug_asok +ceph config get mon.a debug_asok | grep 33 +# nested .-prefix scoping +ceph config set client.foo debug_asok 44 +ceph config get client.foo.bar debug_asok | grep 44 +ceph config get client.foo.bar.baz debug_asok | grep 44 +ceph config set client.foo.bar debug_asok 55 +ceph config get client.foo.bar.baz debug_asok | grep 55 +ceph config rm client.foo debug_asok +ceph config get client.foo.bar.baz debug_asok | grep 55 +ceph config rm client.foo.bar debug_asok +ceph config get client.foo.bar.baz debug_asok | grep 33 +ceph config rm global debug_asok + +# whitespace keys +ceph config set client.foo 'debug asok' 44 +ceph config get client.foo 'debug asok' | grep 44 +ceph config set client.foo debug_asok 55 +ceph config get client.foo 'debug asok' | grep 55 +ceph config set client.foo 'debug asok' 66 +ceph config get client.foo debug_asok | grep 66 +ceph config rm client.foo debug_asok +ceph config set client.foo debug_asok 66 +ceph config rm client.foo 'debug asok' + +# help +ceph config help debug_asok | grep debug_asok + +# show +ceph config set osd.0 debug_asok 33 +while ! ceph config show osd.0 | grep debug_asok | grep 33 | grep mon +do + sleep 1 +done +ceph config set osd.0 debug_asok 22 +while ! ceph config show osd.0 | grep debug_asok | grep 22 | grep mon +do + sleep 1 +done + +ceph tell osd.0 config set debug_asok 99 +while ! ceph config show osd.0 | grep debug_asok | grep 99 +do + sleep 1 +done +ceph config show osd.0 | grep debug_asok | grep 'override mon' +ceph tell osd.0 config unset debug_asok +ceph tell osd.0 config unset debug_asok + +ceph config rm osd.0 debug_asok +while ceph config show osd.0 | grep debug_asok | grep mon +do + sleep 1 +done +ceph config show osd.0 | grep -c debug_asok | grep 0 + +ceph config set osd.0 osd_scrub_cost 123 +while ! ceph config show osd.0 | grep osd_scrub_cost | grep mon +do + sleep 1 +done +ceph config rm osd.0 osd_scrub_cost + +# show-with-defaults +ceph config show-with-defaults osd.0 | grep debug_asok + +# assimilate +t1=`mktemp` +t2=`mktemp` +cat <<EOF > $t1 +[osd.0] +keyring = foo +debug_asok = 66 +EOF +ceph config assimilate-conf -i $t1 | tee $t2 + +grep keyring $t2 +expect_false grep debug_asok $t2 +rm -f $t1 $t2 + +expect_false ceph config reset +expect_false ceph config reset -1 +# we are at end of testing, so it's okay to revert everything +ceph config reset 0 + +echo OK diff --git a/qa/workunits/mon/crush_ops.sh b/qa/workunits/mon/crush_ops.sh new file mode 100755 index 000000000..a68761985 --- /dev/null +++ b/qa/workunits/mon/crush_ops.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash + +set -ex + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +ceph osd crush dump + +# rules +ceph osd crush rule dump +ceph osd crush rule ls +ceph osd crush rule list + +ceph osd crush rule create-simple foo default host +ceph osd crush rule create-simple foo default host +ceph osd crush rule create-simple bar default host + +ceph osd crush rm-device-class all +ceph osd crush set-device-class ssd osd.0 +ceph osd crush set-device-class hdd osd.1 +ceph osd crush rule create-replicated foo-ssd default host ssd +ceph osd crush rule create-replicated foo-hdd default host hdd +ceph osd crush rule ls-by-class ssd | grep 'foo-ssd' +ceph osd crush rule ls-by-class ssd | expect_false grep 'foo-hdd' +ceph osd crush rule ls-by-class hdd | grep 'foo-hdd' +ceph osd crush rule ls-by-class hdd | expect_false grep 'foo-ssd' + +ceph osd erasure-code-profile set ec-foo-ssd crush-device-class=ssd m=2 k=2 +ceph osd pool create ec-foo 2 erasure ec-foo-ssd +ceph osd pool rm ec-foo ec-foo --yes-i-really-really-mean-it + +ceph osd crush rule ls | grep foo + +ceph osd crush rule rename foo foo-asdf +ceph osd crush rule rename foo foo-asdf # idempotent +ceph osd crush rule rename bar bar-asdf +ceph osd crush rule ls | grep 'foo-asdf' +ceph osd crush rule ls | grep 'bar-asdf' +ceph osd crush rule rm foo 2>&1 | grep 'does not exist' +ceph osd crush rule rm bar 2>&1 | grep 'does not exist' +ceph osd crush rule rename foo-asdf foo +ceph osd crush rule rename foo-asdf foo # idempotent +ceph osd crush rule rename bar-asdf bar +ceph osd crush rule ls | expect_false grep 'foo-asdf' +ceph osd crush rule ls | expect_false grep 'bar-asdf' +ceph osd crush rule rm foo +ceph osd crush rule rm foo # idempotent +ceph osd crush rule rm bar + +# can't delete in-use rules, tho: +ceph osd pool create pinning_pool 1 +expect_false ceph osd crush rule rm replicated_rule +ceph osd pool rm pinning_pool pinning_pool --yes-i-really-really-mean-it + +# build a simple map +expect_false ceph osd crush add-bucket foo osd +ceph osd crush add-bucket foo root +o1=`ceph osd create` +o2=`ceph osd create` +ceph osd crush add $o1 1 host=host1 root=foo +ceph osd crush add $o1 1 host=host1 root=foo # idemptoent +ceph osd crush add $o2 1 host=host2 root=foo +ceph osd crush add $o2 1 host=host2 root=foo # idempotent +ceph osd crush add-bucket bar root +ceph osd crush add-bucket bar root # idempotent +ceph osd crush link host1 root=bar +ceph osd crush link host1 root=bar # idempotent +ceph osd crush link host2 root=bar +ceph osd crush link host2 root=bar # idempotent + +ceph osd tree | grep -c osd.$o1 | grep -q 2 +ceph osd tree | grep -c host1 | grep -q 2 +ceph osd tree | grep -c osd.$o2 | grep -q 2 +ceph osd tree | grep -c host2 | grep -q 2 +expect_false ceph osd crush rm host1 foo # not empty +ceph osd crush unlink host1 foo +ceph osd crush unlink host1 foo +ceph osd tree | grep -c host1 | grep -q 1 + +expect_false ceph osd crush rm foo # not empty +expect_false ceph osd crush rm bar # not empty +ceph osd crush unlink host1 bar +ceph osd tree | grep -c host1 | grep -q 1 # now an orphan +ceph osd crush rm osd.$o1 host1 +ceph osd crush rm host1 +ceph osd tree | grep -c host1 | grep -q 0 +expect_false ceph osd tree-from host1 +ceph osd tree-from host2 +expect_false ceph osd tree-from osd.$o2 + +expect_false ceph osd crush rm bar # not empty +ceph osd crush unlink host2 + +ceph osd crush add-bucket host-for-test host root=root-for-test rack=rack-for-test +ceph osd tree | grep host-for-test +ceph osd tree | grep rack-for-test +ceph osd tree | grep root-for-test +ceph osd crush rm host-for-test +ceph osd crush rm rack-for-test +ceph osd crush rm root-for-test + +# reference foo and bar with a rule +ceph osd crush rule create-simple foo-rule foo host firstn +expect_false ceph osd crush rm foo +ceph osd crush rule rm foo-rule + +ceph osd crush rm bar +ceph osd crush rm foo +ceph osd crush rm osd.$o2 host2 +ceph osd crush rm host2 + +ceph osd crush add-bucket foo host +ceph osd crush move foo root=default rack=localrack + +ceph osd crush create-or-move osd.$o1 1.0 root=default +ceph osd crush move osd.$o1 host=foo +ceph osd find osd.$o1 | grep host | grep foo + +ceph osd crush rm osd.$o1 +ceph osd crush rm osd.$o2 + +ceph osd crush rm foo + +# test reweight +o3=`ceph osd create` +ceph osd crush add $o3 123 root=default +ceph osd tree | grep osd.$o3 | grep 123 +ceph osd crush reweight osd.$o3 113 +expect_false ceph osd crush reweight osd.$o3 123456 +ceph osd tree | grep osd.$o3 | grep 113 +ceph osd crush rm osd.$o3 +ceph osd rm osd.$o3 + +# test reweight-subtree +o4=`ceph osd create` +o5=`ceph osd create` +ceph osd crush add $o4 123 root=default host=foobaz +ceph osd crush add $o5 123 root=default host=foobaz +ceph osd tree | grep osd.$o4 | grep 123 +ceph osd tree | grep osd.$o5 | grep 123 +ceph osd crush reweight-subtree foobaz 155 +expect_false ceph osd crush reweight-subtree foobaz 123456 +ceph osd tree | grep osd.$o4 | grep 155 +ceph osd tree | grep osd.$o5 | grep 155 +ceph osd crush rm osd.$o4 +ceph osd crush rm osd.$o5 +ceph osd rm osd.$o4 +ceph osd rm osd.$o5 + +# weight sets +# make sure we require luminous before testing weight-sets +ceph osd set-require-min-compat-client luminous +ceph osd crush weight-set dump +ceph osd crush weight-set ls +expect_false ceph osd crush weight-set reweight fooset osd.0 .9 +ceph osd pool create fooset 8 +ceph osd pool create barset 8 +ceph osd pool set barset size 3 +expect_false ceph osd crush weight-set reweight fooset osd.0 .9 +ceph osd crush weight-set create fooset flat +ceph osd crush weight-set create barset positional +ceph osd crush weight-set ls | grep fooset +ceph osd crush weight-set ls | grep barset +ceph osd crush weight-set dump +ceph osd crush weight-set reweight fooset osd.0 .9 +expect_false ceph osd crush weight-set reweight fooset osd.0 .9 .9 +expect_false ceph osd crush weight-set reweight barset osd.0 .9 +ceph osd crush weight-set reweight barset osd.0 .9 .9 .9 +ceph osd crush weight-set ls | grep -c fooset | grep -q 1 +ceph osd crush weight-set rm fooset +ceph osd crush weight-set ls | grep -c fooset | grep -q 0 +ceph osd crush weight-set ls | grep barset +ceph osd crush weight-set rm barset +ceph osd crush weight-set ls | grep -c barset | grep -q 0 +ceph osd crush weight-set create-compat +ceph osd crush weight-set ls | grep '(compat)' +ceph osd crush weight-set rm-compat + +# weight set vs device classes +ceph osd pool create cool 2 +ceph osd pool create cold 2 +ceph osd pool set cold size 2 +ceph osd crush weight-set create-compat +ceph osd crush weight-set create cool flat +ceph osd crush weight-set create cold positional +ceph osd crush rm-device-class osd.0 +ceph osd crush weight-set reweight-compat osd.0 10.5 +ceph osd crush weight-set reweight cool osd.0 11.5 +ceph osd crush weight-set reweight cold osd.0 12.5 12.4 +ceph osd crush set-device-class fish osd.0 +ceph osd crush tree --show-shadow | grep osd\\.0 | grep fish | grep 10\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep fish | grep 11\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep fish | grep 12\\. +ceph osd crush rm-device-class osd.0 +ceph osd crush set-device-class globster osd.0 +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 10\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 11\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 12\\. +ceph osd crush weight-set reweight-compat osd.0 7.5 +ceph osd crush weight-set reweight cool osd.0 8.5 +ceph osd crush weight-set reweight cold osd.0 6.5 6.6 +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 7\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 8\\. +ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 6\\. +ceph osd crush rm-device-class osd.0 +ceph osd pool rm cool cool --yes-i-really-really-mean-it +ceph osd pool rm cold cold --yes-i-really-really-mean-it +ceph osd crush weight-set rm-compat + +# weight set vs device classes vs move +ceph osd crush weight-set create-compat +ceph osd crush add-bucket fooo host +ceph osd crush move fooo root=default +ceph osd crush add-bucket barr rack +ceph osd crush move barr root=default +ceph osd crush move fooo rack=barr +ceph osd crush rm fooo +ceph osd crush rm barr +ceph osd crush weight-set rm-compat + +# this sequence would crash at one point +ceph osd crush weight-set create-compat +ceph osd crush add-bucket r1 rack root=default +for f in `seq 1 32`; do + ceph osd crush add-bucket h$f host rack=r1 +done +for f in `seq 1 32`; do + ceph osd crush rm h$f +done +ceph osd crush rm r1 +ceph osd crush weight-set rm-compat + +echo OK diff --git a/qa/workunits/mon/osd.sh b/qa/workunits/mon/osd.sh new file mode 100755 index 000000000..535d6c137 --- /dev/null +++ b/qa/workunits/mon/osd.sh @@ -0,0 +1,24 @@ +#!/bin/sh -x + +set -e + +ua=`uuidgen` +ub=`uuidgen` + +# should get same id with same uuid +na=`ceph osd create $ua` +test $na -eq `ceph osd create $ua` + +nb=`ceph osd create $ub` +test $nb -eq `ceph osd create $ub` +test $nb -ne $na + +ceph osd rm $na +ceph osd rm $na +ceph osd rm $nb +ceph osd rm 1000 + +na2=`ceph osd create $ua` + +echo OK + diff --git a/qa/workunits/mon/pg_autoscaler.sh b/qa/workunits/mon/pg_autoscaler.sh new file mode 100755 index 000000000..4cf71a31c --- /dev/null +++ b/qa/workunits/mon/pg_autoscaler.sh @@ -0,0 +1,156 @@ +#!/bin/bash -ex + +NUM_OSDS=$(ceph osd ls | wc -l) +if [ $NUM_OSDS -lt 6 ]; then + echo "test requires at least 6 OSDs" + exit 1 +fi + +NUM_POOLS=$(ceph osd pool ls | wc -l) +if [ $NUM_POOLS -gt 0 ]; then + echo "test requires no preexisting pools" + exit 1 +fi + +function wait_for() { + local sec=$1 + local cmd=$2 + + while true ; do + if bash -c "$cmd" ; then + break + fi + sec=$(( $sec - 1 )) + if [ $sec -eq 0 ]; then + echo failed + return 1 + fi + sleep 1 + done + return 0 +} + +function power2() { echo "x=l($1)/l(2); scale=0; 2^((x+0.5)/1)" | bc -l;} + +function eval_actual_expected_val() { + local actual_value=$1 + local expected_value=$2 + if [[ $actual_value = $expected_value ]] + then + echo "Success: " $actual_value "=" $expected_value + else + echo "Error: " $actual_value "!=" $expected_value + exit 1 + fi +} + +# enable +ceph config set mgr mgr/pg_autoscaler/sleep_interval 60 +ceph mgr module enable pg_autoscaler +# ceph config set global osd_pool_default_pg_autoscale_mode on + +# pg_num_min +ceph osd pool create meta0 16 +ceph osd pool create bulk0 16 --bulk +ceph osd pool create bulk1 16 --bulk +ceph osd pool create bulk2 16 --bulk +ceph osd pool set meta0 pg_autoscale_mode on +ceph osd pool set bulk0 pg_autoscale_mode on +ceph osd pool set bulk1 pg_autoscale_mode on +ceph osd pool set bulk2 pg_autoscale_mode on +# set pool size +ceph osd pool set meta0 size 2 +ceph osd pool set bulk0 size 2 +ceph osd pool set bulk1 size 2 +ceph osd pool set bulk2 size 2 + +# get num pools again since we created more pools +NUM_POOLS=$(ceph osd pool ls | wc -l) + +# get bulk flag of each pool through the command ceph osd pool autoscale-status +BULK_FLAG_1=$(ceph osd pool autoscale-status | grep 'meta0' | grep -o -m 1 'True\|False' || true) +BULK_FLAG_2=$(ceph osd pool autoscale-status | grep 'bulk0' | grep -o -m 1 'True\|False' || true) +BULK_FLAG_3=$(ceph osd pool autoscale-status | grep 'bulk1' | grep -o -m 1 'True\|False' || true) +BULK_FLAG_4=$(ceph osd pool autoscale-status | grep 'bulk2' | grep -o -m 1 'True\|False' || true) + +# evaluate the accuracy of ceph osd pool autoscale-status specifically the `BULK` column + +eval_actual_expected_val $BULK_FLAG_1 'False' +eval_actual_expected_val $BULK_FLAG_2 'True' +eval_actual_expected_val $BULK_FLAG_3 'True' +eval_actual_expected_val $BULK_FLAG_4 'True' + +# This part of this code will now evaluate the accuracy of the autoscaler + +# get pool size +POOL_SIZE_1=$(ceph osd pool get meta0 size| grep -Eo '[0-9]{1,4}') +POOL_SIZE_2=$(ceph osd pool get bulk0 size| grep -Eo '[0-9]{1,4}') +POOL_SIZE_3=$(ceph osd pool get bulk1 size| grep -Eo '[0-9]{1,4}') +POOL_SIZE_4=$(ceph osd pool get bulk2 size| grep -Eo '[0-9]{1,4}') + +# Calculate target pg of each pools +# First Pool is a non-bulk so we do it first. +# Since the Capacity ratio = 0 we first meta pool remains the same pg_num + +TARGET_PG_1=$(ceph osd pool get meta0 pg_num| grep -Eo '[0-9]{1,4}') +PG_LEFT=$NUM_OSDS*100 +NUM_POOLS_LEFT=$NUM_POOLS-1 +# Rest of the pool is bulk and even pools so pretty straight forward +# calculations. +TARGET_PG_2=$(power2 $((($PG_LEFT)/($NUM_POOLS_LEFT)/($POOL_SIZE_2)))) +TARGET_PG_3=$(power2 $((($PG_LEFT)/($NUM_POOLS_LEFT)/($POOL_SIZE_3)))) +TARGET_PG_4=$(power2 $((($PG_LEFT)/($NUM_POOLS_LEFT)/($POOL_SIZE_4)))) + +# evaluate target_pg against pg num of each pools +wait_for 300 "ceph osd pool get meta0 pg_num | grep $TARGET_PG_1" +wait_for 300 "ceph osd pool get bulk0 pg_num | grep $TARGET_PG_2" +wait_for 300 "ceph osd pool get bulk1 pg_num | grep $TARGET_PG_3" +wait_for 300 "ceph osd pool get bulk2 pg_num | grep $TARGET_PG_4" + +# target ratio +ceph osd pool set meta0 target_size_ratio 5 +ceph osd pool set bulk0 target_size_ratio 1 +sleep 60 +APGS=$(ceph osd dump -f json-pretty | jq '.pools[0].pg_num_target') +BPGS=$(ceph osd dump -f json-pretty | jq '.pools[1].pg_num_target') +test $APGS -gt 100 +test $BPGS -gt 10 + +# small ratio change does not change pg_num +ceph osd pool set meta0 target_size_ratio 7 +ceph osd pool set bulk0 target_size_ratio 2 +sleep 60 +APGS2=$(ceph osd dump -f json-pretty | jq '.pools[0].pg_num_target') +BPGS2=$(ceph osd dump -f json-pretty | jq '.pools[1].pg_num_target') +test $APGS -eq $APGS2 +test $BPGS -eq $BPGS2 + +# target_size +ceph osd pool set meta0 target_size_bytes 1000000000000000 +ceph osd pool set bulk0 target_size_bytes 1000000000000000 +ceph osd pool set meta0 target_size_ratio 0 +ceph osd pool set bulk0 target_size_ratio 0 +wait_for 60 "ceph health detail | grep POOL_TARGET_SIZE_BYTES_OVERCOMMITTED" + +ceph osd pool set meta0 target_size_bytes 1000 +ceph osd pool set bulk0 target_size_bytes 1000 +ceph osd pool set meta0 target_size_ratio 1 +wait_for 60 "ceph health detail | grep POOL_HAS_TARGET_SIZE_BYTES_AND_RATIO" + +# test autoscale warn + +ceph osd pool create warn0 1 --autoscale-mode=warn +wait_for 120 "ceph health detail | grep POOL_TOO_FEW_PGS" + +ceph osd pool create warn1 256 --autoscale-mode=warn +wait_for 120 "ceph health detail | grep POOL_TOO_MANY_PGS" + +ceph osd pool rm meta0 meta0 --yes-i-really-really-mean-it +ceph osd pool rm bulk0 bulk0 --yes-i-really-really-mean-it +ceph osd pool rm bulk1 bulk1 --yes-i-really-really-mean-it +ceph osd pool rm bulk2 bulk2 --yes-i-really-really-mean-it +ceph osd pool rm warn0 warn0 --yes-i-really-really-mean-it +ceph osd pool rm warn1 warn1 --yes-i-really-really-mean-it + +echo OK + diff --git a/qa/workunits/mon/ping.py b/qa/workunits/mon/ping.py new file mode 100755 index 000000000..1f6d0a1dd --- /dev/null +++ b/qa/workunits/mon/ping.py @@ -0,0 +1,106 @@ +#!/usr/bin/python3 + +import json +import shlex +import subprocess + + +class UnexpectedReturn(Exception): + def __init__(self, cmd, ret, expected, msg): + if isinstance(cmd, list): + self.cmd = ' '.join(cmd) + else: + assert isinstance(cmd, str), \ + 'cmd needs to be either a list or a str' + self.cmd = cmd + self.cmd = str(self.cmd) + self.ret = int(ret) + self.expected = int(expected) + self.msg = str(msg) + + def __str__(self): + return repr('{c}: expected return {e}, got {r} ({o})'.format( + c=self.cmd, e=self.expected, r=self.ret, o=self.msg)) + + +def call(cmd): + if isinstance(cmd, list): + args = cmd + elif isinstance(cmd, str): + args = shlex.split(cmd) + else: + assert False, 'cmd is not a string/unicode nor a list!' + + print('call: {0}'.format(args)) + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + procout, procerr = proc.communicate(None) + + return proc.returncode, procout, procerr + + +def expect(cmd, expected_ret): + try: + (r, out, err) = call(cmd) + except ValueError as e: + assert False, \ + 'unable to run {c}: {err}'.format(c=repr(cmd), err=str(e)) + + if r != expected_ret: + raise UnexpectedReturn(repr(cmd), r, expected_ret, err) + + return out.decode() if isinstance(out, bytes) else out + + +def get_quorum_status(timeout=300): + cmd = 'ceph quorum_status' + if timeout > 0: + cmd += ' --connect-timeout {0}'.format(timeout) + + out = expect(cmd, 0) + j = json.loads(out) + return j + + +def main(): + quorum_status = get_quorum_status() + mon_names = [mon['name'] for mon in quorum_status['monmap']['mons']] + + print('ping all monitors') + for m in mon_names: + print('ping mon.{0}'.format(m)) + out = expect('ceph ping mon.{0}'.format(m), 0) + reply = json.loads(out) + + assert reply['mon_status']['name'] == m, \ + 'reply obtained from mon.{0}, expected mon.{1}'.format( + reply['mon_status']['name'], m) + + print('test out-of-quorum reply') + for m in mon_names: + print('testing mon.{0}'.format(m)) + expect('ceph daemon mon.{0} quorum exit'.format(m), 0) + + quorum_status = get_quorum_status() + assert m not in quorum_status['quorum_names'], \ + 'mon.{0} was not supposed to be in quorum ({1})'.format( + m, quorum_status['quorum_names']) + + out = expect('ceph ping mon.{0}'.format(m), 0) + reply = json.loads(out) + mon_status = reply['mon_status'] + + assert mon_status['name'] == m, \ + 'reply obtained from mon.{0}, expected mon.{1}'.format( + mon_status['name'], m) + + assert mon_status['state'] == 'electing', \ + 'mon.{0} is in state {1}, expected electing'.format( + m, mon_status['state']) + + expect('ceph daemon mon.{0} quorum enter'.format(m), 0) + + print('OK') + + +if __name__ == '__main__': + main() diff --git a/qa/workunits/mon/pool_ops.sh b/qa/workunits/mon/pool_ops.sh new file mode 100755 index 000000000..23bb3c0be --- /dev/null +++ b/qa/workunits/mon/pool_ops.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +set -ex + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +function get_config_value_or_die() +{ + local pool_name config_opt raw val + + pool_name=$1 + config_opt=$2 + + raw="`$SUDO ceph osd pool get $pool_name $config_opt 2>/dev/null`" + if [[ $? -ne 0 ]]; then + echo "error obtaining config opt '$config_opt' from '$pool_name': $raw" + exit 1 + fi + + raw=`echo $raw | sed -e 's/[{} "]//g'` + val=`echo $raw | cut -f2 -d:` + + echo "$val" + return 0 +} + +function expect_config_value() +{ + local pool_name config_opt expected_val val + pool_name=$1 + config_opt=$2 + expected_val=$3 + + val=$(get_config_value_or_die $pool_name $config_opt) + + if [[ "$val" != "$expected_val" ]]; then + echo "expected '$expected_val', got '$val'" + exit 1 + fi +} + +# pg_num min/max +TEST_POOL=testpool1234 +ceph osd pool create testpool1234 8 --autoscale-mode off +ceph osd pool set $TEST_POOL pg_num_min 2 +ceph osd pool get $TEST_POOL pg_num_min | grep 2 +ceph osd pool set $TEST_POOL pg_num_max 33 +ceph osd pool get $TEST_POOL pg_num_max | grep 33 +expect_false ceph osd pool set $TEST_POOL pg_num_min 9 +expect_false ceph osd pool set $TEST_POOL pg_num_max 7 +expect_false ceph osd pool set $TEST_POOL pg_num 1 +expect_false ceph osd pool set $TEST_POOL pg_num 44 +ceph osd pool set $TEST_POOL pg_num_min 0 +expect_false ceph osd pool get $TEST_POOL pg_num_min +ceph osd pool set $TEST_POOL pg_num_max 0 +expect_false ceph osd pool get $TEST_POOL pg_num_max +ceph osd pool delete $TEST_POOL $TEST_POOL --yes-i-really-really-mean-it + +# note: we need to pass the other args or ceph_argparse.py will take +# 'invalid' that is not replicated|erasure and assume it is the next +# argument, which is a string. +expect_false ceph osd pool create foo 123 123 invalid foo-profile foo-rule + +ceph osd pool create foo 123 123 replicated +ceph osd pool create fooo 123 123 erasure default +ceph osd pool create foooo 123 + +ceph osd pool create foo 123 # idempotent + +ceph osd pool set foo size 1 --yes-i-really-mean-it +expect_config_value "foo" "min_size" 1 +ceph osd pool set foo size 4 +expect_config_value "foo" "min_size" 2 +ceph osd pool set foo size 10 +expect_config_value "foo" "min_size" 5 +expect_false ceph osd pool set foo size 0 +expect_false ceph osd pool set foo size 20 + +# should fail due to safety interlock +expect_false ceph osd pool delete foo +expect_false ceph osd pool delete foo foo +expect_false ceph osd pool delete foo foo --force +expect_false ceph osd pool delete foo fooo --yes-i-really-mean-it +expect_false ceph osd pool delete foo --yes-i-really-mean-it foo + +ceph osd pool delete foooo foooo --yes-i-really-really-mean-it +ceph osd pool delete fooo fooo --yes-i-really-really-mean-it +ceph osd pool delete foo foo --yes-i-really-really-mean-it + +# idempotent +ceph osd pool delete foo foo --yes-i-really-really-mean-it +ceph osd pool delete fooo fooo --yes-i-really-really-mean-it +ceph osd pool delete fooo fooo --yes-i-really-really-mean-it + +# non-existent pool +ceph osd pool delete fuggg fuggg --yes-i-really-really-mean-it + +echo OK + + diff --git a/qa/workunits/mon/rbd_snaps_ops.sh b/qa/workunits/mon/rbd_snaps_ops.sh new file mode 100755 index 000000000..eb88565ea --- /dev/null +++ b/qa/workunits/mon/rbd_snaps_ops.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# attempt to trigger #6047 + + +cmd_no=0 +expect() +{ + cmd_no=$(($cmd_no+1)) + cmd="$1" + expected=$2 + echo "[$cmd_no] $cmd" + eval $cmd + ret=$? + if [[ $ret -ne $expected ]]; then + echo "[$cmd_no] unexpected return '$ret', expected '$expected'" + exit 1 + fi +} + +ceph osd pool delete test test --yes-i-really-really-mean-it || true +expect 'ceph osd pool create test 8 8' 0 +expect 'ceph osd pool application enable test rbd' +expect 'ceph osd pool mksnap test snapshot' 0 +expect 'ceph osd pool rmsnap test snapshot' 0 + +expect 'rbd --pool=test --rbd_validate_pool=false create --size=102400 image' 0 +expect 'rbd --pool=test snap create image@snapshot' 22 + +expect 'ceph osd pool delete test test --yes-i-really-really-mean-it' 0 +expect 'ceph osd pool create test 8 8' 0 +expect 'rbd --pool=test pool init' 0 +expect 'rbd --pool=test create --size=102400 image' 0 +expect 'rbd --pool=test snap create image@snapshot' 0 +expect 'rbd --pool=test snap ls image' 0 +expect 'rbd --pool=test snap rm image@snapshot' 0 + +expect 'ceph osd pool mksnap test snapshot' 22 + +expect 'ceph osd pool delete test test --yes-i-really-really-mean-it' 0 + +# reproduce 7210 and expect it to be fixed +# basically create such a scenario where we end up deleting what used to +# be an unmanaged snapshot from a not-unmanaged pool + +ceph osd pool delete test-foo test-foo --yes-i-really-really-mean-it || true +expect 'ceph osd pool create test-foo 8' 0 +expect 'ceph osd pool application enable test-foo rbd' +expect 'rbd --pool test-foo create --size 1024 image' 0 +expect 'rbd --pool test-foo snap create image@snapshot' 0 + +ceph osd pool delete test-bar test-bar --yes-i-really-really-mean-it || true +expect 'ceph osd pool create test-bar 8' 0 +expect 'ceph osd pool application enable test-bar rbd' +expect 'rados cppool test-foo test-bar --yes-i-really-mean-it' 0 +expect 'rbd --pool test-bar snap rm image@snapshot' 95 +expect 'ceph osd pool delete test-foo test-foo --yes-i-really-really-mean-it' 0 +expect 'ceph osd pool delete test-bar test-bar --yes-i-really-really-mean-it' 0 + + +echo OK diff --git a/qa/workunits/mon/test_config_key_caps.sh b/qa/workunits/mon/test_config_key_caps.sh new file mode 100755 index 000000000..77b4b53b7 --- /dev/null +++ b/qa/workunits/mon/test_config_key_caps.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash + +set -x +set -e + +tmp=$(mktemp -d -p /tmp test_mon_config_key_caps.XXXXX) +entities=() + +function cleanup() +{ + set +e + set +x + if [[ -e $tmp/keyring ]] && [[ -e $tmp/keyring.orig ]]; then + grep '\[.*\..*\]' $tmp/keyring.orig > $tmp/entities.orig + for e in $(grep '\[.*\..*\]' $tmp/keyring | \ + diff $tmp/entities.orig - | \ + sed -n 's/^.*\[\(.*\..*\)\]/\1/p'); + do + ceph auth rm $e 2>&1 >& /dev/null + done + fi + #rm -fr $tmp +} + +trap cleanup 0 # cleanup on exit + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +# for cleanup purposes +ceph auth export -o $tmp/keyring.orig + +k=$tmp/keyring + +# setup a few keys +ceph config-key ls +ceph config-key set daemon-private/osd.123/test-foo +ceph config-key set mgr/test-foo +ceph config-key set device/test-foo +ceph config-key set test/foo + +allow_aa=client.allow_aa +allow_bb=client.allow_bb +allow_cc=client.allow_cc + +mgr_a=mgr.a +mgr_b=mgr.b +osd_a=osd.100 +osd_b=osd.200 + +prefix_aa=client.prefix_aa +prefix_bb=client.prefix_bb +prefix_cc=client.prefix_cc +match_aa=client.match_aa +match_bb=client.match_bb + +fail_aa=client.fail_aa +fail_bb=client.fail_bb +fail_cc=client.fail_cc +fail_dd=client.fail_dd +fail_ee=client.fail_ee +fail_ff=client.fail_ff +fail_gg=client.fail_gg +fail_writes=client.fail_writes + +ceph auth get-or-create $allow_aa mon 'allow *' +ceph auth get-or-create $allow_bb mon 'allow service config-key rwx' +ceph auth get-or-create $allow_cc mon 'allow command "config-key get"' + +ceph auth get-or-create $mgr_a mon 'allow profile mgr' +ceph auth get-or-create $mgr_b mon 'allow profile mgr' +ceph auth get-or-create $osd_a mon 'allow profile osd' +ceph auth get-or-create $osd_b mon 'allow profile osd' + +ceph auth get-or-create $prefix_aa mon \ + "allow command \"config-key get\" with key prefix client/$prefix_aa" + +cap="allow command \"config-key set\" with key prefix client/" +cap="$cap,allow command \"config-key get\" with key prefix client/$prefix_bb" +ceph auth get-or-create $prefix_bb mon "$cap" + +cap="allow command \"config-key get\" with key prefix client/" +cap="$cap, allow command \"config-key set\" with key prefix client/" +cap="$cap, allow command \"config-key ls\"" +ceph auth get-or-create $prefix_cc mon "$cap" + +cap="allow command \"config-key get\" with key=client/$match_aa/foo" +ceph auth get-or-create $match_aa mon "$cap" +cap="allow command \"config-key get\" with key=client/$match_bb/foo" +cap="$cap,allow command \"config-key set\" with key=client/$match_bb/foo" +ceph auth get-or-create $match_bb mon "$cap" + +ceph auth get-or-create $fail_aa mon 'allow rx' +ceph auth get-or-create $fail_bb mon 'allow r,allow w' +ceph auth get-or-create $fail_cc mon 'allow rw' +ceph auth get-or-create $fail_dd mon 'allow rwx' +ceph auth get-or-create $fail_ee mon 'allow profile bootstrap-rgw' +ceph auth get-or-create $fail_ff mon 'allow profile bootstrap-rbd' +# write commands will require rw; wx is not enough +ceph auth get-or-create $fail_gg mon 'allow service config-key wx' +# read commands will only require 'r'; 'rx' should be enough. +ceph auth get-or-create $fail_writes mon 'allow service config-key rx' + +# grab keyring +ceph auth export -o $k + +# keys will all the caps can do whatever +for c in $allow_aa $allow_bb $allow_cc $mgr_a $mgr_b; do + ceph -k $k --name $c config-key get daemon-private/osd.123/test-foo + ceph -k $k --name $c config-key get mgr/test-foo + ceph -k $k --name $c config-key get device/test-foo + ceph -k $k --name $c config-key get test/foo +done + +for c in $osd_a $osd_b; do + ceph -k $k --name $c config-key put daemon-private/$c/test-foo + ceph -k $k --name $c config-key get daemon-private/$c/test-foo + expect_false ceph -k $k --name $c config-key ls + expect_false ceph -k $k --name $c config-key get mgr/test-foo + expect_false ceph -k $k --name $c config-key get device/test-foo + expect_false ceph -k $k --name $c config-key get test/foo +done + +expect_false ceph -k $k --name $osd_a get daemon-private/$osd_b/test-foo +expect_false ceph -k $k --name $osd_b get daemon-private/$osd_a/test-foo + +expect_false ceph -k $k --name $prefix_aa \ + config-key ls +expect_false ceph -k $k --name $prefix_aa \ + config-key get daemon-private/osd.123/test-foo +expect_false ceph -k $k --name $prefix_aa \ + config-key set test/bar +expect_false ceph -k $k --name $prefix_aa \ + config-key set client/$prefix_aa/foo + +# write something so we can read, use a custom entity +ceph -k $k --name $allow_bb config-key set client/$prefix_aa/foo +ceph -k $k --name $prefix_aa config-key get client/$prefix_aa/foo +# check one writes to the other's prefix, the other is able to read +ceph -k $k --name $prefix_bb config-key set client/$prefix_aa/bar +ceph -k $k --name $prefix_aa config-key get client/$prefix_aa/bar + +ceph -k $k --name $prefix_bb config-key set client/$prefix_bb/foo +ceph -k $k --name $prefix_bb config-key get client/$prefix_bb/foo + +expect_false ceph -k $k --name $prefix_bb config-key get client/$prefix_aa/bar +expect_false ceph -k $k --name $prefix_bb config-key ls +expect_false ceph -k $k --name $prefix_bb \ + config-key get daemon-private/osd.123/test-foo +expect_false ceph -k $k --name $prefix_bb config-key get mgr/test-foo +expect_false ceph -k $k --name $prefix_bb config-key get device/test-foo +expect_false ceph -k $k --name $prefix_bb config-key get test/bar +expect_false ceph -k $k --name $prefix_bb config-key set test/bar + +ceph -k $k --name $prefix_cc config-key set client/$match_aa/foo +ceph -k $k --name $prefix_cc config-key set client/$match_bb/foo +ceph -k $k --name $prefix_cc config-key get client/$match_aa/foo +ceph -k $k --name $prefix_cc config-key get client/$match_bb/foo +expect_false ceph -k $k --name $prefix_cc config-key set other/prefix +expect_false ceph -k $k --name $prefix_cc config-key get mgr/test-foo +ceph -k $k --name $prefix_cc config-key ls >& /dev/null + +ceph -k $k --name $match_aa config-key get client/$match_aa/foo +expect_false ceph -k $k --name $match_aa config-key get client/$match_bb/foo +expect_false ceph -k $k --name $match_aa config-key set client/$match_aa/foo +ceph -k $k --name $match_bb config-key get client/$match_bb/foo +ceph -k $k --name $match_bb config-key set client/$match_bb/foo +expect_false ceph -k $k --name $match_bb config-key get client/$match_aa/foo +expect_false ceph -k $k --name $match_bb config-key set client/$match_aa/foo + +keys=(daemon-private/osd.123/test-foo + mgr/test-foo + device/test-foo + test/foo + client/$prefix_aa/foo + client/$prefix_bb/foo + client/$match_aa/foo + client/$match_bb/foo +) +# expect these all to fail accessing config-key +for c in $fail_aa $fail_bb $fail_cc \ + $fail_dd $fail_ee $fail_ff \ + $fail_gg; do + for m in get set; do + for key in ${keys[*]} client/$prefix_aa/foo client/$prefix_bb/foo; do + expect_false ceph -k $k --name $c config-key $m $key + done + done +done + +# fail writes but succeed on reads +expect_false ceph -k $k --name $fail_writes config-key set client/$match_aa/foo +expect_false ceph -k $k --name $fail_writes config-key set test/foo +ceph -k $k --name $fail_writes config-key ls +ceph -k $k --name $fail_writes config-key get client/$match_aa/foo +ceph -k $k --name $fail_writes config-key get daemon-private/osd.123/test-foo + +echo "OK" diff --git a/qa/workunits/mon/test_mon_config_key.py b/qa/workunits/mon/test_mon_config_key.py new file mode 100755 index 000000000..f81804c8a --- /dev/null +++ b/qa/workunits/mon/test_mon_config_key.py @@ -0,0 +1,463 @@ +#!/usr/bin/python3 +# +# test_mon_config_key - Test 'ceph config-key' interface +# +# Copyright (C) 2013 Inktank +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1, as published by the Free Software +# Foundation. See file COPYING. +# +import argparse +import base64 +import errno +import json +import logging +import os +import random +import string +import subprocess +import sys +import time +from typing import List, Dict + +# +# Accepted Environment variables: +# CEPH_TEST_VERBOSE - be more verbose; '1' enables; '0' disables +# CEPH_TEST_DURATION - test duration in seconds +# CEPH_TEST_SEED - seed to be used during the test +# +# Accepted arguments and options (see --help): +# -v, --verbose - be more verbose +# -d, --duration SECS - test duration in seconds +# -s, --seed SEED - seed to be used during the test +# + + +LOG = logging.getLogger(os.path.basename(sys.argv[0].replace('.py', ''))) + +SIZES = [ + (0, 0), + (10, 0), + (25, 0), + (50, 0), + (100, 0), + (1000, 0), + (64 * 1024, 0), + (64 * 1024 + 1, -errno.EFBIG), + (128 * 1024, -errno.EFBIG) +] + +# tests will be randomly selected from the keys here, and the test +# suboperation will be randomly selected from the list in the values +# here. i.e. 'exists/existing' would test that a key the test put into +# the store earlier actually does still exist in the config store, +# and that's a separate test case from 'exists/enoent', which tests +# nonexistence of a key known to not be present. + +OPS = { + 'put': ['existing', 'new'], + 'del': ['existing', 'enoent'], + 'exists': ['existing', 'enoent'], + 'get': ['existing', 'enoent'], + 'list': ['existing', 'enoent'], + 'dump': ['existing', 'enoent'], +} + +CONFIG_PUT: List[str] = [] # list: keys +CONFIG_DEL: List[str] = [] # list: keys +CONFIG_EXISTING: Dict[str, int] = {} # map: key -> size + + +def run_cmd(cmd, expects=0): + full_cmd = ['ceph', 'config-key'] + cmd + + if expects < 0: + expects = -expects + + cmdlog = LOG.getChild('run_cmd') + cmdlog.debug('{fc}'.format(fc=' '.join(full_cmd))) + + proc = subprocess.run(full_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + if proc.returncode != expects: + cmdlog.error(f'cmd > {proc.args}') + cmdlog.error(f'expected return "{expects}" got "{proc.returncode}"') + cmdlog.error('stdout') + cmdlog.error(proc.stdout) + cmdlog.error('stderr') + cmdlog.error(proc.stderr) + + +def gen_data(size, rnd): + chars = string.ascii_letters + string.digits + return ''.join(rnd.choice(chars) for _ in range(size)) + + +def gen_key(rnd): + return gen_data(20, rnd) + + +def gen_tmp_file_path(rnd): + file_name = gen_data(20, rnd) + file_path = os.path.join('/tmp', 'ceph-test.' + file_name) + return file_path + + +def destroy_tmp_file(fpath): + if os.path.exists(fpath) and os.path.isfile(fpath): + os.unlink(fpath) + + +def write_data_file(data, rnd): + file_path = gen_tmp_file_path(rnd) + data_file = open(file_path, 'a+') + data_file.truncate() + data_file.write(data) + data_file.close() + return file_path + + +# end write_data_file + +def choose_random_op(rnd): + op = rnd.choice( + list(OPS.keys()) + ) + sop = rnd.choice(OPS[op]) + return op, sop + + +def parse_args(args): + parser = argparse.ArgumentParser( + description="Test the monitor's 'config-key' API", + ) + parser.add_argument( + '-v', '--verbose', + action='store_true', + help='be more verbose', + ) + parser.add_argument( + '-s', '--seed', + metavar='SEED', + help='use SEED instead of generating it in run-time', + ) + parser.add_argument( + '-d', '--duration', + metavar='SECS', + help='run test for SECS seconds (default: 300)', + ) + parser.set_defaults( + seed=None, + duration=300, + verbose=False, + ) + return parser.parse_args(args) + + +def main(): + args = parse_args(sys.argv[1:]) + + verbose = args.verbose + if os.environ.get('CEPH_TEST_VERBOSE') is not None: + verbose = (os.environ.get('CEPH_TEST_VERBOSE') == '1') + + duration = int(os.environ.get('CEPH_TEST_DURATION', args.duration)) + seed = os.environ.get('CEPH_TEST_SEED', args.seed) + seed = int(time.time()) if seed is None else int(seed) + + rnd = random.Random() + rnd.seed(seed) + + loglevel = logging.INFO + if verbose: + loglevel = logging.DEBUG + + logging.basicConfig(level=loglevel) + + LOG.info('seed: {s}'.format(s=seed)) + + start = time.time() + + while (time.time() - start) < duration: + (op, sop) = choose_random_op(rnd) + + LOG.info('{o}({s})'.format(o=op, s=sop)) + op_log = LOG.getChild('{o}({s})'.format(o=op, s=sop)) + + if op == 'put': + via_file = (rnd.uniform(0, 100) < 50.0) + + expected = 0 + cmd = ['put'] + key = None + + if sop == 'existing': + if len(CONFIG_EXISTING) == 0: + op_log.debug('no existing keys; continue') + continue + key = rnd.choice(CONFIG_PUT) + assert key in CONFIG_EXISTING, \ + "key '{k_}' not in CONFIG_EXISTING".format(k_=key) + + expected = 0 # the store just overrides the value if the key exists + # end if sop == 'existing' + elif sop == 'new': + for x in range(0, 10): + key = gen_key(rnd) + if key not in CONFIG_EXISTING: + break + key = None + if key is None: + op_log.error('unable to generate an unique key -- try again later.') + continue + + assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \ + 'key {k} was not supposed to exist!'.format(k=key) + + assert key is not None, \ + 'key must be != None' + + cmd += [key] + + (size, error) = rnd.choice(SIZES) + if size > 25: + via_file = True + + data = gen_data(size, rnd) + + if error == 0: # only add if we expect the put to be successful + if sop == 'new': + CONFIG_PUT.append(key) + CONFIG_EXISTING[key] = size + expected = error + + if via_file: + data_file = write_data_file(data, rnd) + cmd += ['-i', data_file] + else: + cmd += [data] + + op_log.debug('size: {sz}, via: {v}'.format( + sz=size, + v='file: {f}'.format(f=data_file) if via_file == True else 'cli') + ) + run_cmd(cmd, expects=expected) + if via_file: + destroy_tmp_file(data_file) + continue + + elif op == 'del': + expected = 0 + cmd = ['del'] + key = None + + if sop == 'existing': + if len(CONFIG_EXISTING) == 0: + op_log.debug('no existing keys; continue') + continue + key = rnd.choice(CONFIG_PUT) + assert key in CONFIG_EXISTING, \ + "key '{k_}' not in CONFIG_EXISTING".format(k_=key) + + if sop == 'enoent': + for x in range(0, 10): + key = base64.b64encode(os.urandom(20)).decode() + if key not in CONFIG_EXISTING: + break + key = None + if key is None: + op_log.error('unable to generate an unique key -- try again later.') + continue + assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \ + 'key {k} was not supposed to exist!'.format(k=key) + expected = 0 # deleting a non-existent key succeeds + + assert key is not None, \ + 'key must be != None' + + cmd += [key] + op_log.debug('key: {k}'.format(k=key)) + run_cmd(cmd, expects=expected) + if sop == 'existing': + CONFIG_DEL.append(key) + CONFIG_PUT.remove(key) + del CONFIG_EXISTING[key] + continue + + elif op == 'exists': + expected = 0 + cmd = ['exists'] + key = None + + if sop == 'existing': + if len(CONFIG_EXISTING) == 0: + op_log.debug('no existing keys; continue') + continue + key = rnd.choice(CONFIG_PUT) + assert key in CONFIG_EXISTING, \ + "key '{k_}' not in CONFIG_EXISTING".format(k_=key) + + if sop == 'enoent': + for x in range(0, 10): + key = base64.b64encode(os.urandom(20)).decode() + if key not in CONFIG_EXISTING: + break + key = None + if key is None: + op_log.error('unable to generate an unique key -- try again later.') + continue + assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \ + 'key {k} was not supposed to exist!'.format(k=key) + expected = -errno.ENOENT + + assert key is not None, \ + 'key must be != None' + + cmd += [key] + op_log.debug('key: {k}'.format(k=key)) + run_cmd(cmd, expects=expected) + continue + + elif op == 'get': + expected = 0 + cmd = ['get'] + key = None + + if sop == 'existing': + if len(CONFIG_EXISTING) == 0: + op_log.debug('no existing keys; continue') + continue + key = rnd.choice(CONFIG_PUT) + assert key in CONFIG_EXISTING, \ + "key '{k_}' not in CONFIG_EXISTING".format(k_=key) + + if sop == 'enoent': + for x in range(0, 10): + key = base64.b64encode(os.urandom(20)).decode() + if key not in CONFIG_EXISTING: + break + key = None + if key is None: + op_log.error('unable to generate an unique key -- try again later.') + continue + assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \ + 'key {k} was not supposed to exist!'.format(k=key) + expected = -errno.ENOENT + + assert key is not None, \ + 'key must be != None' + + file_path = gen_tmp_file_path(rnd) + cmd += [key, '-o', file_path] + op_log.debug('key: {k}'.format(k=key)) + run_cmd(cmd, expects=expected) + if sop == 'existing': + try: + temp_file = open(file_path, 'r+') + except IOError as err: + if err.errno == errno.ENOENT: + assert CONFIG_EXISTING[key] == 0, \ + "error opening '{fp}': {e}".format(fp=file_path, e=err) + continue + else: + assert False, \ + 'some error occurred: {e}'.format(e=err) + cnt = 0 + while True: + read_data = temp_file.read() + if read_data == '': + break + cnt += len(read_data) + assert cnt == CONFIG_EXISTING[key], \ + "wrong size from store for key '{k}': {sz}, expected {es}".format( + k=key, sz=cnt, es=CONFIG_EXISTING[key]) + destroy_tmp_file(file_path) + continue + + elif op == 'list' or op == 'dump': + expected = 0 + cmd = [op] + key = None + + if sop == 'existing': + if len(CONFIG_EXISTING) == 0: + op_log.debug('no existing keys; continue') + continue + key = rnd.choice(CONFIG_PUT) + assert key in CONFIG_EXISTING, \ + "key '{k_}' not in CONFIG_EXISTING".format(k_=key) + + if sop == 'enoent': + for x in range(0, 10): + key = base64.b64encode(os.urandom(20)).decode() + if key not in CONFIG_EXISTING: + break + key = None + if key is None: + op_log.error('unable to generate an unique key -- try again later.') + continue + assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \ + 'key {k} was not supposed to exist!'.format(k=key) + + assert key is not None, \ + 'key must be != None' + + file_path = gen_tmp_file_path(rnd) + cmd += ['-o', file_path] + op_log.debug('key: {k}'.format(k=key)) + run_cmd(cmd, expects=expected) + try: + temp_file = open(file_path, 'r+') + except IOError as err: + if err.errno == errno.ENOENT: + assert CONFIG_EXISTING[key] == 0, \ + "error opening '{fp}': {e}".format(fp=file_path, e=err) + continue + else: + assert False, \ + 'some error occurred: {e}'.format(e=err) + cnt = 0 + try: + read_data = json.load(temp_file) + except ValueError: + temp_file.seek(0) + assert False, "{op} output was not valid JSON:\n{filedata}".format( + op=op, filedata=temp_file.readlines()) + + if sop == 'existing': + assert key in read_data, "key '{k}' not found in list/dump output".format(k=key) + if op == 'dump': + cnt = len(read_data[key]) + assert cnt == CONFIG_EXISTING[key], \ + "wrong size from list for key '{k}': {sz}, expected {es}".format( + k=key, sz=cnt, es=CONFIG_EXISTING[key]) + elif sop == 'enoent': + assert key not in read_data, "key '{k}' found in list/dump output".format(k=key) + destroy_tmp_file(file_path) + continue + else: + assert False, 'unknown op {o}'.format(o=op) + + # check if all keys in 'CONFIG_PUT' exist and + # if all keys on 'CONFIG_DEL' don't. + # but first however, remove all keys in CONFIG_PUT that might + # be in CONFIG_DEL as well. + config_put_set = set(CONFIG_PUT) + config_del_set = set(CONFIG_DEL).difference(config_put_set) + + LOG.info('perform sanity checks on store') + + for k in config_put_set: + LOG.getChild('check(puts)').debug('key: {k_}'.format(k_=k)) + run_cmd(['exists', k], expects=0) + for k in config_del_set: + LOG.getChild('check(dels)').debug('key: {k_}'.format(k_=k)) + run_cmd(['exists', k], expects=-errno.ENOENT) + + +if __name__ == "__main__": + main() diff --git a/qa/workunits/mon/test_mon_osdmap_prune.sh b/qa/workunits/mon/test_mon_osdmap_prune.sh new file mode 100755 index 000000000..9cdd72179 --- /dev/null +++ b/qa/workunits/mon/test_mon_osdmap_prune.sh @@ -0,0 +1,205 @@ +#!/bin/bash + +. $(dirname $0)/../../standalone/ceph-helpers.sh + +set -x + +function wait_for_osdmap_manifest() { + + local what=${1:-"true"} + + local -a delays=($(get_timeout_delays $TIMEOUT .1)) + local -i loop=0 + + for ((i=0; i < ${#delays[*]}; ++i)); do + has_manifest=$(ceph report | jq 'has("osdmap_manifest")') + if [[ "$has_manifest" == "$what" ]]; then + return 0 + fi + + sleep ${delays[$i]} + done + + echo "osdmap_manifest never outputted on report" + ceph report + return 1 +} + +function wait_for_trim() { + + local -i epoch=$1 + local -a delays=($(get_timeout_delays $TIMEOUT .1)) + local -i loop=0 + + for ((i=0; i < ${#delays[*]}; ++i)); do + fc=$(ceph report | jq '.osdmap_first_committed') + if [[ $fc -eq $epoch ]]; then + return 0 + fi + sleep ${delays[$i]} + done + + echo "never trimmed up to epoch $epoch" + ceph report + return 1 +} + +function test_osdmap() { + + local epoch=$1 + local ret=0 + + tmp_map=$(mktemp) + ceph osd getmap $epoch -o $tmp_map || return 1 + if ! osdmaptool --print $tmp_map | grep "epoch $epoch" ; then + echo "ERROR: failed processing osdmap epoch $epoch" + ret=1 + fi + rm $tmp_map + return $ret +} + +function generate_osdmaps() { + + local -i num=$1 + + cmds=( set unset ) + for ((i=0; i < num; ++i)); do + ceph osd ${cmds[$((i%2))]} noup || return 1 + done + return 0 +} + +function test_mon_osdmap_prune() { + + create_pool foo 32 + wait_for_clean || return 1 + + ceph config set mon mon_debug_block_osdmap_trim true || return 1 + + generate_osdmaps 500 || return 1 + + report="$(ceph report)" + fc=$(jq '.osdmap_first_committed' <<< $report) + lc=$(jq '.osdmap_last_committed' <<< $report) + + [[ $((lc-fc)) -ge 500 ]] || return 1 + + wait_for_osdmap_manifest || return 1 + + manifest="$(ceph report | jq '.osdmap_manifest')" + + first_pinned=$(jq '.first_pinned' <<< $manifest) + last_pinned=$(jq '.last_pinned' <<< $manifest) + pinned_maps=( $(jq '.pinned_maps[]' <<< $manifest) ) + + # validate pinned maps list + [[ $first_pinned -eq ${pinned_maps[0]} ]] || return 1 + [[ $last_pinned -eq ${pinned_maps[-1]} ]] || return 1 + + # validate pinned maps range + [[ $first_pinned -lt $last_pinned ]] || return 1 + [[ $last_pinned -lt $lc ]] || return 1 + [[ $first_pinned -eq $fc ]] || return 1 + + # ensure all the maps are available, and work as expected + # this can take a while... + + for ((i=$first_pinned; i <= $last_pinned; ++i)); do + test_osdmap $i || return 1 + done + + # update pinned maps state: + # the monitor may have pruned & pinned additional maps since we last + # assessed state, given it's an iterative process. + # + manifest="$(ceph report | jq '.osdmap_manifest')" + first_pinned=$(jq '.first_pinned' <<< $manifest) + last_pinned=$(jq '.last_pinned' <<< $manifest) + pinned_maps=( $(jq '.pinned_maps[]' <<< $manifest) ) + + # test trimming maps + # + # we're going to perform the following tests: + # + # 1. force trim to a pinned map + # 2. force trim to a pinned map's previous epoch + # 3. trim all maps except the last 200 or so. + # + + # 1. force trim to a pinned map + # + [[ ${#pinned_maps[@]} -gt 10 ]] || return 1 + + trim_to=${pinned_maps[1]} + ceph config set mon mon_osd_force_trim_to $trim_to + ceph config set mon mon_min_osdmap_epochs 100 + ceph config set mon paxos_service_trim_min 1 + ceph config set mon mon_debug_block_osdmap_trim false + + # generate an epoch so we get to trim maps + ceph osd set noup + ceph osd unset noup + + wait_for_trim $trim_to || return 1 + + report="$(ceph report)" + fc=$(jq '.osdmap_first_committed' <<< $report) + [[ $fc -eq $trim_to ]] || return 1 + + old_first_pinned=$first_pinned + old_last_pinned=$last_pinned + first_pinned=$(jq '.osdmap_manifest.first_pinned' <<< $report) + last_pinned=$(jq '.osdmap_manifest.last_pinned' <<< $report) + [[ $first_pinned -eq $trim_to ]] || return 1 + [[ $first_pinned -gt $old_first_pinned ]] || return 1 + [[ $last_pinned -gt $old_first_pinned ]] || return 1 + + test_osdmap $trim_to || return 1 + test_osdmap $(( trim_to+1 )) || return 1 + + pinned_maps=( $(jq '.osdmap_manifest.pinned_maps[]' <<< $report) ) + + # 2. force trim to a pinned map's previous epoch + # + [[ ${#pinned_maps[@]} -gt 2 ]] || return 1 + trim_to=$(( ${pinned_maps[1]} - 1)) + ceph config set mon mon_osd_force_trim_to $trim_to + + # generate an epoch so we get to trim maps + ceph osd set noup + ceph osd unset noup + + wait_for_trim $trim_to || return 1 + + report="$(ceph report)" + fc=$(jq '.osdmap_first_committed' <<< $report) + [[ $fc -eq $trim_to ]] || return 1 + + old_first_pinned=$first_pinned + old_last_pinned=$last_pinned + first_pinned=$(jq '.osdmap_manifest.first_pinned' <<< $report) + last_pinned=$(jq '.osdmap_manifest.last_pinned' <<< $report) + pinned_maps=( $(jq '.osdmap_manifest.pinned_maps[]' <<< $report) ) + [[ $first_pinned -eq $trim_to ]] || return 1 + [[ ${pinned_maps[1]} -eq $(( trim_to+1)) ]] || return 1 + + test_osdmap $first_pinned || return 1 + test_osdmap $(( first_pinned + 1 )) || return 1 + + # 3. trim everything + # + ceph config set mon mon_osd_force_trim_to 0 + + # generate an epoch so we get to trim maps + ceph osd set noup + ceph osd unset noup + + wait_for_osdmap_manifest "false" || return 1 + + return 0 +} + +test_mon_osdmap_prune || exit 1 + +echo "OK" diff --git a/qa/workunits/mon/test_noautoscale_flag.sh b/qa/workunits/mon/test_noautoscale_flag.sh new file mode 100755 index 000000000..e1a45a4d8 --- /dev/null +++ b/qa/workunits/mon/test_noautoscale_flag.sh @@ -0,0 +1,104 @@ +#!/bin/bash -ex + +unset CEPH_CLI_TEST_DUP_COMMAND + +NUM_POOLS=$(ceph osd pool ls | wc -l) + +if [ "$NUM_POOLS" -gt 0 ]; then + echo "test requires no preexisting pools" + exit 1 +fi + +ceph osd pool set noautoscale + +ceph osd pool create pool_a + +echo 'pool_a autoscale_mode:' $(ceph osd pool autoscale-status | grep pool_a | grep -o -m 1 'on\|off') + +NUM_POOLS=$[NUM_POOLS+1] + +sleep 2 + +# Count the number of Pools with AUTOSCALE `off` + +RESULT1=$(ceph osd pool autoscale-status | grep -oe 'off' | wc -l) + +# number of Pools with AUTOSCALE `off` should equal to $NUM_POOLS + +test "$RESULT1" -eq "$NUM_POOLS" + +ceph osd pool unset noautoscale + +echo $(ceph osd pool get noautoscale) + + +ceph osd pool create pool_b + +echo 'pool_a autoscale_mode:' $(ceph osd pool autoscale-status | grep pool_a | grep -o -m 1 'on\|off') + +echo 'pool_b autoscale_mode:' $(ceph osd pool autoscale-status | grep pool_b | grep -o -m 1 'on\|off') + + +NUM_POOLS=$[NUM_POOLS+1] + +sleep 2 + +# Count the number of Pools with AUTOSCALE `on` + +RESULT2=$(ceph osd pool autoscale-status | grep -oe 'on' | wc -l) + +# number of Pools with AUTOSCALE `on` should equal to 3 + +test "$RESULT2" -eq "$[NUM_POOLS-1]" + +ceph osd pool set noautoscale + +ceph osd pool create pool_c + +echo 'pool_a autoscale_mode:' $(ceph osd pool autoscale-status | grep pool_a | grep -o -m 1 'on\|off') + +echo 'pool_b autoscale_mode:' $(ceph osd pool autoscale-status | grep pool_b | grep -o -m 1 'on\|off') + +echo 'pool_c autoscale_mode:' $(ceph osd pool autoscale-status | grep pool_c | grep -o -m 1 'on\|off') + + +NUM_POOLS=$[NUM_POOLS+1] + +sleep 2 + +# Count the number of Pools with AUTOSCALE `off` + +RESULT3=$(ceph osd pool autoscale-status | grep -oe 'off' | wc -l) + +# number of Pools with AUTOSCALE `off` should equal to 4 + +test "$RESULT3" -eq "$NUM_POOLS" + +# Now we test if we retain individual pool state of autoscale mode +# when we set and unset the noautoscale flag. + +ceph osd pool unset noautoscale + +ceph osd pool set pool_a pg_autoscale_mode on + +ceph osd pool set pool_b pg_autoscale_mode warn + +ceph osd pool set noautoscale + +ceph osd pool unset noautoscale + +RESULT4=$(ceph osd pool autoscale-status | grep pool_a | grep -o -m 1 'on\|off\|warn') +RESULT5=$(ceph osd pool autoscale-status | grep pool_b | grep -o -m 1 'on\|off\|warn') +RESULT6=$(ceph osd pool autoscale-status | grep pool_c | grep -o -m 1 'on\|off\|warn') + +test "$RESULT4" == 'on' +test "$RESULT5" == 'warn' +test "$RESULT6" == 'off' + +ceph osd pool rm pool_a pool_a --yes-i-really-really-mean-it + +ceph osd pool rm pool_b pool_b --yes-i-really-really-mean-it + +ceph osd pool rm pool_c pool_c --yes-i-really-really-mean-it + +echo OK diff --git a/qa/workunits/objectstore/test_fuse.sh b/qa/workunits/objectstore/test_fuse.sh new file mode 100755 index 000000000..f1dcbd04f --- /dev/null +++ b/qa/workunits/objectstore/test_fuse.sh @@ -0,0 +1,129 @@ +#!/bin/sh -ex + +if ! id -u | grep -q '^0$'; then + echo "not root, re-running self via sudo" + sudo PATH=$PATH TYPE=$TYPE $0 + exit 0 +fi + +expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +COT=ceph-objectstore-tool +DATA=store_test_fuse_dir +[ -z "$TYPE" ] && TYPE=bluestore +MNT=store_test_fuse_mnt + +rm -rf $DATA +mkdir -p $DATA + +test -d $MNT && fusermount -u $MNT || true +rmdir $MNT || true +mkdir $MNT + +export CEPH_ARGS=--enable_experimental_unrecoverable_data_corrupting_features=bluestore + +$COT --no-mon-config --op mkfs --data-path $DATA --type $TYPE +$COT --no-mon-config --op fuse --data-path $DATA --mountpoint $MNT & + +while ! test -e $MNT/type ; do + echo waiting for $MNT/type to appear + sleep 1 +done + +umask 0 + +grep $TYPE $MNT/type + +# create collection +mkdir $MNT/meta +test -e $MNT/meta/bitwise_hash_start +test -d $MNT/meta/all +test -d $MNT/meta/by_bitwise_hash + +# create object +mkdir $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0# +test -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data +test -d $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr +test -d $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap +test -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/bitwise_hash +test -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap_header + +# omap header +echo omap header > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap_header +grep -q omap $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap_header + +# omap +echo value a > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keya +echo value b > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keyb +ls $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap | grep -c key | grep -q 2 +grep 'value a' $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keya +grep 'value b' $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keyb +rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keya +test ! -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keya +rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keyb +test ! -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keyb + +# attr +echo value a > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keya +echo value b > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keyb +ls $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr | grep -c key | grep -q 2 +grep 'value a' $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keya +grep 'value b' $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keyb +rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keya +test ! -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keya +rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keyb +test ! -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keyb + +# data +test ! -s $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data +echo asdfasdfasdf > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data +test -s $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data +grep -q asdfasdfasdf $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data +truncate --size 4 $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data +stat --format=%s $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data | grep -q ^4$ +expect_false grep -q asdfasdfasdf $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data +rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data +test ! -s $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data + + +# create pg collection +mkdir --mode 0003 $MNT/0.0_head +grep -q 00000000 $MNT/0.0_head/bitwise_hash_start +if [ "$TYPE" = "bluestore" ]; then + cat $MNT/0.0_head/bitwise_hash_bits + grep -q 3 $MNT/0.0_head/bitwise_hash_bits + grep -q 1fffffff $MNT/0.0_head/bitwise_hash_end +fi +test -d $MNT/0.0_head/all + +mkdir --mode 0003 $MNT/0.1_head +grep -q 80000000 $MNT/0.1_head/bitwise_hash_start +if [ "$TYPE" = "bluestore" ]; then + grep -q 3 $MNT/0.1_head/bitwise_hash_bits + grep -q 9fffffff $MNT/0.1_head/bitwise_hash_end +fi + +# create pg object +mkdir $MNT/0.0_head/all/#0:00000000::::head#/ +mkdir $MNT/0.0_head/all/#0:10000000:::foo:head#/ + +# verify pg bounds check +if [ "$TYPE" = "bluestore" ]; then + expect_false mkdir $MNT/0.0_head/all/#0:20000000:::bar:head#/ +fi + +# remove a collection +expect_false rmdir $MNT/0.0_head +rmdir $MNT/0.0_head/all/#0:10000000:::foo:head#/ +rmdir $MNT/0.0_head/all/#0:00000000::::head#/ +rmdir $MNT/0.0_head +rmdir $MNT/0.1_head + +fusermount -u $MNT +wait + +echo OK diff --git a/qa/workunits/osdc/stress_objectcacher.sh b/qa/workunits/osdc/stress_objectcacher.sh new file mode 100755 index 000000000..67baadc33 --- /dev/null +++ b/qa/workunits/osdc/stress_objectcacher.sh @@ -0,0 +1,28 @@ +#!/bin/sh -ex + +for i in $(seq 1 10) +do + for DELAY in 0 1000 + do + for OPS in 1000 10000 + do + for OBJECTS in 10 50 100 + do + for READS in 0.90 0.50 0.10 + do + for OP_SIZE in 4096 131072 1048576 + do + for MAX_DIRTY in 0 25165824 + do + ceph_test_objectcacher_stress --ops $OPS --percent-read $READS --delay-ns $DELAY --objects $OBJECTS --max-op-size $OP_SIZE --client-oc-max-dirty $MAX_DIRTY --stress-test > /dev/null 2>&1 + done + done + done + done + done + done +done + +ceph_test_objectcacher_stress --correctness-test > /dev/null 2>&1 + +echo OK diff --git a/qa/workunits/post-file.sh b/qa/workunits/post-file.sh new file mode 100755 index 000000000..120fb2634 --- /dev/null +++ b/qa/workunits/post-file.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -ex + +what="$1" +[ -z "$what" ] && what=/etc/udev/rules.d +sudo ceph-post-file -d ceph-test-workunit $what + +echo OK diff --git a/qa/workunits/rados/clone.sh b/qa/workunits/rados/clone.sh new file mode 100755 index 000000000..281e89f71 --- /dev/null +++ b/qa/workunits/rados/clone.sh @@ -0,0 +1,13 @@ +#!/bin/sh -x + +set -e + +rados -p data rm foo || true +rados -p data put foo.tmp /etc/passwd --object-locator foo +rados -p data clonedata foo.tmp foo --object-locator foo +rados -p data get foo /tmp/foo +cmp /tmp/foo /etc/passwd +rados -p data rm foo.tmp --object-locator foo +rados -p data rm foo + +echo OK
\ No newline at end of file diff --git a/qa/workunits/rados/load-gen-big.sh b/qa/workunits/rados/load-gen-big.sh new file mode 100755 index 000000000..6715658ec --- /dev/null +++ b/qa/workunits/rados/load-gen-big.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +rados -p rbd load-gen \ + --num-objects 10240 \ + --min-object-size 1048576 \ + --max-object-size 25600000 \ + --max-ops 1024 \ + --max-backlog 1024 \ + --read-percent 50 \ + --run-length 1200 diff --git a/qa/workunits/rados/load-gen-mix-small-long.sh b/qa/workunits/rados/load-gen-mix-small-long.sh new file mode 100755 index 000000000..593bad51d --- /dev/null +++ b/qa/workunits/rados/load-gen-mix-small-long.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +rados -p rbd load-gen \ + --num-objects 1024 \ + --min-object-size 1 \ + --max-object-size 1048576 \ + --max-ops 128 \ + --max-backlog 128 \ + --read-percent 50 \ + --run-length 1800 diff --git a/qa/workunits/rados/load-gen-mix-small.sh b/qa/workunits/rados/load-gen-mix-small.sh new file mode 100755 index 000000000..02db77bd0 --- /dev/null +++ b/qa/workunits/rados/load-gen-mix-small.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +rados -p rbd load-gen \ + --num-objects 1024 \ + --min-object-size 1 \ + --max-object-size 1048576 \ + --max-ops 128 \ + --max-backlog 128 \ + --read-percent 50 \ + --run-length 600 diff --git a/qa/workunits/rados/load-gen-mix.sh b/qa/workunits/rados/load-gen-mix.sh new file mode 100755 index 000000000..ad3b4be84 --- /dev/null +++ b/qa/workunits/rados/load-gen-mix.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +rados -p rbd load-gen \ + --num-objects 10240 \ + --min-object-size 1 \ + --max-object-size 1048576 \ + --max-ops 128 \ + --max-backlog 128 \ + --read-percent 50 \ + --run-length 600 diff --git a/qa/workunits/rados/load-gen-mostlyread.sh b/qa/workunits/rados/load-gen-mostlyread.sh new file mode 100755 index 000000000..236f82dd4 --- /dev/null +++ b/qa/workunits/rados/load-gen-mostlyread.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +rados -p rbd load-gen \ + --num-objects 51200 \ + --min-object-size 1 \ + --max-object-size 1048576 \ + --max-ops 128 \ + --max-backlog 128 \ + --read-percent 90 \ + --run-length 600 diff --git a/qa/workunits/rados/stress_watch.sh b/qa/workunits/rados/stress_watch.sh new file mode 100755 index 000000000..49f144bbc --- /dev/null +++ b/qa/workunits/rados/stress_watch.sh @@ -0,0 +1,7 @@ +#!/bin/sh -e + +ceph_test_stress_watch +ceph_multi_stress_watch rep reppool repobj +ceph_multi_stress_watch ec ecpool ecobj + +exit 0 diff --git a/qa/workunits/rados/test.sh b/qa/workunits/rados/test.sh new file mode 100755 index 000000000..daa25fe4d --- /dev/null +++ b/qa/workunits/rados/test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -ex + +parallel=1 +[ "$1" = "--serial" ] && parallel=0 + +color="" +[ -t 1 ] && color="--gtest_color=yes" + +function cleanup() { + pkill -P $$ || true +} +trap cleanup EXIT ERR HUP INT QUIT + +declare -A pids + +for f in \ + api_aio api_aio_pp \ + api_io api_io_pp \ + api_asio api_list \ + api_lock api_lock_pp \ + api_misc api_misc_pp \ + api_tier_pp \ + api_pool \ + api_snapshots api_snapshots_pp \ + api_stat api_stat_pp \ + api_watch_notify api_watch_notify_pp \ + api_cmd api_cmd_pp \ + api_service api_service_pp \ + api_c_write_operations \ + api_c_read_operations \ + api_cls_remote_reads \ + list_parallel \ + open_pools_parallel \ + delete_pools_parallel +do + if [ $parallel -eq 1 ]; then + r=`printf '%25s' $f` + ff=`echo $f | awk '{print $1}'` + bash -o pipefail -exc "ceph_test_rados_$f $color 2>&1 | tee ceph_test_rados_$ff.log | sed \"s/^/$r: /\"" & + pid=$! + echo "test $f on pid $pid" + pids[$f]=$pid + else + ceph_test_rados_$f + fi +done + +ret=0 +if [ $parallel -eq 1 ]; then +for t in "${!pids[@]}" +do + pid=${pids[$t]} + if ! wait $pid + then + echo "error in $t ($pid)" + ret=1 + fi +done +fi + +exit $ret diff --git a/qa/workunits/rados/test_alloc_hint.sh b/qa/workunits/rados/test_alloc_hint.sh new file mode 100755 index 000000000..535201ca3 --- /dev/null +++ b/qa/workunits/rados/test_alloc_hint.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash + +set -ex +shopt -s nullglob # fns glob expansion in expect_alloc_hint_eq() + +# +# Helpers +# + +function get_xml_val() { + local xml="$1" + local tag="$2" + + local regex=".*<${tag}>(.*)</${tag}>.*" + if [[ ! "${xml}" =~ ${regex} ]]; then + echo "'${xml}' xml doesn't match '${tag}' tag regex" >&2 + return 2 + fi + + echo "${BASH_REMATCH[1]}" +} + +function get_conf_val() { + set -e + + local entity="$1" + local option="$2" + + local val + val="$(sudo ceph daemon "${entity}" config get --format=xml "${option}")" + val="$(get_xml_val "${val}" "${option}")" + + echo "${val}" +} + +function setup_osd_data() { + for (( i = 0 ; i < "${NUM_OSDS}" ; i++ )); do + OSD_DATA[i]="$(get_conf_val "osd.$i" "osd_data")" + done +} + +function setup_pgid() { + local poolname="$1" + local objname="$2" + + local pgid + pgid="$(ceph osd map "${poolname}" "${objname}" --format=xml)" + pgid="$(get_xml_val "${pgid}" "pgid")" + + PGID="${pgid}" +} + +function expect_alloc_hint_eq() { + export CEPH_ARGS="--osd-objectstore=filestore" + local expected_extsize="$1" + + for (( i = 0 ; i < "${NUM_OSDS}" ; i++ )); do + # Make sure that stuff is flushed from the journal to the store + # by the time we get to it, as we prod the actual files and not + # the journal. + sudo ceph daemon "osd.${i}" "flush_journal" + + # e.g., .../25.6_head/foo__head_7FC1F406__19 + # .../26.bs1_head/bar__head_EFE6384B__1a_ffffffffffffffff_1 + local fns=$(sudo sh -c "ls ${OSD_DATA[i]}/current/${PGID}*_head/${OBJ}_*") + local count="${#fns[@]}" + if [ "${count}" -ne 1 ]; then + echo "bad fns count: ${count}" >&2 + return 2 + fi + + local extsize + extsize="$(sudo xfs_io -c extsize "${fns[0]}")" + local extsize_regex="^\[(.*)\] ${fns[0]}$" + if [[ ! "${extsize}" =~ ${extsize_regex} ]]; then + echo "extsize doesn't match extsize_regex: ${extsize}" >&2 + return 2 + fi + extsize="${BASH_REMATCH[1]}" + + if [ "${extsize}" -ne "${expected_extsize}" ]; then + echo "FAIL: alloc_hint: actual ${extsize}, expected ${expected_extsize}" >&2 + return 1 + fi + done +} + +# +# Global setup +# + +EC_K="2" +EC_M="1" +NUM_OSDS="$((EC_K + EC_M))" + +NUM_PG="12" +NUM_PGP="${NUM_PG}" + +LOW_CAP="$(get_conf_val "osd.0" "filestore_max_alloc_hint_size")" +HIGH_CAP="$((LOW_CAP * 10))" # 10M, assuming 1M default cap +SMALL_HINT="$((LOW_CAP / 4))" # 256K, assuming 1M default cap +BIG_HINT="$((LOW_CAP * 6))" # 6M, assuming 1M default cap + +setup_osd_data + +# +# ReplicatedBackend tests +# + +POOL="alloc_hint-rep" +ceph osd pool create "${POOL}" "${NUM_PG}" +ceph osd pool set "${POOL}" size "${NUM_OSDS}" --yes-i-really-mean-it +ceph osd pool application enable "${POOL}" rados + +OBJ="foo" +setup_pgid "${POOL}" "${OBJ}" +rados -p "${POOL}" create "${OBJ}" + +# Empty object, SMALL_HINT - expect SMALL_HINT +rados -p "${POOL}" set-alloc-hint "${OBJ}" "${SMALL_HINT}" "${SMALL_HINT}" +expect_alloc_hint_eq "${SMALL_HINT}" + +# Try changing to BIG_HINT (1) - expect LOW_CAP (BIG_HINT > LOW_CAP) +rados -p "${POOL}" set-alloc-hint "${OBJ}" "${BIG_HINT}" "${BIG_HINT}" +expect_alloc_hint_eq "${LOW_CAP}" + +# Bump the cap to HIGH_CAP +ceph tell 'osd.*' injectargs "--filestore_max_alloc_hint_size ${HIGH_CAP}" + +# Try changing to BIG_HINT (2) - expect BIG_HINT (BIG_HINT < HIGH_CAP) +rados -p "${POOL}" set-alloc-hint "${OBJ}" "${BIG_HINT}" "${BIG_HINT}" +expect_alloc_hint_eq "${BIG_HINT}" + +ceph tell 'osd.*' injectargs "--filestore_max_alloc_hint_size ${LOW_CAP}" + +# Populate object with some data +rados -p "${POOL}" put "${OBJ}" /etc/passwd + +# Try changing back to SMALL_HINT - expect BIG_HINT (non-empty object) +rados -p "${POOL}" set-alloc-hint "${OBJ}" "${SMALL_HINT}" "${SMALL_HINT}" +expect_alloc_hint_eq "${BIG_HINT}" + +OBJ="bar" +setup_pgid "${POOL}" "${OBJ}" + +# Non-existent object, SMALL_HINT - expect SMALL_HINT (object creation) +rados -p "${POOL}" set-alloc-hint "${OBJ}" "${SMALL_HINT}" "${SMALL_HINT}" +expect_alloc_hint_eq "${SMALL_HINT}" + +ceph osd pool delete "${POOL}" "${POOL}" --yes-i-really-really-mean-it + +# +# ECBackend tests +# + +PROFILE="alloc_hint-ecprofile" +POOL="alloc_hint-ec" +ceph osd erasure-code-profile set "${PROFILE}" k=2 m=1 crush-failure-domain=osd +ceph osd erasure-code-profile get "${PROFILE}" # just so it's logged +ceph osd pool create "${POOL}" "${NUM_PG}" "${NUM_PGP}" erasure "${PROFILE}" +ceph osd pool application enable "${POOL}" rados + +OBJ="baz" +setup_pgid "${POOL}" "${OBJ}" +rados -p "${POOL}" create "${OBJ}" + +# Empty object, SMALL_HINT - expect scaled-down SMALL_HINT +rados -p "${POOL}" set-alloc-hint "${OBJ}" "${SMALL_HINT}" "${SMALL_HINT}" +expect_alloc_hint_eq "$((SMALL_HINT / EC_K))" + +ceph osd pool delete "${POOL}" "${POOL}" --yes-i-really-really-mean-it + +# +# Global teardown +# + +echo "OK" diff --git a/qa/workunits/rados/test_cache_pool.sh b/qa/workunits/rados/test_cache_pool.sh new file mode 100755 index 000000000..f4187a98a --- /dev/null +++ b/qa/workunits/rados/test_cache_pool.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash + +set -ex + +expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +# create pools, set up tier relationship +ceph osd pool create base_pool 2 +ceph osd pool application enable base_pool rados +ceph osd pool create partial_wrong 2 +ceph osd pool create wrong_cache 2 +ceph osd tier add base_pool partial_wrong +ceph osd tier add base_pool wrong_cache + +# populate base_pool with some data +echo "foo" > foo.txt +echo "bar" > bar.txt +echo "baz" > baz.txt +rados -p base_pool put fooobj foo.txt +rados -p base_pool put barobj bar.txt +# fill in wrong_cache backwards so we can tell we read from it +rados -p wrong_cache put fooobj bar.txt +rados -p wrong_cache put barobj foo.txt +# partial_wrong gets barobj backwards so we can check promote and non-promote +rados -p partial_wrong put barobj foo.txt + +# get the objects back before setting a caching pool +rados -p base_pool get fooobj tmp.txt +diff -q tmp.txt foo.txt +rados -p base_pool get barobj tmp.txt +diff -q tmp.txt bar.txt + +# set up redirect and make sure we get backwards results +ceph osd tier set-overlay base_pool wrong_cache +ceph osd tier cache-mode wrong_cache writeback +rados -p base_pool get fooobj tmp.txt +diff -q tmp.txt bar.txt +rados -p base_pool get barobj tmp.txt +diff -q tmp.txt foo.txt + +# switch cache pools and make sure we're doing promote +ceph osd tier remove-overlay base_pool +ceph osd tier set-overlay base_pool partial_wrong +ceph osd tier cache-mode partial_wrong writeback +rados -p base_pool get fooobj tmp.txt +diff -q tmp.txt foo.txt # hurray, it promoted! +rados -p base_pool get barobj tmp.txt +diff -q tmp.txt foo.txt # yep, we read partial_wrong's local object! + +# try a nonexistent object and make sure we get an error +expect_false rados -p base_pool get bazobj tmp.txt + +# drop the cache entirely and make sure contents are still the same +ceph osd tier remove-overlay base_pool +rados -p base_pool get fooobj tmp.txt +diff -q tmp.txt foo.txt +rados -p base_pool get barobj tmp.txt +diff -q tmp.txt bar.txt + +# create an empty cache pool and make sure it has objects after reading +ceph osd pool create empty_cache 2 + +touch empty.txt +rados -p empty_cache ls > tmp.txt +diff -q tmp.txt empty.txt + +ceph osd tier add base_pool empty_cache +ceph osd tier set-overlay base_pool empty_cache +ceph osd tier cache-mode empty_cache writeback +rados -p base_pool get fooobj tmp.txt +rados -p base_pool get barobj tmp.txt +expect_false rados -p base_pool get bazobj tmp.txt + +rados -p empty_cache ls > tmp.txt +expect_false diff -q tmp.txt empty.txt + +# cleanup +ceph osd tier remove-overlay base_pool +ceph osd tier remove base_pool wrong_cache +ceph osd tier remove base_pool partial_wrong +ceph osd tier remove base_pool empty_cache +ceph osd pool delete base_pool base_pool --yes-i-really-really-mean-it +ceph osd pool delete empty_cache empty_cache --yes-i-really-really-mean-it +ceph osd pool delete wrong_cache wrong_cache --yes-i-really-really-mean-it +ceph osd pool delete partial_wrong partial_wrong --yes-i-really-really-mean-it + +## set of base, cache +ceph osd pool create base 8 +ceph osd pool application enable base rados +ceph osd pool create cache 8 + +ceph osd tier add base cache +ceph osd tier cache-mode cache writeback +ceph osd tier set-overlay base cache + +# cache-flush, cache-evict +rados -p base put foo /etc/passwd +expect_false rados -p base cache-evict foo +expect_false rados -p base cache-flush foo +expect_false rados -p cache cache-evict foo +rados -p cache cache-flush foo +rados -p cache cache-evict foo +rados -p cache ls - | wc -l | grep 0 + +# cache-try-flush, cache-evict +rados -p base put foo /etc/passwd +expect_false rados -p base cache-evict foo +expect_false rados -p base cache-flush foo +expect_false rados -p cache cache-evict foo +rados -p cache cache-try-flush foo +rados -p cache cache-evict foo +rados -p cache ls - | wc -l | grep 0 + +# cache-flush-evict-all +rados -p base put bar /etc/passwd +rados -p cache ls - | wc -l | grep 1 +expect_false rados -p base cache-flush-evict-all +rados -p cache cache-flush-evict-all +rados -p cache ls - | wc -l | grep 0 + +# cache-try-flush-evict-all +rados -p base put bar /etc/passwd +rados -p cache ls - | wc -l | grep 1 +expect_false rados -p base cache-flush-evict-all +rados -p cache cache-try-flush-evict-all +rados -p cache ls - | wc -l | grep 0 + +# cache flush/evit when clone objects exist +rados -p base put testclone /etc/passwd +rados -p cache ls - | wc -l | grep 1 +ceph osd pool mksnap base snap +rados -p base put testclone /etc/hosts +rados -p cache cache-flush-evict-all +rados -p cache ls - | wc -l | grep 0 + +ceph osd tier cache-mode cache proxy --yes-i-really-mean-it +rados -p base -s snap get testclone testclone.txt +diff -q testclone.txt /etc/passwd +rados -p base get testclone testclone.txt +diff -q testclone.txt /etc/hosts + +# test --with-clones option +ceph osd tier cache-mode cache writeback +rados -p base put testclone2 /etc/passwd +rados -p cache ls - | wc -l | grep 1 +ceph osd pool mksnap base snap1 +rados -p base put testclone2 /etc/hosts +expect_false rados -p cache cache-flush testclone2 +rados -p cache cache-flush testclone2 --with-clones +expect_false rados -p cache cache-evict testclone2 +rados -p cache cache-evict testclone2 --with-clones +rados -p cache ls - | wc -l | grep 0 + +rados -p base -s snap1 get testclone2 testclone2.txt +diff -q testclone2.txt /etc/passwd +rados -p base get testclone2 testclone2.txt +diff -q testclone2.txt /etc/hosts + +# cleanup +ceph osd tier remove-overlay base +ceph osd tier remove base cache + +ceph osd pool delete cache cache --yes-i-really-really-mean-it +ceph osd pool delete base base --yes-i-really-really-mean-it + +echo OK diff --git a/qa/workunits/rados/test_crash.sh b/qa/workunits/rados/test_crash.sh new file mode 100755 index 000000000..26a4c9bdc --- /dev/null +++ b/qa/workunits/rados/test_crash.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +set -x + +# run on a single-node three-OSD cluster + +sudo killall -ABRT ceph-osd +sleep 5 + +# kill caused coredumps; find them and delete them, carefully, so as +# not to disturb other coredumps, or else teuthology will see them +# and assume test failure. sudos are because the core files are +# root/600 +for f in $(find $TESTDIR/archive/coredump -type f); do + gdb_output=$(echo "quit" | sudo gdb /usr/bin/ceph-osd $f) + if expr match "$gdb_output" ".*generated.*ceph-osd.*" && \ + ( \ + + expr match "$gdb_output" ".*terminated.*signal 6.*" || \ + expr match "$gdb_output" ".*terminated.*signal SIGABRT.*" \ + ) + then + sudo rm $f + fi +done + +# ceph-crash runs as the unprivileged "ceph" user, but when under test +# the ceph osd daemons are running as root, so their crash files aren't +# readable. let's chown them so they behave as they would in real life. +sudo chown -R ceph:ceph /var/lib/ceph/crash + +# let daemon find crashdumps on startup +sudo systemctl restart ceph-crash +sleep 30 + +# must be 3 crashdumps registered and moved to crash/posted +[ $(ceph crash ls | wc -l) = 4 ] || exit 1 # 4 here bc of the table header +[ $(sudo find /var/lib/ceph/crash/posted/ -name meta | wc -l) = 3 ] || exit 1 + +# there should be a health warning +ceph health detail | grep RECENT_CRASH || exit 1 +ceph crash archive-all +sleep 30 +ceph health detail | grep -c RECENT_CRASH | grep 0 # should be gone! diff --git a/qa/workunits/rados/test_crushdiff.sh b/qa/workunits/rados/test_crushdiff.sh new file mode 100755 index 000000000..833ecbd0b --- /dev/null +++ b/qa/workunits/rados/test_crushdiff.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +set -ex + +REP_POOL= +EC_POOL= +TEMPDIR= + +OSD_NUM=$(ceph osd ls | wc -l) +test ${OSD_NUM} -gt 0 + +setup() { + local pool + + TEMPDIR=`mktemp -d` + + pool=test-crushdiff-rep-$$ + ceph osd pool create ${pool} 32 + REP_POOL=${pool} + rados -p ${REP_POOL} bench 5 write --no-cleanup + + if [ ${OSD_NUM} -gt 3 ]; then + pool=test-crushdiff-ec-$$ + ceph osd pool create ${pool} 32 32 erasure + EC_POOL=${pool} + rados -p ${EC_POOL} bench 5 write --no-cleanup + fi +} + +cleanup() { + set +e + + test -n "${EC_POOL}" && + ceph osd pool delete "${EC_POOL}" "${EC_POOL}" \ + --yes-i-really-really-mean-it + EC_POOL= + + test -n "${REP_POOL}" && + ceph osd pool delete "${REP_POOL}" "${REP_POOL}" \ + --yes-i-really-really-mean-it + REP_POOL= + + test -n "${TEMPDIR}" && rm -Rf ${TEMPDIR} + TEMPDIR= +} + +trap "cleanup" INT TERM EXIT + +setup + +# test without crushmap modification + +crushdiff export ${TEMPDIR}/cm.txt --verbose +crushdiff compare ${TEMPDIR}/cm.txt --verbose +crushdiff import ${TEMPDIR}/cm.txt --verbose + +# test using a compiled crushmap + +crushdiff export ${TEMPDIR}/cm --compiled --verbose +crushdiff compare ${TEMPDIR}/cm --compiled --verbose +crushdiff import ${TEMPDIR}/cm --compiled --verbose + +# test using "offline" osdmap and pg-dump + +ceph osd getmap -o ${TEMPDIR}/osdmap +ceph pg dump --format json > ${TEMPDIR}/pg-dump + +crushdiff export ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap --verbose +crushdiff compare ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap \ + --pg-dump ${TEMPDIR}/pg-dump --verbose | tee ${TEMPDIR}/compare.txt + +# test the diff is zero when the crushmap is not modified + +grep '^0/[0-9]* (0\.00%) pgs affected' ${TEMPDIR}/compare.txt +grep '^0/[0-9]* (0\.00%) objects affected' ${TEMPDIR}/compare.txt +grep '^0/[0-9]* (0\.00%) pg shards to move' ${TEMPDIR}/compare.txt +grep '^0/[0-9]* (0\.00%) pg object shards to move' ${TEMPDIR}/compare.txt +grep '^0\.00/.* (0\.00%) bytes to move' ${TEMPDIR}/compare.txt +crushdiff import ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap --verbose + +if [ ${OSD_NUM} -gt 3 ]; then + + # test the diff is non-zero when the crushmap is modified + + cat ${TEMPDIR}/cm.txt >&2 + + weight=$(awk '/item osd\.0 weight ([0-9.]+)/ {print $4 * 3}' \ + ${TEMPDIR}/cm.txt) + test -n "${weight}" + sed -i -Ee 's/^(.*item osd\.0 weight )[0-9.]+/\1'${weight}'/' \ + ${TEMPDIR}/cm.txt + crushdiff compare ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap \ + --pg-dump ${TEMPDIR}/pg-dump --verbose | tee ${TEMPDIR}/compare.txt + grep '^[1-9][0-9]*/[0-9]* (.*%) pgs affected' ${TEMPDIR}/compare.txt + grep '^[1-9][0-9]*/[0-9]* (.*%) objects affected' ${TEMPDIR}/compare.txt + grep '^[1-9][0-9]*/[0-9]* (.*%) pg shards to move' ${TEMPDIR}/compare.txt + grep '^[1-9][0-9]*/[0-9]* (.*%) pg object shards to move' \ + ${TEMPDIR}/compare.txt + grep '^.*/.* (.*%) bytes to move' ${TEMPDIR}/compare.txt + crushdiff import ${TEMPDIR}/cm.txt --osdmap ${TEMPDIR}/osdmap --verbose +fi + +echo OK diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh new file mode 100755 index 000000000..18deb331b --- /dev/null +++ b/qa/workunits/rados/test_dedup_tool.sh @@ -0,0 +1,458 @@ +#!/usr/bin/env bash + +set -x + +die() { + echo "$@" + exit 1 +} + +do_run() { + if [ "$1" == "--tee" ]; then + shift + tee_out="$1" + shift + "$@" | tee $tee_out + else + "$@" + fi +} + +run_expect_succ() { + echo "RUN_EXPECT_SUCC: " "$@" + do_run "$@" + [ $? -ne 0 ] && die "expected success, but got failure! cmd: $@" +} + +run() { + echo "RUN: " $@ + do_run "$@" +} + +if [ -n "$CEPH_BIN" ] ; then + # CMake env + RADOS_TOOL="$CEPH_BIN/rados" + CEPH_TOOL="$CEPH_BIN/ceph" + DEDUP_TOOL="$CEPH_BIN/ceph-dedup-tool" +else + # executables should be installed by the QA env + RADOS_TOOL=$(which rados) + CEPH_TOOL=$(which ceph) + DEDUP_TOOL=$(which ceph-dedup-tool) +fi + +POOL=dedup_pool +OBJ=test_rados_obj + +[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test" +[ -x "$CEPH_TOOL" ] || die "couldn't find $CEPH_TOOL binary to test" + +run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8 +sleep 5 + +function test_dedup_ratio_fixed() +{ + # case 1 + dd if=/dev/urandom of=dedup_object_1k bs=1K count=1 + for num in `seq 1 50` + do + dd if=dedup_object_1k of=dedup_object_100k bs=1K oflag=append conv=notrunc + done + for num in `seq 1 50` + do + dd if=/dev/zero of=dedup_object_100k bs=1K count=1 oflag=append conv=notrunc + done + + $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_100k + RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 1024 --chunk-algorithm fixed --fingerprint-algorithm sha1 | grep chunk_size_average | awk '{print$2}' | sed "s/\,//g") + # total size / the number of deduped object = 100K / 1 + if [ 51200 -ne $RESULT ]; + then + die "Estimate failed expecting 51200 result $RESULT" + fi + + # case 2 + dd if=/dev/zero of=dedup_object_10m bs=10M count=1 + + $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_10m + RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 | grep examined_bytes | awk '{print$2}') + # 10485760 + if [ 10485760 -ne $RESULT ]; + then + die "Estimate failed expecting 10485760 result $RESULT" + fi + + # case 3 max_thread + for num in `seq 0 20` + do + dd if=/dev/zero of=dedup_object_$num bs=4M count=1 + $RADOS_TOOL -p $POOL put dedup_object_$num ./dedup_object_$num + done + + RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096 --chunk-algorithm fixed --fingerprint-algorithm sha1 --max-thread 4 | grep chunk_size_average | awk '{print$2}' | sed "s/\,//g") + + if [ 98566144 -ne $RESULT ]; + then + die "Estimate failed expecting 98566144 result $RESULT" + fi + + rm -rf ./dedup_object_1k ./dedup_object_100k ./dedup_object_10m + for num in `seq 0 20` + do + rm -rf ./dedup_object_$num + done + $RADOS_TOOL -p $POOL rm $OBJ + for num in `seq 0 20` + do + $RADOS_TOOL -p $POOL rm dedup_object_$num + done +} + +function test_dedup_chunk_scrub() +{ + + CHUNK_POOL=dedup_chunk_pool + run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8 + + echo "hi there" > foo + + echo "hi there" > bar + + echo "there" > foo-chunk + + echo "CHUNK" > bar-chunk + + $CEPH_TOOL osd pool set $POOL fingerprint_algorithm sha1 --yes-i-really-mean-it + $CEPH_TOOL osd pool set $POOL dedup_chunk_algorithm fastcdc --yes-i-really-mean-it + $CEPH_TOOL osd pool set $POOL dedup_cdc_chunk_size 4096 --yes-i-really-mean-it + $CEPH_TOOL osd pool set $POOL dedup_tier $CHUNK_POOL --yes-i-really-mean-it + + $RADOS_TOOL -p $POOL put foo ./foo + $RADOS_TOOL -p $POOL put bar ./bar + + $RADOS_TOOL -p $CHUNK_POOL put bar-chunk ./bar-chunk + $RADOS_TOOL -p $CHUNK_POOL put foo-chunk ./foo-chunk + + $RADOS_TOOL -p $POOL set-chunk bar 0 8 --target-pool $CHUNK_POOL bar-chunk 0 --with-reference + + echo -n "There hi" > test_obj + # dirty + $RADOS_TOOL -p $POOL put foo ./test_obj + $RADOS_TOOL -p $POOL set-chunk foo 0 8 --target-pool $CHUNK_POOL foo-chunk 0 --with-reference + # flush + $RADOS_TOOL -p $POOL tier-flush foo + sleep 2 + + $RADOS_TOOL ls -p $CHUNK_POOL + CHUNK_OID=$(echo -n "There hi" | sha1sum | awk '{print $1}') + + POOL_ID=$($CEPH_TOOL osd pool ls detail | grep $POOL | awk '{print$2}') + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref bar --target-ref-pool-id $POOL_ID + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID) + + RESULT=$($DEDUP_TOOL --op chunk-scrub --chunk-pool $CHUNK_POOL | grep "Damaged object" | awk '{print$4}') + if [ $RESULT -ne "1" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Chunk-scrub failed expecting damaged objects is not 1" + fi + + $DEDUP_TOOL --op chunk-put-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref bar --target-ref-pool-id $POOL_ID + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep bar) + if [ -n "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + + rm -rf ./foo ./bar ./foo-chunk ./bar-chunk ./test_obj + $RADOS_TOOL -p $POOL rm foo + $RADOS_TOOL -p $POOL rm bar +} + +function test_dedup_chunk_repair() +{ + + CHUNK_POOL=dedup_chunk_pool + run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8 + + echo -n "hi there" > foo + + echo -n "hi there" > bar + + echo -n "there" > foo-chunk + + echo -n "CHUNK" > bar-chunk + + $CEPH_TOOL osd pool set $POOL fingerprint_algorithm sha1 --yes-i-really-mean-it + $CEPH_TOOL osd pool set $POOL dedup_chunk_algorithm fastcdc --yes-i-really-mean-it + $CEPH_TOOL osd pool set $POOL dedup_cdc_chunk_size 4096 --yes-i-really-mean-it + $CEPH_TOOL osd pool set $POOL dedup_tier $CHUNK_POOL --yes-i-really-mean-it + + $RADOS_TOOL -p $POOL put foo ./foo + $RADOS_TOOL -p $POOL put bar ./bar + + $RADOS_TOOL -p $CHUNK_POOL put bar-chunk ./bar-chunk + $RADOS_TOOL -p $CHUNK_POOL put foo-chunk ./foo-chunk + + $RADOS_TOOL ls -p $CHUNK_POOL + CHUNK_OID=$(echo -n "hi there" | sha1sum | awk '{print $1}') + + POOL_ID=$($CEPH_TOOL osd pool ls detail | grep $POOL | awk '{print$2}') + $RADOS_TOOL -p $CHUNK_POOL put $CHUNK_OID ./foo + + # increase ref count by two, resuling in mismatch + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref foo --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref foo --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref foo --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref foo --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object bar-chunk --target-ref bar --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object bar-chunk --target-ref bar --target-ref-pool-id $POOL_ID + + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID) + RESULT=$($DEDUP_TOOL --op chunk-scrub --chunk-pool $CHUNK_POOL | grep "Damaged object" | awk '{print$4}') + if [ $RESULT -ne "2" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Chunk-scrub failed expecting damaged objects is not 1" + fi + + $DEDUP_TOOL --op chunk-repair --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref foo --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-repair --chunk-pool $CHUNK_POOL --object bar-chunk --target-ref bar --target-ref-pool-id $POOL_ID + + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep foo | wc -l) + if [ 0 -ne "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object bar-chunk | grep bar | wc -l) + if [ 0 -ne "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + + rm -rf ./foo ./bar ./foo-chunk ./bar-chunk ./test_obj + $RADOS_TOOL -p $POOL rm foo + $RADOS_TOOL -p $POOL rm bar +} + +function test_dedup_object() +{ + + CHUNK_POOL=dedup_chunk_pool + run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8 + + echo "There hiHI" > foo + + $CEPH_TOOL osd pool set $POOL dedup_tier $CHUNK_POOL --yes-i-really-mean-it + $RADOS_TOOL -p $POOL put foo ./foo + + sleep 2 + + rados ls -p $CHUNK_POOL + + RESULT=$($DEDUP_TOOL --pool $POOL --op chunk-dedup --object foo --chunk-pool $CHUNK_POOL --source-off 0 --source-length 10 --fingerprint-algorithm sha1 ) + + POOL_ID=$($CEPH_TOOL osd pool ls detail | grep $POOL | awk '{print$2}') + CHUNK_OID=$(echo -n "There hiHI" | sha1sum | awk '{print $1}') + + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep foo) + + if [ -z "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + $RADOS_TOOL -p $CHUNK_POOL get $CHUNK_OID ./chunk + VERIFY=$(cat ./chunk | sha1sum | awk '{print $1}') + if [ "$CHUNK_OID" != "$VERIFY" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Comparing failed expecting chunk mismatch" + fi + + echo -n "There hihiHI" > bar + + $RADOS_TOOL -p $POOL put bar ./bar + RESULT=$($DEDUP_TOOL --pool $POOL --op object-dedup --object bar --chunk-pool $CHUNK_POOL --fingerprint-algorithm sha1 --dedup-cdc-chunk-size 4096) + + CHUNK_OID=$(echo -n "There hihiHI" | sha1sum | awk '{print $1}') + + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep bar) + if [ -z "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + $RADOS_TOOL -p $CHUNK_POOL get $CHUNK_OID ./chunk + VERIFY=$(cat ./chunk | sha1sum | awk '{print $1}') + if [ "$CHUNK_OID" != "$VERIFY" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Comparing failed expecting chunk mismatch" + fi + + echo -n "THERE HIHIHI" > bar + $RADOS_TOOL -p $POOL put bar ./bar + $RADOS_TOOL -p $POOL mksnap mysnap + + echo -n "There HIHIHI" > bar + $RADOS_TOOL -p $POOL put bar ./bar + + RESULT=$($DEDUP_TOOL --pool $POOL --op object-dedup --object bar --chunk-pool $CHUNK_POOL --fingerprint-algorithm sha1 --dedup-cdc-chunk-size 4096 --snap) + + CHUNK_OID=$(echo -n "THERE HIHIHI" | sha1sum | awk '{print $1}') + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep bar) + if [ -z "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + CHUNK_OID=$(echo -n "There HIHIHI" | sha1sum | awk '{print $1}') + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep bar) + if [ -z "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + # rerun tier-flush + + RESULT=$($DEDUP_TOOL --pool $POOL --op object-dedup --object bar --chunk-pool $CHUNK_POOL --fingerprint-algorithm sha1 --dedup-cdc-chunk-size 4096) + + CHUNK_OID=$(echo -n "There HIHIHI" | sha1sum | awk '{print $1}') + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep bar) + if [ -z "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + + rm -rf ./foo ./bar ./chunk + $RADOS_TOOL -p $POOL rm foo + $RADOS_TOOL -p $POOL rm bar +} + +function test_sample_dedup() +{ + CHUNK_POOL=dedup_chunk_pool + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + + sleep 2 + + run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8 + run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8 + run_expect_succ "$CEPH_TOOL" osd pool set "$POOL" dedup_tier "$CHUNK_POOL" + run_expect_succ "$CEPH_TOOL" osd pool set "$POOL" dedup_chunk_algorithm fastcdc + run_expect_succ "$CEPH_TOOL" osd pool set "$POOL" dedup_cdc_chunk_size 8192 + run_expect_succ "$CEPH_TOOL" osd pool set "$POOL" fingerprint_algorithm sha1 + + # 8 Dedupable objects + CONTENT_1="There hiHI" + echo $CONTENT_1 > foo + for num in `seq 1 8` + do + $RADOS_TOOL -p $POOL put foo_$num ./foo + done + + # 1 Unique object + CONTENT_3="There hiHI3" + echo $CONTENT_3 > foo3 + $RADOS_TOOL -p $POOL put foo3_1 ./foo3 + + sleep 2 + + # Execute dedup crawler + RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --chunk-dedup-threshold 3 --sampling-ratio 50) + + CHUNK_OID_1=$(echo $CONTENT_1 | sha1sum | awk '{print $1}') + CHUNK_OID_3=$(echo $CONTENT_3 | sha1sum | awk '{print $1}') + + # Find chunk object has references of 8 dedupable meta objects + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID_1) + DEDUP_COUNT=0 + for num in `seq 1 8` + do + GREP_RESULT=$(echo $RESULT | grep foo_$num) + if [ -n "$GREP_RESULT" ]; then + DEDUP_COUNT=$(($DEDUP_COUNT + 1)) + fi + done + if [ $DEDUP_COUNT -lt 2 ]; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Chunk object has no reference of first meta object" + fi + + # 7 Duplicated objects but less than chunk dedup threshold + CONTENT_2="There hiHI2" + echo $CONTENT_2 > foo2 + for num in `seq 1 7` + do + $RADOS_TOOL -p $POOL put foo2_$num ./foo2 + done + CHUNK_OID_2=$(echo $CONTENT_2 | sha1sum | awk '{print $1}') + + RESULT=$($DEDUP_TOOL --pool $POOL --chunk-pool $CHUNK_POOL --op sample-dedup --chunk-algorithm fastcdc --fingerprint-algorithm sha1 --sampling-ratio 100 --chunk-dedup-threshold 2) + + # Objects duplicates less than chunk dedup threshold should be deduplicated because of they satisfies object-dedup-threshold + # The only object, which is crawled at the very first, should not be deduplicated because it was not duplicated at initial time + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID_2) + DEDUP_COUNT=0 + for num in `seq 1 7` + do + GREP_RESULT=$(echo $RESULT | grep foo2_$num) + if [ -n "$GREP_RESULT" ]; then + DEDUP_COUNT=$(($DEDUP_COUNT + 1)) + fi + done + if [ $DEDUP_COUNT -ne 6 ]; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Chunk object has no reference of first meta object" + fi + + # Unique object should not be deduplicated + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID_3) + GREP_RESULT=$($RESULT | grep $CHUNK_OID_3) + if [ -n "$GREP_RESULT" ]; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Chunk object has no reference of second meta object" + fi + + rm -rf ./foo ./foo2 ./foo3 + for num in `seq 1 8` + do + $RADOS_TOOL -p $POOL rm foo_$num + done + for num in `seq 1 2` + do + $RADOS_TOOL -p $POOL rm foo2_$num + done + $RADOS_TOOL -p $POOL rm foo3_1 + + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it +} + +test_dedup_ratio_fixed +test_dedup_chunk_scrub +test_dedup_chunk_repair +test_dedup_object +test_sample_dedup + +$CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + +echo "SUCCESS!" +exit 0 diff --git a/qa/workunits/rados/test_envlibrados_for_rocksdb.sh b/qa/workunits/rados/test_envlibrados_for_rocksdb.sh new file mode 100755 index 000000000..371452f40 --- /dev/null +++ b/qa/workunits/rados/test_envlibrados_for_rocksdb.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -ex + +############################################ +# Helper functions +############################################ +source $(dirname $0)/../ceph-helpers-root.sh + +############################################ +# Install required tools +############################################ +echo "Install required tools" + +CURRENT_PATH=`pwd` + +############################################ +# Compile&Start RocksDB +############################################ +# install prerequisites +# for rocksdb +case $(distro_id) in + ubuntu|debian|devuan|softiron) + install git g++ libsnappy-dev zlib1g-dev libbz2-dev libradospp-dev cmake + ;; + centos|fedora|rhel) + case $(distro_id) in + rhel) + # RHEL needs CRB repo for snappy-devel + sudo subscription-manager repos --enable "codeready-builder-for-rhel-8-x86_64-rpms" + ;; + esac + install git gcc-c++.x86_64 snappy-devel zlib zlib-devel bzip2 bzip2-devel libradospp-devel.x86_64 cmake libarchive-3.3.3 + ;; + opensuse*|suse|sles) + install git gcc-c++ snappy-devel zlib-devel libbz2-devel libradospp-devel + ;; + *) + echo "$(distro_id) is unknown, $@ will have to be installed manually." + ;; +esac + +# # gflags +# sudo yum install gflags-devel +# +# wget https://github.com/schuhschuh/gflags/archive/master.zip +# unzip master.zip +# cd gflags-master +# mkdir build && cd build +# export CXXFLAGS="-fPIC" && cmake .. && make VERBOSE=1 +# make && make install + +# # snappy-devel + + +echo "Compile rocksdb" +if [ -e rocksdb ]; then + rm -fr rocksdb +fi + +pushd $(dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/bash.sh)/../../../ +git submodule update --init src/rocksdb +popd +git clone $(dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/bash.sh)/../../../src/rocksdb rocksdb + +# compile code +cd rocksdb +if type cmake3 > /dev/null 2>&1 ; then + CMAKE=cmake3 +else + CMAKE=cmake +fi + +[ -z "$BUILD_DIR" ] && BUILD_DIR=build +mkdir ${BUILD_DIR} && cd ${BUILD_DIR} && ${CMAKE} -DCMAKE_BUILD_TYPE=Debug -DWITH_TESTS=ON -DWITH_LIBRADOS=ON -DWITH_SNAPPY=ON -DWITH_GFLAGS=OFF -DFAIL_ON_WARNINGS=OFF .. +make rocksdb_env_librados_test -j8 + +echo "Copy ceph.conf" +# prepare ceph.conf +mkdir -p ../ceph/src/ +if [ -f "/etc/ceph/ceph.conf" ]; then + cp /etc/ceph/ceph.conf ../ceph/src/ +elif [ -f "/etc/ceph/ceph/ceph.conf" ]; then + cp /etc/ceph/ceph/ceph.conf ../ceph/src/ +else + echo "/etc/ceph/ceph/ceph.conf doesn't exist" +fi + +echo "Run EnvLibrados test" +# run test +if [ -f "../ceph/src/ceph.conf" ] + then + cp env_librados_test ~/cephtest/archive + ./env_librados_test +else + echo "../ceph/src/ceph.conf doesn't exist" +fi +cd ${CURRENT_PATH} diff --git a/qa/workunits/rados/test_hang.sh b/qa/workunits/rados/test_hang.sh new file mode 100755 index 000000000..724e0bb82 --- /dev/null +++ b/qa/workunits/rados/test_hang.sh @@ -0,0 +1,8 @@ +#!/bin/sh -ex + +# Hang forever for manual testing using the thrasher +while(true) +do + sleep 300 +done +exit 0 diff --git a/qa/workunits/rados/test_health_warnings.sh b/qa/workunits/rados/test_health_warnings.sh new file mode 100755 index 000000000..d393e5c68 --- /dev/null +++ b/qa/workunits/rados/test_health_warnings.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +set -uex + +# number of osds = 10 +crushtool -o crushmap --build --num_osds 10 host straw 2 rack straw 2 row straw 2 root straw 0 +ceph osd setcrushmap -i crushmap +ceph osd tree +ceph tell osd.* injectargs --osd_max_markdown_count 1024 --osd_max_markdown_period 1 +ceph osd set noout + +wait_for_healthy() { + while ceph health | grep down + do + sleep 1 + done +} + +test_mark_two_osds_same_host_down() { + ceph osd set noup + ceph osd down osd.0 osd.1 + ceph health detail + ceph health | grep "1 host" + ceph health | grep "2 osds" + ceph health detail | grep "osd.0" + ceph health detail | grep "osd.1" + ceph osd unset noup + wait_for_healthy +} + +test_mark_two_osds_same_rack_down() { + ceph osd set noup + ceph osd down osd.8 osd.9 + ceph health detail + ceph health | grep "1 host" + ceph health | grep "1 rack" + ceph health | grep "1 row" + ceph health | grep "2 osds" + ceph health detail | grep "osd.8" + ceph health detail | grep "osd.9" + ceph osd unset noup + wait_for_healthy +} + +test_mark_all_but_last_osds_down() { + ceph osd set noup + ceph osd down $(ceph osd ls | sed \$d) + ceph health detail + ceph health | grep "1 row" + ceph health | grep "2 racks" + ceph health | grep "4 hosts" + ceph health | grep "9 osds" + ceph osd unset noup + wait_for_healthy +} + +test_mark_two_osds_same_host_down_with_classes() { + ceph osd set noup + ceph osd crush set-device-class ssd osd.0 osd.2 osd.4 osd.6 osd.8 + ceph osd crush set-device-class hdd osd.1 osd.3 osd.5 osd.7 osd.9 + ceph osd down osd.0 osd.1 + ceph health detail + ceph health | grep "1 host" + ceph health | grep "2 osds" + ceph health detail | grep "osd.0" + ceph health detail | grep "osd.1" + ceph osd unset noup + wait_for_healthy +} + +test_mark_two_osds_same_host_down +test_mark_two_osds_same_rack_down +test_mark_all_but_last_osds_down +test_mark_two_osds_same_host_down_with_classes + +exit 0 diff --git a/qa/workunits/rados/test_large_omap_detection.py b/qa/workunits/rados/test_large_omap_detection.py new file mode 100755 index 000000000..b5c81a8d8 --- /dev/null +++ b/qa/workunits/rados/test_large_omap_detection.py @@ -0,0 +1,134 @@ +#!/usr/bin/python3 +# -*- mode:python -*- +# vim: ts=4 sw=4 smarttab expandtab +# +# Copyright (C) 2017 Red Hat <contact@redhat.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# + +import json +import rados +import shlex +import subprocess +import time + +def cleanup(cluster): + cluster.delete_pool('large-omap-test-pool') + cluster.shutdown() + +def init(): + # For local testing + #cluster = rados.Rados(conffile='./ceph.conf') + cluster = rados.Rados(conffile='/etc/ceph/ceph.conf') + cluster.connect() + print("\nCluster ID: " + cluster.get_fsid()) + cluster.create_pool('large-omap-test-pool') + ioctx = cluster.open_ioctx('large-omap-test-pool') + ioctx.write_full('large-omap-test-object1', b"Lorem ipsum") + op = ioctx.create_write_op() + + keys = [] + values = [] + for x in range(20001): + keys.append(str(x)) + values.append(b"X") + + ioctx.set_omap(op, tuple(keys), tuple(values)) + ioctx.operate_write_op(op, 'large-omap-test-object1', 0) + ioctx.release_write_op(op) + + ioctx.write_full('large-omap-test-object2', b"Lorem ipsum dolor") + op = ioctx.create_write_op() + + buffer = ("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut " + "enim ad minim veniam, quis nostrud exercitation ullamco laboris " + "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in " + "reprehenderit in voluptate velit esse cillum dolore eu fugiat " + "nulla pariatur. Excepteur sint occaecat cupidatat non proident, " + "sunt in culpa qui officia deserunt mollit anim id est laborum.") + + keys = [] + values = [] + for x in range(20000): + keys.append(str(x)) + values.append(buffer.encode()) + + ioctx.set_omap(op, tuple(keys), tuple(values)) + ioctx.operate_write_op(op, 'large-omap-test-object2', 0) + ioctx.release_write_op(op) + ioctx.close() + return cluster + +def get_deep_scrub_timestamp(pgid): + cmd = ['ceph', 'pg', 'dump', '--format=json-pretty'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out = proc.communicate()[0] + try: + pgstats = json.loads(out)['pg_map']['pg_stats'] + except KeyError: + pgstats = json.loads(out)['pg_stats'] + for stat in pgstats: + if stat['pgid'] == pgid: + return stat['last_deep_scrub_stamp'] + +def wait_for_scrub(): + osds = set(); + pgs = dict(); + cmd = ['ceph', 'osd', 'map', 'large-omap-test-pool', + 'large-omap-test-object1', '--format=json-pretty'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out = proc.communicate()[0] + osds.add(json.loads(out)['acting_primary']) + pgs[json.loads(out)['pgid']] = get_deep_scrub_timestamp(json.loads(out)['pgid']) + cmd = ['ceph', 'osd', 'map', 'large-omap-test-pool', + 'large-omap-test-object2', '--format=json-pretty'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out = proc.communicate()[0] + osds.add(json.loads(out)['acting_primary']) + pgs[json.loads(out)['pgid']] = get_deep_scrub_timestamp(json.loads(out)['pgid']) + + for pg in pgs: + command = "ceph pg deep-scrub " + str(pg) + subprocess.check_call(shlex.split(command)) + + for pg in pgs: + RETRIES = 0 + while RETRIES < 60 and pgs[pg] == get_deep_scrub_timestamp(pg): + time.sleep(10) + RETRIES += 1 + +def check_health_output(): + RETRIES = 0 + result = 0 + while RETRIES < 6 and result != 2: + result = 0 + RETRIES += 1 + output = subprocess.check_output(["ceph", "health", "detail"]) + for line in output.splitlines(): + result += int(line.find(b'2 large omap objects') != -1) + time.sleep(10) + + if result != 2: + print("Error, got invalid output:") + print(output) + raise Exception + +def main(): + cluster = init() + wait_for_scrub() + check_health_output() + + cleanup(cluster) + +if __name__ == '__main__': + main() diff --git a/qa/workunits/rados/test_libcephsqlite.sh b/qa/workunits/rados/test_libcephsqlite.sh new file mode 100755 index 000000000..1810a3f3f --- /dev/null +++ b/qa/workunits/rados/test_libcephsqlite.sh @@ -0,0 +1,136 @@ +#!/bin/bash -ex + +# The main point of these tests beyond ceph_test_libcephsqlite is to: +# +# - Ensure you can load the Ceph VFS via the dynamic load extension mechanism +# in SQLite. +# - Check the behavior of a dead application, that it does not hold locks +# indefinitely. + +pool="$1" +ns="$(basename $0)" + +function sqlite { + background="$1" + if [ "$background" = b ]; then + shift + fi + a=$(cat) + printf "%s" "$a" >&2 + # We're doing job control gymnastics here to make sure that sqlite3 is the + # main process (i.e. the process group leader) in the background, not a bash + # function or job pipeline. + sqlite3 -cmd '.output /dev/null' -cmd '.load libcephsqlite.so' -cmd 'pragma journal_mode = PERSIST' -cmd ".open file:///$pool:$ns/baz.db?vfs=ceph" -cmd '.output stdout' <<<"$a" & + if [ "$background" != b ]; then + wait + fi +} + +function striper { + rados --pool=$pool --namespace="$ns" --striper "$@" +} + +function repeat { + n=$1 + shift + for ((i = 0; i < "$n"; ++i)); do + echo "$*" + done +} + +striper rm baz.db || true + +time sqlite <<EOF +create table if not exists foo (a INT); +insert into foo (a) values (RANDOM()); +drop table foo; +EOF + +striper stat baz.db +striper rm baz.db + +time sqlite <<EOF +CREATE TABLE IF NOT EXISTS rand(text BLOB NOT NULL); +$(repeat 10 'INSERT INTO rand (text) VALUES (RANDOMBLOB(4096));') +SELECT LENGTH(text) FROM rand; +DROP TABLE rand; +EOF + +time sqlite <<EOF +BEGIN TRANSACTION; +CREATE TABLE IF NOT EXISTS rand(text BLOB NOT NULL); +$(repeat 100 'INSERT INTO rand (text) VALUES (RANDOMBLOB(4096));') +COMMIT; +SELECT LENGTH(text) FROM rand; +DROP TABLE rand; +EOF + +# Connection death drops the lock: + +striper rm baz.db +date +sqlite b <<EOF +CREATE TABLE foo (a BLOB); +INSERT INTO foo VALUES ("start"); +WITH RECURSIVE c(x) AS + ( + VALUES(1) + UNION ALL + SELECT x+1 + FROM c + ) +INSERT INTO foo (a) + SELECT RANDOMBLOB(1<<20) + FROM c + LIMIT (1<<20); +EOF + +# Let it chew on that INSERT for a while so it writes data, it will not finish as it's trying to write 2^40 bytes... +sleep 10 +echo done + +jobs -l +kill -KILL -- $(jobs -p) +date +wait +date + +n=$(sqlite <<<"SELECT COUNT(*) FROM foo;") +[ "$n" -eq 1 ] + +# Connection "hang" loses the lock and cannot reacquire it: + +striper rm baz.db +date +sqlite b <<EOF +CREATE TABLE foo (a BLOB); +INSERT INTO foo VALUES ("start"); +WITH RECURSIVE c(x) AS + ( + VALUES(1) + UNION ALL + SELECT x+1 + FROM c + ) +INSERT INTO foo (a) + SELECT RANDOMBLOB(1<<20) + FROM c + LIMIT (1<<20); +EOF + +# Same thing, let it chew on the INSERT for a while... +sleep 20 +jobs -l +kill -STOP -- $(jobs -p) +# cephsqlite_lock_renewal_timeout is 30s +sleep 45 +date +kill -CONT -- $(jobs -p) +sleep 10 +date +# it should exit with an error as it lost the lock +wait +date + +n=$(sqlite <<<"SELECT COUNT(*) FROM foo;") +[ "$n" -eq 1 ] diff --git a/qa/workunits/rados/test_librados_build.sh b/qa/workunits/rados/test_librados_build.sh new file mode 100755 index 000000000..14e332515 --- /dev/null +++ b/qa/workunits/rados/test_librados_build.sh @@ -0,0 +1,87 @@ +#!/bin/bash -ex +# +# Compile and run a librados application outside of the ceph build system, so +# that we can be sure librados.h[pp] is still usable and hasn't accidentally +# started depending on internal headers. +# +# The script assumes all dependencies - e.g. curl, make, gcc, librados headers, +# libradosstriper headers, boost headers, etc. - are already installed. +# + +source $(dirname $0)/../ceph-helpers-root.sh + +trap cleanup EXIT + +SOURCES="hello_radosstriper.cc +hello_world_c.c +hello_world.cc +Makefile +" +BINARIES_TO_RUN="hello_world_c +hello_world_cpp +" +BINARIES="${BINARIES_TO_RUN}hello_radosstriper_cpp +" +# parse output like "octopus (dev)" +case $(librados-config --release | grep -Po ' \(\K[^\)]+') in + dev) + BRANCH=main;; + rc|stable) + BRANCH=$(librados-config --release | cut -d' ' -f1);; + *) + echo "unknown release '$(librados-config --release)'" >&2 + return 1;; +esac +DL_PREFIX="http://git.ceph.com/?p=ceph.git;a=blob_plain;hb=${BRANCH};f=examples/librados/" +#DL_PREFIX="https://raw.githubusercontent.com/ceph/ceph/master/examples/librados/" +DESTDIR=$(pwd) + +function cleanup () { + for f in $BINARIES$SOURCES ; do + rm -f "${DESTDIR}/$f" + done +} + +function get_sources () { + for s in $SOURCES ; do + curl --progress-bar --output $s -L ${DL_PREFIX}$s + done +} + +function check_sources () { + for s in $SOURCES ; do + test -f $s + done +} + +function check_binaries () { + for b in $BINARIES ; do + file $b + test -f $b + done +} + +function run_binaries () { + for b in $BINARIES_TO_RUN ; do + ./$b -c /etc/ceph/ceph.conf + done +} + +pushd $DESTDIR +case $(distro_id) in + centos|fedora|rhel|opensuse*|suse|sles) + install gcc-c++ make libradospp-devel librados-devel;; + ubuntu) + install gcc-11 g++-11 make libradospp-dev librados-dev + export CXX_FLAGS="-std=c++20";; + debian|devuan|softiron) + install g++ make libradospp-dev librados-dev;; + *) + echo "$(distro_id) is unknown, $@ will have to be installed manually." +esac +get_sources +check_sources +make all-system +check_binaries +run_binaries +popd diff --git a/qa/workunits/rados/test_pool_access.sh b/qa/workunits/rados/test_pool_access.sh new file mode 100755 index 000000000..4082870bc --- /dev/null +++ b/qa/workunits/rados/test_pool_access.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +set -ex + +KEYRING=$(mktemp) +trap cleanup EXIT ERR HUP INT QUIT + +cleanup() { + (ceph auth del client.mon_read || true) >/dev/null 2>&1 + (ceph auth del client.mon_write || true) >/dev/null 2>&1 + + rm -f $KEYRING +} + +expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +create_pool_op() { + ID=$1 + POOL=$2 + + cat << EOF | CEPH_ARGS="-k $KEYRING" python3 +import rados + +cluster = rados.Rados(conffile="", rados_id="${ID}") +cluster.connect() +cluster.create_pool("${POOL}") +EOF +} + +delete_pool_op() { + ID=$1 + POOL=$2 + + cat << EOF | CEPH_ARGS="-k $KEYRING" python3 +import rados + +cluster = rados.Rados(conffile="", rados_id="${ID}") +cluster.connect() +cluster.delete_pool("${POOL}") +EOF +} + +create_pool_snap_op() { + ID=$1 + POOL=$2 + SNAP=$3 + + cat << EOF | CEPH_ARGS="-k $KEYRING" python3 +import rados + +cluster = rados.Rados(conffile="", rados_id="${ID}") +cluster.connect() +ioctx = cluster.open_ioctx("${POOL}") + +ioctx.create_snap("${SNAP}") +EOF +} + +remove_pool_snap_op() { + ID=$1 + POOL=$2 + SNAP=$3 + + cat << EOF | CEPH_ARGS="-k $KEYRING" python3 +import rados + +cluster = rados.Rados(conffile="", rados_id="${ID}") +cluster.connect() +ioctx = cluster.open_ioctx("${POOL}") + +ioctx.remove_snap("${SNAP}") +EOF +} + +test_pool_op() +{ + ceph auth get-or-create client.mon_read mon 'allow r' >> $KEYRING + ceph auth get-or-create client.mon_write mon 'allow *' >> $KEYRING + + expect_false create_pool_op mon_read pool1 + create_pool_op mon_write pool1 + + expect_false create_pool_snap_op mon_read pool1 snap1 + create_pool_snap_op mon_write pool1 snap1 + + expect_false remove_pool_snap_op mon_read pool1 snap1 + remove_pool_snap_op mon_write pool1 snap1 + + expect_false delete_pool_op mon_read pool1 + delete_pool_op mon_write pool1 +} + +key=`ceph auth get-or-create-key client.poolaccess1 mon 'allow r' osd 'allow *'` +rados --id poolaccess1 --key $key -p rbd ls + +key=`ceph auth get-or-create-key client.poolaccess2 mon 'allow r' osd 'allow * pool=nopool'` +expect_false rados --id poolaccess2 --key $key -p rbd ls + +key=`ceph auth get-or-create-key client.poolaccess3 mon 'allow r' osd 'allow rw pool=nopool'` +expect_false rados --id poolaccess3 --key $key -p rbd ls + +test_pool_op + +echo OK diff --git a/qa/workunits/rados/test_pool_quota.sh b/qa/workunits/rados/test_pool_quota.sh new file mode 100755 index 000000000..0eacefc64 --- /dev/null +++ b/qa/workunits/rados/test_pool_quota.sh @@ -0,0 +1,68 @@ +#!/bin/sh -ex + +p=`uuidgen` + +# objects +ceph osd pool create $p 12 +ceph osd pool set-quota $p max_objects 10 +ceph osd pool application enable $p rados + +for f in `seq 1 10` ; do + rados -p $p put obj$f /etc/passwd +done + +sleep 30 + +rados -p $p put onemore /etc/passwd & +pid=$! + +ceph osd pool set-quota $p max_objects 100 +wait $pid +[ $? -ne 0 ] && exit 1 || true + +rados -p $p put twomore /etc/passwd + +# bytes +ceph osd pool set-quota $p max_bytes 100 +sleep 30 + +rados -p $p put two /etc/passwd & +pid=$! + +ceph osd pool set-quota $p max_bytes 0 +ceph osd pool set-quota $p max_objects 0 +wait $pid +[ $? -ne 0 ] && exit 1 || true + +rados -p $p put three /etc/passwd + + +#one pool being full does not block a different pool + +pp=`uuidgen` + +ceph osd pool create $pp 12 +ceph osd pool application enable $pp rados + +# set objects quota +ceph osd pool set-quota $pp max_objects 10 +sleep 30 + +for f in `seq 1 10` ; do + rados -p $pp put obj$f /etc/passwd +done + +sleep 30 + +rados -p $p put threemore /etc/passwd + +ceph osd pool set-quota $p max_bytes 0 +ceph osd pool set-quota $p max_objects 0 + +sleep 30 +# done +ceph osd pool delete $p $p --yes-i-really-really-mean-it +ceph osd pool delete $pp $pp --yes-i-really-really-mean-it + +echo OK + diff --git a/qa/workunits/rados/test_python.sh b/qa/workunits/rados/test_python.sh new file mode 100755 index 000000000..cf4597a41 --- /dev/null +++ b/qa/workunits/rados/test_python.sh @@ -0,0 +1,5 @@ +#!/bin/sh -ex + +ceph osd pool create rbd +${PYTHON:-python3} -m pytest -v $(dirname $0)/../../../src/test/pybind/test_rados.py "$@" +exit 0 diff --git a/qa/workunits/rados/test_rados_timeouts.sh b/qa/workunits/rados/test_rados_timeouts.sh new file mode 100755 index 000000000..327c7ab32 --- /dev/null +++ b/qa/workunits/rados/test_rados_timeouts.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -x + +delay_mon() { + MSGTYPE=$1 + shift + $@ --rados-mon-op-timeout 1 --ms-inject-delay-type mon --ms-inject-delay-max 10000000 --ms-inject-delay-probability 1 --ms-inject-delay-msg-type $MSGTYPE + if [ $? -eq 0 ]; then + exit 1 + fi +} + +delay_osd() { + MSGTYPE=$1 + shift + $@ --rados-osd-op-timeout 1 --ms-inject-delay-type osd --ms-inject-delay-max 10000000 --ms-inject-delay-probability 1 --ms-inject-delay-msg-type $MSGTYPE + if [ $? -eq 0 ]; then + exit 2 + fi +} + +# pool ops +delay_mon omap rados lspools +delay_mon poolopreply ceph osd pool create test 8 +delay_mon poolopreply rados mksnap -p test snap +delay_mon poolopreply ceph osd pool rm test test --yes-i-really-really-mean-it + +# other mon ops +delay_mon getpoolstats rados df +delay_mon mon_command ceph df +delay_mon omap ceph osd dump +delay_mon omap ceph -s + +# osd ops +delay_osd osd_op_reply rados -p data put ls /bin/ls +delay_osd osd_op_reply rados -p data get ls - >/dev/null +delay_osd osd_op_reply rados -p data ls +delay_osd command_reply ceph tell osd.0 bench 1 1 + +# rbd commands, using more kinds of osd ops +rbd create -s 1 test +delay_osd osd_op_reply rbd watch test +delay_osd osd_op_reply rbd info test +delay_osd osd_op_reply rbd snap create test@snap +delay_osd osd_op_reply rbd import /bin/ls ls +rbd rm test + +echo OK diff --git a/qa/workunits/rados/test_rados_tool.sh b/qa/workunits/rados/test_rados_tool.sh new file mode 100755 index 000000000..9d025eee8 --- /dev/null +++ b/qa/workunits/rados/test_rados_tool.sh @@ -0,0 +1,924 @@ +#!/usr/bin/env bash + +set -x + +die() { + echo "$@" + exit 1 +} + +usage() { + cat <<EOF +test_rados_tool.sh: tests rados_tool +-c: RADOS configuration file to use [optional] +-k: keep temp files +-h: this help message +-p: set temporary pool to use [optional] +EOF +} + +do_run() { + if [ "$1" == "--tee" ]; then + shift + tee_out="$1" + shift + "$@" | tee $tee_out + else + "$@" + fi +} + +run_expect_fail() { + echo "RUN_EXPECT_FAIL: " "$@" + do_run "$@" + [ $? -eq 0 ] && die "expected failure, but got success! cmd: $@" +} + +run_expect_succ() { + echo "RUN_EXPECT_SUCC: " "$@" + do_run "$@" + [ $? -ne 0 ] && die "expected success, but got failure! cmd: $@" +} + +run_expect_nosignal() { + echo "RUN_EXPECT_NOSIGNAL: " "$@" + do_run "$@" + [ $? -ge 128 ] && die "expected success or fail, but got signal! cmd: $@" +} + +run() { + echo "RUN: " $@ + do_run "$@" +} + +if [ -n "$CEPH_BIN" ] ; then + # CMake env + RADOS_TOOL="$CEPH_BIN/rados" + CEPH_TOOL="$CEPH_BIN/ceph" +else + # executables should be installed by the QA env + RADOS_TOOL=$(which rados) + CEPH_TOOL=$(which ceph) +fi + +KEEP_TEMP_FILES=0 +POOL=trs_pool +POOL_CP_TARGET=trs_pool.2 +POOL_EC=trs_pool_ec + +[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test" +[ -x "$CEPH_TOOL" ] || die "couldn't find $CEPH_TOOL binary to test" + +while getopts "c:hkp:" flag; do + case $flag in + c) RADOS_TOOL="$RADOS_TOOL -c $OPTARG";; + k) KEEP_TEMP_FILES=1;; + h) usage; exit 0;; + p) POOL=$OPTARG;; + *) echo; usage; exit 1;; + esac +done + +TDIR=`mktemp -d -t test_rados_tool.XXXXXXXXXX` || die "mktemp failed" +[ $KEEP_TEMP_FILES -eq 0 ] && trap "rm -rf ${TDIR}; exit" INT TERM EXIT + +# ensure rados doesn't segfault without --pool +run_expect_nosignal "$RADOS_TOOL" --snap "asdf" ls +run_expect_nosignal "$RADOS_TOOL" --snapid "0" ls +run_expect_nosignal "$RADOS_TOOL" --object-locator "asdf" ls +run_expect_nosignal "$RADOS_TOOL" --namespace "asdf" ls + +run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8 +run_expect_succ "$CEPH_TOOL" osd erasure-code-profile set myprofile k=2 m=1 stripe_unit=2K crush-failure-domain=osd --force +run_expect_succ "$CEPH_TOOL" osd pool create "$POOL_EC" 100 100 erasure myprofile + + +# expb happens to be the empty export for legacy reasons +run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/expb" + +# expa has objects foo, foo2 and bar +run_expect_succ "$RADOS_TOOL" -p "$POOL" put foo /etc/fstab +run_expect_succ "$RADOS_TOOL" -p "$POOL" put foo2 /etc/fstab +run_expect_succ "$RADOS_TOOL" -p "$POOL" put bar /etc/fstab +run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/expa" + +# expc has foo and foo2 with some attributes and omaps set +run_expect_succ "$RADOS_TOOL" -p "$POOL" rm bar +run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr foo "rados.toothbrush" "toothbrush" +run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr foo "rados.toothpaste" "crest" +run_expect_succ "$RADOS_TOOL" -p "$POOL" setomapval foo "rados.floss" "myfloss" +run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr foo2 "rados.toothbrush" "green" +run_expect_succ "$RADOS_TOOL" -p "$POOL" setomapheader foo2 "foo2.header" +run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/expc" + +# make sure that --create works +run "$CEPH_TOOL" osd pool rm "$POOL" "$POOL" --yes-i-really-really-mean-it +run_expect_succ "$RADOS_TOOL" -p "$POOL" --create import "$TDIR/expa" + +# make sure that lack of --create fails +run_expect_succ "$CEPH_TOOL" osd pool rm "$POOL" "$POOL" --yes-i-really-really-mean-it +run_expect_fail "$RADOS_TOOL" -p "$POOL" import "$TDIR/expa" + +run_expect_succ "$RADOS_TOOL" -p "$POOL" --create import "$TDIR/expa" + +# inaccessible import src should fail +run_expect_fail "$RADOS_TOOL" -p "$POOL" import "$TDIR/dir_nonexistent" + +# export an empty pool to test purge +run_expect_succ "$RADOS_TOOL" purge "$POOL" --yes-i-really-really-mean-it +run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/empty" +cmp -s "$TDIR/expb" "$TDIR/empty" \ + || die "failed to export the same stuff we imported!" +rm -f "$TDIR/empty" + +# import some stuff with extended attributes on it +run_expect_succ "$RADOS_TOOL" -p "$POOL" import "$TDIR/expc" +VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"` +[ ${VAL} = "toothbrush" ] || die "Invalid attribute after import" + +# the second time, the xattrs should match, so there should be nothing to do. +run_expect_succ "$RADOS_TOOL" -p "$POOL" import "$TDIR/expc" +VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"` +[ "${VAL}" = "toothbrush" ] || die "Invalid attribute after second import" + +# Now try with --no-overwrite option after changing an attribute +run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr foo "rados.toothbrush" "dentist" +run_expect_succ "$RADOS_TOOL" -p "$POOL" import --no-overwrite "$TDIR/expc" +VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"` +[ "${VAL}" = "dentist" ] || die "Invalid attribute after second import" + +# now force it to copy everything +run_expect_succ "$RADOS_TOOL" -p "$POOL" import "$TDIR/expc" +VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"` +[ "${VAL}" = "toothbrush" ] || die "Invalid attribute after second import" + +# test copy pool +run "$CEPH_TOOL" osd pool rm "$POOL" "$POOL" --yes-i-really-really-mean-it +run "$CEPH_TOOL" osd pool rm "$POOL_CP_TARGET" "$POOL_CP_TARGET" --yes-i-really-really-mean-it +run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8 +run_expect_succ "$CEPH_TOOL" osd pool create "$POOL_CP_TARGET" 8 + +# create src files +mkdir -p "$TDIR/dir_cp_src" +for i in `seq 1 5`; do + fname="$TDIR/dir_cp_src/f.$i" + objname="f.$i" + dd if=/dev/urandom of="$fname" bs=$((1024*1024)) count=$i + run_expect_succ "$RADOS_TOOL" -p "$POOL" put $objname "$fname" + +# a few random attrs + for j in `seq 1 4`; do + rand_str=`dd if=/dev/urandom bs=4 count=1 | hexdump -x` + run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr $objname attr.$j "$rand_str" + run_expect_succ --tee "$fname.attr.$j" "$RADOS_TOOL" -p "$POOL" getxattr $objname attr.$j + done + + rand_str=`dd if=/dev/urandom bs=4 count=1 | hexdump -x` + run_expect_succ "$RADOS_TOOL" -p "$POOL" setomapheader $objname "$rand_str" + run_expect_succ --tee "$fname.omap.header" "$RADOS_TOOL" -p "$POOL" getomapheader $objname + +# a few random omap keys + for j in `seq 1 4`; do + rand_str=`dd if=/dev/urandom bs=4 count=1 | hexdump -x` + run_expect_succ "$RADOS_TOOL" -p "$POOL" setomapval $objname key.$j "$rand_str" + done + run_expect_succ --tee "$fname.omap.vals" "$RADOS_TOOL" -p "$POOL" listomapvals $objname +done + +run_expect_succ "$RADOS_TOOL" cppool "$POOL" "$POOL_CP_TARGET" + +mkdir -p "$TDIR/dir_cp_dst" +for i in `seq 1 5`; do + fname="$TDIR/dir_cp_dst/f.$i" + objname="f.$i" + run_expect_succ "$RADOS_TOOL" -p "$POOL_CP_TARGET" get $objname "$fname" + +# a few random attrs + for j in `seq 1 4`; do + run_expect_succ --tee "$fname.attr.$j" "$RADOS_TOOL" -p "$POOL_CP_TARGET" getxattr $objname attr.$j + done + + run_expect_succ --tee "$fname.omap.header" "$RADOS_TOOL" -p "$POOL_CP_TARGET" getomapheader $objname + run_expect_succ --tee "$fname.omap.vals" "$RADOS_TOOL" -p "$POOL_CP_TARGET" listomapvals $objname +done + +diff -q -r "$TDIR/dir_cp_src" "$TDIR/dir_cp_dst" \ + || die "copy pool validation failed!" + +for opt in \ + block-size \ + concurrent-ios \ + min-object-size \ + max-object-size \ + min-op-len \ + max-op-len \ + max-ops \ + max-backlog \ + target-throughput \ + read-percent \ + num-objects \ + run-length \ + ; do + run_expect_succ "$RADOS_TOOL" --$opt 4 df + run_expect_fail "$RADOS_TOOL" --$opt 4k df +done + +run_expect_succ "$RADOS_TOOL" lock list f.1 --lock-duration 4 --pool "$POOL" +echo # previous command doesn't output an end of line: issue #9735 +run_expect_fail "$RADOS_TOOL" lock list f.1 --lock-duration 4k --pool "$POOL" + +run_expect_succ "$RADOS_TOOL" mksnap snap1 --pool "$POOL" +snapid=$("$RADOS_TOOL" lssnap --pool "$POOL" | grep snap1 | cut -f1) +[ $? -ne 0 ] && die "expected success, but got failure! cmd: \"$RADOS_TOOL\" lssnap --pool \"$POOL\" | grep snap1 | cut -f1" +run_expect_succ "$RADOS_TOOL" ls --pool "$POOL" --snapid="$snapid" +run_expect_fail "$RADOS_TOOL" ls --pool "$POOL" --snapid="$snapid"k + +run_expect_succ "$RADOS_TOOL" truncate f.1 0 --pool "$POOL" +run_expect_fail "$RADOS_TOOL" truncate f.1 0k --pool "$POOL" + +run "$CEPH_TOOL" osd pool rm delete_me_mkpool_test delete_me_mkpool_test --yes-i-really-really-mean-it +run_expect_succ "$CEPH_TOOL" osd pool create delete_me_mkpool_test 1 + +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 write +run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 1k write +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 write --format json --output "$TDIR/bench.json" +run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 1 write --output "$TDIR/bench.json" +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --format json --no-cleanup +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 rand --format json +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 rand -f json +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 seq --format json +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 seq -f json +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-omap +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-object +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-xattr +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-xattr --write-object +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-xattr --write-omap +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-omap --write-object +run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-xattr --write-omap --write-object +run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-omap +run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-object +run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-xattr +run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-xattr --write-object +run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-xattr --write-omap +run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-omap --write-object +run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-xattr --write-omap --write-object + +for i in $("$RADOS_TOOL" --pool "$POOL" ls | grep "benchmark_data"); do + "$RADOS_TOOL" --pool "$POOL" truncate $i 0 +done + +run_expect_nosignal "$RADOS_TOOL" --pool "$POOL" bench 1 rand +run_expect_nosignal "$RADOS_TOOL" --pool "$POOL" bench 1 seq + +set -e + +OBJ=test_rados_obj + +expect_false() +{ + if "$@"; then return 1; else return 0; fi +} + +cleanup() { + $RADOS_TOOL -p $POOL rm $OBJ > /dev/null 2>&1 || true + $RADOS_TOOL -p $POOL_EC rm $OBJ > /dev/null 2>&1 || true +} + +test_omap() { + cleanup + for i in $(seq 1 1 10) + do + if [ $(($i % 2)) -eq 0 ]; then + $RADOS_TOOL -p $POOL setomapval $OBJ $i $i + else + echo -n "$i" | $RADOS_TOOL -p $POOL setomapval $OBJ $i + fi + $RADOS_TOOL -p $POOL getomapval $OBJ $i | grep -q "|$i|\$" + done + $RADOS_TOOL -p $POOL listomapvals $OBJ | grep -c value | grep 10 + for i in $(seq 1 1 5) + do + $RADOS_TOOL -p $POOL rmomapkey $OBJ $i + done + $RADOS_TOOL -p $POOL listomapvals $OBJ | grep -c value | grep 5 + $RADOS_TOOL -p $POOL clearomap $OBJ + $RADOS_TOOL -p $POOL listomapvals $OBJ | wc -l | grep 0 + cleanup + + for i in $(seq 1 1 10) + do + dd if=/dev/urandom bs=128 count=1 > $TDIR/omap_key + if [ $(($i % 2)) -eq 0 ]; then + $RADOS_TOOL -p $POOL --omap-key-file $TDIR/omap_key setomapval $OBJ $i + else + echo -n "$i" | $RADOS_TOOL -p $POOL --omap-key-file $TDIR/omap_key setomapval $OBJ + fi + $RADOS_TOOL -p $POOL --omap-key-file $TDIR/omap_key getomapval $OBJ | grep -q "|$i|\$" + $RADOS_TOOL -p $POOL --omap-key-file $TDIR/omap_key rmomapkey $OBJ + $RADOS_TOOL -p $POOL listomapvals $OBJ | grep -c value | grep 0 + done + cleanup +} + +test_xattr() { + cleanup + $RADOS_TOOL -p $POOL put $OBJ /etc/passwd + V1=`mktemp fooattrXXXXXXX` + V2=`mktemp fooattrXXXXXXX` + echo -n fooval > $V1 + expect_false $RADOS_TOOL -p $POOL setxattr $OBJ 2>/dev/null + expect_false $RADOS_TOOL -p $POOL setxattr $OBJ foo fooval extraarg 2>/dev/null + $RADOS_TOOL -p $POOL setxattr $OBJ foo fooval + $RADOS_TOOL -p $POOL getxattr $OBJ foo > $V2 + cmp $V1 $V2 + cat $V1 | $RADOS_TOOL -p $POOL setxattr $OBJ bar + $RADOS_TOOL -p $POOL getxattr $OBJ bar > $V2 + cmp $V1 $V2 + $RADOS_TOOL -p $POOL listxattr $OBJ > $V1 + grep -q foo $V1 + grep -q bar $V1 + [ `cat $V1 | wc -l` -eq 2 ] + rm $V1 $V2 + cleanup +} +test_rmobj() { + p=`uuidgen` + $CEPH_TOOL osd pool create $p 1 + $CEPH_TOOL osd pool set-quota $p max_objects 1 + V1=`mktemp fooattrXXXXXXX` + $RADOS_TOOL put $OBJ $V1 -p $p + while ! $CEPH_TOOL osd dump | grep 'full_quota max_objects' + do + sleep 2 + done + $RADOS_TOOL -p $p rm $OBJ --force-full + $CEPH_TOOL osd pool rm $p $p --yes-i-really-really-mean-it + rm $V1 +} + +test_ls() { + echo "Testing rados ls command" + p=`uuidgen` + $CEPH_TOOL osd pool create $p 1 + NS=10 + OBJS=20 + # Include default namespace (0) in the total + TOTAL=$(expr $OBJS \* $(expr $NS + 1)) + + for nsnum in `seq 0 $NS` + do + for onum in `seq 1 $OBJS` + do + if [ "$nsnum" = "0" ]; + then + "$RADOS_TOOL" -p $p put obj${onum} /etc/fstab 2> /dev/null + else + "$RADOS_TOOL" -p $p -N "NS${nsnum}" put obj${onum} /etc/fstab 2> /dev/null + fi + done + done + CHECK=$("$RADOS_TOOL" -p $p ls 2> /dev/null | wc -l) + if [ "$OBJS" -ne "$CHECK" ]; + then + die "Created $OBJS objects in default namespace but saw $CHECK" + fi + TESTNS=NS${NS} + CHECK=$("$RADOS_TOOL" -p $p -N $TESTNS ls 2> /dev/null | wc -l) + if [ "$OBJS" -ne "$CHECK" ]; + then + die "Created $OBJS objects in $TESTNS namespace but saw $CHECK" + fi + CHECK=$("$RADOS_TOOL" -p $p --all ls 2> /dev/null | wc -l) + if [ "$TOTAL" -ne "$CHECK" ]; + then + die "Created $TOTAL objects but saw $CHECK" + fi + + $CEPH_TOOL osd pool rm $p $p --yes-i-really-really-mean-it +} + +test_cleanup() { + echo "Testing rados cleanup command" + p=`uuidgen` + $CEPH_TOOL osd pool create $p 1 + NS=5 + OBJS=4 + # Include default namespace (0) in the total + TOTAL=$(expr $OBJS \* $(expr $NS + 1)) + + for nsnum in `seq 0 $NS` + do + for onum in `seq 1 $OBJS` + do + if [ "$nsnum" = "0" ]; + then + "$RADOS_TOOL" -p $p put obj${onum} /etc/fstab 2> /dev/null + else + "$RADOS_TOOL" -p $p -N "NS${nsnum}" put obj${onum} /etc/fstab 2> /dev/null + fi + done + done + + $RADOS_TOOL -p $p --all ls > $TDIR/before.ls.out 2> /dev/null + + $RADOS_TOOL -p $p bench 3 write --no-cleanup 2> /dev/null + $RADOS_TOOL -p $p -N NS1 bench 3 write --no-cleanup 2> /dev/null + $RADOS_TOOL -p $p -N NS2 bench 3 write --no-cleanup 2> /dev/null + $RADOS_TOOL -p $p -N NS3 bench 3 write --no-cleanup 2> /dev/null + # Leave dangling objects without a benchmark_last_metadata in NS4 + expect_false timeout 3 $RADOS_TOOL -p $p -N NS4 bench 30 write --no-cleanup 2> /dev/null + $RADOS_TOOL -p $p -N NS5 bench 3 write --no-cleanup 2> /dev/null + + $RADOS_TOOL -p $p -N NS3 cleanup 2> /dev/null + #echo "Check NS3 after specific cleanup" + CHECK=$($RADOS_TOOL -p $p -N NS3 ls | wc -l) + if [ "$OBJS" -ne "$CHECK" ] ; + then + die "Expected $OBJS objects in NS3 but saw $CHECK" + fi + + #echo "Try to cleanup all" + $RADOS_TOOL -p $p --all cleanup + #echo "Check all namespaces" + $RADOS_TOOL -p $p --all ls > $TDIR/after.ls.out 2> /dev/null + CHECK=$(cat $TDIR/after.ls.out | wc -l) + if [ "$TOTAL" -ne "$CHECK" ]; + then + die "Expected $TOTAL objects but saw $CHECK" + fi + if ! diff $TDIR/before.ls.out $TDIR/after.ls.out + then + die "Different objects found after cleanup" + fi + + set +e + run_expect_fail $RADOS_TOOL -p $p cleanup --prefix illegal_prefix + run_expect_succ $RADOS_TOOL -p $p cleanup --prefix benchmark_data_otherhost + set -e + + $CEPH_TOOL osd pool rm $p $p --yes-i-really-really-mean-it +} + +function test_append() +{ + cleanup + + # create object + touch ./rados_append_null + $RADOS_TOOL -p $POOL append $OBJ ./rados_append_null + $RADOS_TOOL -p $POOL get $OBJ ./rados_append_0_out + cmp ./rados_append_null ./rados_append_0_out + + # append 4k, total size 4k + dd if=/dev/zero of=./rados_append_4k bs=4k count=1 + $RADOS_TOOL -p $POOL append $OBJ ./rados_append_4k + $RADOS_TOOL -p $POOL get $OBJ ./rados_append_4k_out + cmp ./rados_append_4k ./rados_append_4k_out + + # append 4k, total size 8k + $RADOS_TOOL -p $POOL append $OBJ ./rados_append_4k + $RADOS_TOOL -p $POOL get $OBJ ./rados_append_4k_out + read_size=`ls -l ./rados_append_4k_out | awk -F ' ' '{print $5}'` + if [ 8192 -ne $read_size ]; + then + die "Append failed expecting 8192 read $read_size" + fi + + # append 10M, total size 10493952 + dd if=/dev/zero of=./rados_append_10m bs=10M count=1 + $RADOS_TOOL -p $POOL append $OBJ ./rados_append_10m + $RADOS_TOOL -p $POOL get $OBJ ./rados_append_10m_out + read_size=`ls -l ./rados_append_10m_out | awk -F ' ' '{print $5}'` + if [ 10493952 -ne $read_size ]; + then + die "Append failed expecting 10493952 read $read_size" + fi + + # cleanup + cleanup + + # create object + $RADOS_TOOL -p $POOL_EC append $OBJ ./rados_append_null + $RADOS_TOOL -p $POOL_EC get $OBJ ./rados_append_0_out + cmp rados_append_null rados_append_0_out + + # append 4k, total size 4k + $RADOS_TOOL -p $POOL_EC append $OBJ ./rados_append_4k + $RADOS_TOOL -p $POOL_EC get $OBJ ./rados_append_4k_out + cmp rados_append_4k rados_append_4k_out + + # append 4k, total size 8k + $RADOS_TOOL -p $POOL_EC append $OBJ ./rados_append_4k + $RADOS_TOOL -p $POOL_EC get $OBJ ./rados_append_4k_out + read_size=`ls -l ./rados_append_4k_out | awk -F ' ' '{print $5}'` + if [ 8192 -ne $read_size ]; + then + die "Append failed expecting 8192 read $read_size" + fi + + # append 10M, total size 10493952 + $RADOS_TOOL -p $POOL_EC append $OBJ ./rados_append_10m + $RADOS_TOOL -p $POOL_EC get $OBJ ./rados_append_10m_out + read_size=`ls -l ./rados_append_10m_out | awk -F ' ' '{print $5}'` + if [ 10493952 -ne $read_size ]; + then + die "Append failed expecting 10493952 read $read_size" + fi + + cleanup + rm -rf ./rados_append_null ./rados_append_0_out + rm -rf ./rados_append_4k ./rados_append_4k_out ./rados_append_10m ./rados_append_10m_out +} + +function test_put() +{ + # rados put test: + cleanup + + # create file in local fs + dd if=/dev/urandom of=rados_object_10k bs=1K count=10 + + # test put command + $RADOS_TOOL -p $POOL put $OBJ ./rados_object_10k + $RADOS_TOOL -p $POOL get $OBJ ./rados_object_10k_out + cmp ./rados_object_10k ./rados_object_10k_out + cleanup + + # test put command with offset 0 + $RADOS_TOOL -p $POOL put $OBJ ./rados_object_10k --offset 0 + $RADOS_TOOL -p $POOL get $OBJ ./rados_object_offset_0_out + cmp ./rados_object_10k ./rados_object_offset_0_out + cleanup + + # test put command with offset 1000 + $RADOS_TOOL -p $POOL put $OBJ ./rados_object_10k --offset 1000 + $RADOS_TOOL -p $POOL get $OBJ ./rados_object_offset_1000_out + cmp ./rados_object_10k ./rados_object_offset_1000_out 0 1000 + cleanup + + rm -rf ./rados_object_10k ./rados_object_10k_out ./rados_object_offset_0_out ./rados_object_offset_1000_out +} + +function test_stat() +{ + bluestore=$("$CEPH_TOOL" osd metadata | grep '"osd_objectstore": "bluestore"' | cut -f1) + # create file in local fs + dd if=/dev/urandom of=rados_object_128k bs=64K count=2 + + # rados df test (replicated_pool): + $RADOS_TOOL purge $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool rm $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool create $POOL 8 + $CEPH_TOOL osd pool set $POOL size 3 + + # put object with 1 MB gap in front + $RADOS_TOOL -p $POOL put $OBJ ./rados_object_128k --offset=1048576 + MATCH_CNT=0 + if [ "" == "$bluestore" ]; + then + STORED=1.1 + STORED_UNIT="MiB" + else + STORED=384 + STORED_UNIT="KiB" + fi + for i in {1..60} + do + IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "") + [[ -z $IN ]] && sleep 1 && continue + IFS=' ' read -ra VALS <<< "$IN" + + # verification is a bit tricky due to stats report's eventual model + # VALS[1] - STORED + # VALS[2] - STORED units + # VALS[3] - OBJECTS + # VALS[5] - COPIES + # VALS[12] - WR_OPS + # VALS[13] - WR + # VALS[14] - WR uints + # implies replication factor 3 + if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "1" ] && [ ${VALS[13]} == 128 ] && [ ${VALS[14]} == "KiB" ] + then + # enforce multiple match to make sure stats aren't changing any more + MATCH_CNT=$((MATCH_CNT+1)) + [[ $MATCH_CNT == 3 ]] && break + sleep 1 + continue + fi + MATCH_CNT=0 + sleep 1 + continue + done + [[ -z $IN ]] && die "Failed to retrieve any pool stats within 60 seconds" + if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "1" ] || [ ${VALS[13]} != 128 ] || [ ${VALS[14]} != "KiB" ] + then + die "Failed to retrieve proper pool stats within 60 seconds" + fi + + # overwrite data at 1MB offset + $RADOS_TOOL -p $POOL put $OBJ ./rados_object_128k --offset=1048576 + MATCH_CNT=0 + if [ "" == "$bluestore" ]; + then + STORED=1.1 + STORED_UNIT="MiB" + else + STORED=384 + STORED_UNIT="KiB" + fi + for i in {1..60} + do + IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "") + IFS=' ' read -ra VALS <<< "$IN" + + # verification is a bit tricky due to stats report's eventual model + # VALS[1] - STORED + # VALS[2] - STORED units + # VALS[3] - OBJECTS + # VALS[5] - COPIES + # VALS[12] - WR_OPS + # VALS[13] - WR + # VALS[14] - WR uints + # implies replication factor 3 + if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "2" ] && [ ${VALS[13]} == 256 ] && [ ${VALS[14]} == "KiB" ] + then + # enforce multiple match to make sure stats aren't changing any more + MATCH_CNT=$((MATCH_CNT+1)) + [[ $MATCH_CNT == 3 ]] && break + sleep 1 + continue + fi + MATCH_CNT=0 + sleep 1 + continue + done + if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "2" ] || [ ${VALS[13]} != 256 ] || [ ${VALS[14]} != "KiB" ] + then + die "Failed to retrieve proper pool stats within 60 seconds" + fi + + # write data at 64K offset + $RADOS_TOOL -p $POOL put $OBJ ./rados_object_128k --offset=65536 + MATCH_CNT=0 + if [ "" == "$bluestore" ]; + then + STORED=1.1 + STORED_UNIT="MiB" + else + STORED=768 + STORED_UNIT="KiB" + fi + for i in {1..60} + do + IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "") + IFS=' ' read -ra VALS <<< "$IN" + + # verification is a bit tricky due to stats report's eventual model + # VALS[1] - STORED + # VALS[2] - STORED units + # VALS[3] - OBJECTS + # VALS[5] - COPIES + # VALS[12] - WR_OPS + # VALS[13] - WR + # VALS[14] - WR uints + # implies replication factor 3 + if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "3" ] && [ ${VALS[13]} == 384 ] && [ ${VALS[14]} == "KiB" ] + then + # enforce multiple match to make sure stats aren't changing any more + MATCH_CNT=$((MATCH_CNT+1)) + [[ $MATCH_CNT == 3 ]] && break + sleep 1 + continue + fi + MATCH_CNT=0 + sleep 1 + continue + done + if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "3" ] || [ ${VALS[13]} != 384 ] || [ ${VALS[14]} != "KiB" ] + then + die "Failed to retrieve proper pool stats within 60 seconds" + fi + + # overwrite object totally + $RADOS_TOOL -p $POOL put $OBJ ./rados_object_128k + MATCH_CNT=0 + if [ "" == "$bluestore" ]; + then + STORED=128 + STORED_UNIT="KiB" + else + STORED=384 + STORED_UNIT="KiB" + fi + for i in {1..60} + do + IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "") + IFS=' ' read -ra VALS <<< "$IN" + + # verification is a bit tricky due to stats report's eventual model + # VALS[1] - STORED + # VALS[2] - STORED units + # VALS[3] - OBJECTS + # VALS[5] - COPIES + # VALS[12] - WR_OPS + # VALS[13] - WR + # VALS[14] - WR uints + # implies replication factor 3 + if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "4" ] && [ ${VALS[13]} == 512 ] && [ ${VALS[14]} == "KiB" ] + then + # enforce multiple match to make sure stats aren't changing any more + MATCH_CNT=$((MATCH_CNT+1)) + [[ $MATCH_CNT == 3 ]] && break + sleep 1 + continue + fi + MATCH_CNT=0 + sleep 1 + continue + done + if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "4" ] || [ ${VALS[13]} != 512 ] || [ ${VALS[14]} != "KiB" ] + then + die "Failed to retrieve proper pool stats within 60 seconds" + fi + + cleanup + + # after cleanup? + MATCH_CNT=0 + for i in {1..60} + do + IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "") + IFS=' ' read -ra VALS <<< "$IN" + + # verification is a bit tricky due to stats report's eventual model + # VALS[1] - STORED + # VALS[2] - STORED units + # VALS[3] - OBJECTS + # VALS[5] - COPIES + # VALS[12] - WR_OPS + # VALS[13] - WR + # VALS[14] - WR uints + # implies replication factor 3 + if [ ${VALS[1]} == 0 ] && [ ${VALS[2]} == "B" ] && [ ${VALS[3]} == "0" ] && [ ${VALS[5]} == "0" ] && [ ${VALS[12]} == "5" ] && [ ${VALS[13]} == 512 ] && [ ${VALS[14]} == "KiB" ] + then + # enforce multiple match to make sure stats aren't changing any more + MATCH_CNT=$((MATCH_CNT+1)) + [[ $MATCH_CNT == 3 ]] && break + sleep 1 + continue + fi + MATCH_CNT=0 + sleep 1 + continue + done + if [ ${VALS[1]} != 0 ] || [ ${VALS[2]} != "B" ] || [ ${VALS[3]} != "0" ] || [ ${VALS[5]} != "0" ] || [ ${VALS[12]} != "5" ] || [ ${VALS[13]} != 512 ] || [ ${VALS[14]} != "KiB" ] + then + die "Failed to retrieve proper pool stats within 60 seconds" + fi + + ############ rados df test (EC pool): ############## + $RADOS_TOOL purge $POOL_EC --yes-i-really-really-mean-it + $CEPH_TOOL osd pool rm $POOL_EC $POOL_EC --yes-i-really-really-mean-it + $CEPH_TOOL osd erasure-code-profile set myprofile k=2 m=1 stripe_unit=2K crush-failure-domain=osd --force + $CEPH_TOOL osd pool create $POOL_EC 8 8 erasure + + # put object + $RADOS_TOOL -p $POOL_EC put $OBJ ./rados_object_128k + MATCH_CNT=0 + if [ "" == "$bluestore" ]; + then + STORED=128 + STORED_UNIT="KiB" + else + STORED=192 + STORED_UNIT="KiB" + fi + for i in {1..60} + do + IN=$($RADOS_TOOL -p $POOL_EC df | grep $POOL_EC ; [[ ! -z $? ]] && echo "") + [[ -z $IN ]] && sleep 1 && continue + IFS=' ' read -ra VALS <<< "$IN" + + # verification is a bit tricky due to stats report's eventual model + # VALS[1] - STORED + # VALS[2] - STORED units + # VALS[3] - OBJECTS + # VALS[5] - COPIES + # VALS[12] - WR_OPS + # VALS[13] - WR + # VALS[14] - WR uints + # implies replication factor 2+1 + if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "1" ] && [ ${VALS[13]} == 128 ] && [ ${VALS[14]} == "KiB" ] + then + # enforce multiple match to make sure stats aren't changing any more + MATCH_CNT=$((MATCH_CNT+1)) + [[ $MATCH_CNT == 3 ]] && break + sleep 1 + continue + fi + MATCH_CNT=0 + sleep 1 + continue + done + [[ -z $IN ]] && die "Failed to retrieve any pool stats within 60 seconds" + if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "1" ] || [ ${VALS[13]} != 128 ] || [ ${VALS[14]} != "KiB" ] + then + die "Failed to retrieve proper pool stats within 60 seconds" + fi + + # overwrite object + $RADOS_TOOL -p $POOL_EC put $OBJ ./rados_object_128k + MATCH_CNT=0 + if [ "" == "$bluestore" ]; + then + STORED=128 + STORED_UNIT="KiB" + else + STORED=192 + STORED_UNIT="KiB" + fi + for i in {1..60} + do + IN=$($RADOS_TOOL -p $POOL_EC df | grep $POOL_EC ; [[ ! -z $? ]] && echo "") + IFS=' ' read -ra VALS <<< "$IN" + + # verification is a bit tricky due to stats report's eventual model + # VALS[1] - STORED + # VALS[2] - STORED units + # VALS[3] - OBJECTS + # VALS[5] - COPIES + # VALS[12] - WR_OPS + # VALS[13] - WR + # VALS[14] - WR uints + # implies replication factor 2+1 + if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "2" ] && [ ${VALS[13]} == 256 ] && [ ${VALS[14]} == "KiB" ] + then + # enforce multiple match to make sure stats aren't changing any more + MATCH_CNT=$((MATCH_CNT+1)) + [[ $MATCH_CNT == 3 ]] && break + sleep 1 + continue + fi + MATCH_CNT=0 + sleep 1 + continue + done + if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "2" ] || [ ${VALS[13]} != 256 ] || [ ${VALS[14]} != "KiB" ] + then + die "Failed to retrieve proper pool stats within 60 seconds" + fi + + cleanup + + # after cleanup? + MATCH_CNT=0 + for i in {1..60} + do + IN=$($RADOS_TOOL -p $POOL_EC df | grep $POOL_EC ; [[ ! -z $? ]] && echo "") + IFS=' ' read -ra VALS <<< "$IN" + + # verification is a bit tricky due to stats report's eventual model + # VALS[1] - STORED + # VALS[2] - STORED units + # VALS[3] - OBJECTS + # VALS[5] - COPIES + # VALS[12] - WR_OPS + # VALS[13] - WR + # VALS[14] - WR uints + # implies replication factor 2+1 + if [ ${VALS[1]} == 0 ] && [ ${VALS[2]} == "B" ] && [ ${VALS[3]} == "0" ] && [ ${VALS[5]} == "0" ] && [ ${VALS[12]} == "3" ] && [ ${VALS[13]} == 256 ] && [ ${VALS[14]} == "KiB" ] + then + # enforce multiple match to make sure stats aren't changing any more + MATCH_CNT=$((MATCH_CNT+1)) + [[ $MATCH_CNT == 3 ]] && break + sleep 1 + continue + fi + MATCH_CNT=0 + sleep 1 + continue + done + if [ ${VALS[1]} != 0 ] || [ ${VALS[2]} != "B" ] || [ ${VALS[3]} != "0" ] || [ ${VALS[5]} != "0" ] || [ ${VALS[12]} != "3" ] || [ ${VALS[13]} != 256 ] || [ ${VALS[14]} != "KiB" ] + then + die "Failed to retrieve proper pool stats within 60 seconds" + fi + + rm -rf ./rados_object_128k +} + +test_xattr +test_omap +test_rmobj +test_ls +test_cleanup +test_append +test_put +test_stat + +# clean up environment, delete pool +$CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it +$CEPH_TOOL osd pool delete $POOL_EC $POOL_EC --yes-i-really-really-mean-it +$CEPH_TOOL osd pool delete $POOL_CP_TARGET $POOL_CP_TARGET --yes-i-really-really-mean-it + +echo "SUCCESS!" +exit 0 diff --git a/qa/workunits/rados/version_number_sanity.sh b/qa/workunits/rados/version_number_sanity.sh new file mode 100755 index 000000000..e7eb9be64 --- /dev/null +++ b/qa/workunits/rados/version_number_sanity.sh @@ -0,0 +1,30 @@ +#!/bin/bash -ex +# +# test that ceph RPM/DEB package version matches "ceph --version" +# (for a loose definition of "matches") +# +source /etc/os-release +case $ID in +debian|ubuntu) + RPMDEB='DEB' + dpkg-query --show ceph-common + PKG_NAME_AND_VERSION=$(dpkg-query --show ceph-common) + ;; +centos|fedora|rhel|opensuse*|suse|sles) + RPMDEB='RPM' + rpm -q ceph + PKG_NAME_AND_VERSION=$(rpm -q ceph) + ;; +*) + echo "Unsupported distro ->$ID<-! Bailing out." + exit 1 +esac +PKG_CEPH_VERSION=$(perl -e '"'"$PKG_NAME_AND_VERSION"'" =~ m/(\d+(\.\d+)+)/; print "$1\n";') +echo "According to $RPMDEB package, the ceph version under test is ->$PKG_CEPH_VERSION<-" +test -n "$PKG_CEPH_VERSION" +ceph --version +BUFFER=$(ceph --version) +CEPH_CEPH_VERSION=$(perl -e '"'"$BUFFER"'" =~ m/ceph version (\d+(\.\d+)+)/; print "$1\n";') +echo "According to \"ceph --version\", the ceph version under test is ->$CEPH_CEPH_VERSION<-" +test -n "$CEPH_CEPH_VERSION" +test "$PKG_CEPH_VERSION" = "$CEPH_CEPH_VERSION" diff --git a/qa/workunits/rbd/cli_generic.sh b/qa/workunits/rbd/cli_generic.sh new file mode 100755 index 000000000..57279d26d --- /dev/null +++ b/qa/workunits/rbd/cli_generic.sh @@ -0,0 +1,1715 @@ +#!/usr/bin/env bash +set -ex + +. $(dirname $0)/../../standalone/ceph-helpers.sh + +export RBD_FORCE_ALLOW_V1=1 + +# make sure rbd pool is EMPTY.. this is a test script!! +rbd ls | wc -l | grep -v '^0$' && echo "nonempty rbd pool, aborting! run this script on an empty test cluster only." && exit 1 + +IMGS="testimg1 testimg2 testimg3 testimg4 testimg5 testimg6 testimg-diff1 testimg-diff2 testimg-diff3 foo foo2 bar bar2 test1 test2 test3 test4 clone2" + +expect_fail() { + "$@" && return 1 || return 0 +} + +tiered=0 +if ceph osd dump | grep ^pool | grep "'rbd'" | grep tier; then + tiered=1 +fi + +remove_images() { + for img in $IMGS + do + (rbd snap purge $img || true) >/dev/null 2>&1 + (rbd rm $img || true) >/dev/null 2>&1 + done +} + +test_others() { + echo "testing import, export, resize, and snapshots..." + TMP_FILES="/tmp/img1 /tmp/img1.new /tmp/img2 /tmp/img2.new /tmp/img3 /tmp/img3.new /tmp/img-diff1.new /tmp/img-diff2.new /tmp/img-diff3.new /tmp/img1.snap1 /tmp/img1.snap1 /tmp/img-diff1.snap1" + + remove_images + rm -f $TMP_FILES + + # create an image + dd if=/bin/sh of=/tmp/img1 bs=1k count=1 seek=10 + dd if=/bin/dd of=/tmp/img1 bs=1k count=10 seek=100 + dd if=/bin/rm of=/tmp/img1 bs=1k count=100 seek=1000 + dd if=/bin/ls of=/tmp/img1 bs=1k seek=10000 + dd if=/bin/ln of=/tmp/img1 bs=1k seek=100000 + + # import, snapshot + rbd import $RBD_CREATE_ARGS /tmp/img1 testimg1 + rbd resize testimg1 --size=256 --allow-shrink + rbd export testimg1 /tmp/img2 + rbd snap create testimg1 --snap=snap1 + rbd resize testimg1 --size=128 && exit 1 || true # shrink should fail + rbd resize testimg1 --size=128 --allow-shrink + rbd export testimg1 /tmp/img3 + + # info + rbd info testimg1 | grep 'size 128 MiB' + rbd info --snap=snap1 testimg1 | grep 'size 256 MiB' + + # export-diff + rm -rf /tmp/diff-testimg1-1 /tmp/diff-testimg1-2 + rbd export-diff testimg1 --snap=snap1 /tmp/diff-testimg1-1 + rbd export-diff testimg1 --from-snap=snap1 /tmp/diff-testimg1-2 + + # import-diff + rbd create $RBD_CREATE_ARGS --size=1 testimg-diff1 + rbd import-diff --sparse-size 8K /tmp/diff-testimg1-1 testimg-diff1 + rbd import-diff --sparse-size 8K /tmp/diff-testimg1-2 testimg-diff1 + + # info + rbd info testimg1 | grep 'size 128 MiB' + rbd info --snap=snap1 testimg1 | grep 'size 256 MiB' + rbd info testimg-diff1 | grep 'size 128 MiB' + rbd info --snap=snap1 testimg-diff1 | grep 'size 256 MiB' + + # make copies + rbd copy testimg1 --snap=snap1 testimg2 + rbd copy testimg1 testimg3 + rbd copy testimg-diff1 --sparse-size 768K --snap=snap1 testimg-diff2 + rbd copy testimg-diff1 --sparse-size 768K testimg-diff3 + + # verify the result + rbd info testimg2 | grep 'size 256 MiB' + rbd info testimg3 | grep 'size 128 MiB' + rbd info testimg-diff2 | grep 'size 256 MiB' + rbd info testimg-diff3 | grep 'size 128 MiB' + + # deep copies + rbd deep copy testimg1 testimg4 + rbd deep copy testimg1 --snap=snap1 testimg5 + rbd info testimg4 | grep 'size 128 MiB' + rbd info testimg5 | grep 'size 256 MiB' + rbd snap ls testimg4 | grep -v 'SNAPID' | wc -l | grep 1 + rbd snap ls testimg4 | grep '.*snap1.*' + + rbd export testimg1 /tmp/img1.new + rbd export testimg2 /tmp/img2.new + rbd export testimg3 /tmp/img3.new + rbd export testimg-diff1 /tmp/img-diff1.new + rbd export testimg-diff2 /tmp/img-diff2.new + rbd export testimg-diff3 /tmp/img-diff3.new + + cmp /tmp/img2 /tmp/img2.new + cmp /tmp/img3 /tmp/img3.new + cmp /tmp/img2 /tmp/img-diff2.new + cmp /tmp/img3 /tmp/img-diff3.new + + # rollback + rbd snap rollback --snap=snap1 testimg1 + rbd snap rollback --snap=snap1 testimg-diff1 + rbd info testimg1 | grep 'size 256 MiB' + rbd info testimg-diff1 | grep 'size 256 MiB' + rbd export testimg1 /tmp/img1.snap1 + rbd export testimg-diff1 /tmp/img-diff1.snap1 + cmp /tmp/img2 /tmp/img1.snap1 + cmp /tmp/img2 /tmp/img-diff1.snap1 + + # test create, copy of zero-length images + rbd rm testimg2 + rbd rm testimg3 + rbd create testimg2 -s 0 + rbd cp testimg2 testimg3 + rbd deep cp testimg2 testimg6 + + # remove snapshots + rbd snap rm --snap=snap1 testimg1 + rbd snap rm --snap=snap1 testimg-diff1 + rbd info --snap=snap1 testimg1 2>&1 | grep 'error setting snapshot context: (2) No such file or directory' + rbd info --snap=snap1 testimg-diff1 2>&1 | grep 'error setting snapshot context: (2) No such file or directory' + + # sparsify + rbd sparsify testimg1 + + remove_images + rm -f $TMP_FILES +} + +test_rename() { + echo "testing rename..." + remove_images + + rbd create --image-format 1 -s 1 foo + rbd create --image-format 2 -s 1 bar + rbd rename foo foo2 + rbd rename foo2 bar 2>&1 | grep exists + rbd rename bar bar2 + rbd rename bar2 foo2 2>&1 | grep exists + + ceph osd pool create rbd2 8 + rbd pool init rbd2 + rbd create -p rbd2 -s 1 foo + rbd rename rbd2/foo rbd2/bar + rbd -p rbd2 ls | grep bar + rbd rename rbd2/bar foo + rbd rename --pool rbd2 foo bar + ! rbd rename rbd2/bar --dest-pool rbd foo + rbd rename --pool rbd2 bar --dest-pool rbd2 foo + rbd -p rbd2 ls | grep foo + ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it + + remove_images +} + +test_ls() { + echo "testing ls..." + remove_images + + rbd create --image-format 1 -s 1 test1 + rbd create --image-format 1 -s 1 test2 + rbd ls | grep test1 + rbd ls | grep test2 + rbd ls | wc -l | grep 2 + # look for fields in output of ls -l without worrying about space + rbd ls -l | grep 'test1.*1 MiB.*1' + rbd ls -l | grep 'test2.*1 MiB.*1' + + rbd rm test1 + rbd rm test2 + + rbd create --image-format 2 -s 1 test1 + rbd create --image-format 2 -s 1 test2 + rbd ls | grep test1 + rbd ls | grep test2 + rbd ls | wc -l | grep 2 + rbd ls -l | grep 'test1.*1 MiB.*2' + rbd ls -l | grep 'test2.*1 MiB.*2' + + rbd rm test1 + rbd rm test2 + + rbd create --image-format 2 -s 1 test1 + rbd create --image-format 1 -s 1 test2 + rbd ls | grep test1 + rbd ls | grep test2 + rbd ls | wc -l | grep 2 + rbd ls -l | grep 'test1.*1 MiB.*2' + rbd ls -l | grep 'test2.*1 MiB.*1' + remove_images + + # test that many images can be shown by ls + for i in $(seq -w 00 99); do + rbd create image.$i -s 1 + done + rbd ls | wc -l | grep 100 + rbd ls -l | grep image | wc -l | grep 100 + for i in $(seq -w 00 99); do + rbd rm image.$i + done + + for i in $(seq -w 00 99); do + rbd create image.$i --image-format 2 -s 1 + done + rbd ls | wc -l | grep 100 + rbd ls -l | grep image | wc -l | grep 100 + for i in $(seq -w 00 99); do + rbd rm image.$i + done +} + +test_remove() { + echo "testing remove..." + remove_images + + rbd remove "NOT_EXIST" && exit 1 || true # remove should fail + rbd create --image-format 1 -s 1 test1 + rbd rm test1 + rbd ls | wc -l | grep "^0$" + + rbd create --image-format 2 -s 1 test2 + rbd rm test2 + rbd ls | wc -l | grep "^0$" + + # check that remove succeeds even if it's + # interrupted partway through. simulate this + # by removing some objects manually. + + # remove with header missing (old format) + rbd create --image-format 1 -s 1 test1 + rados rm -p rbd test1.rbd + rbd rm test1 + rbd ls | wc -l | grep "^0$" + + if [ $tiered -eq 0 ]; then + # remove with header missing + rbd create --image-format 2 -s 1 test2 + HEADER=$(rados -p rbd ls | grep '^rbd_header') + rados -p rbd rm $HEADER + rbd rm test2 + rbd ls | wc -l | grep "^0$" + + # remove with id missing + rbd create --image-format 2 -s 1 test2 + rados -p rbd rm rbd_id.test2 + rbd rm test2 + rbd ls | wc -l | grep "^0$" + + # remove with header and id missing + rbd create --image-format 2 -s 1 test2 + HEADER=$(rados -p rbd ls | grep '^rbd_header') + rados -p rbd rm $HEADER + rados -p rbd rm rbd_id.test2 + rbd rm test2 + rbd ls | wc -l | grep "^0$" + fi + + # remove with rbd_children object missing (and, by extension, + # with child not mentioned in rbd_children) + rbd create --image-format 2 -s 1 test2 + rbd snap create test2@snap + rbd snap protect test2@snap + rbd clone test2@snap clone --rbd-default-clone-format 1 + + rados -p rbd rm rbd_children + rbd rm clone + rbd ls | grep clone | wc -l | grep '^0$' + + rbd snap unprotect test2@snap + rbd snap rm test2@snap + rbd rm test2 +} + +test_locking() { + echo "testing locking..." + remove_images + + rbd create $RBD_CREATE_ARGS -s 1 test1 + rbd lock list test1 | wc -l | grep '^0$' + rbd lock add test1 id + rbd lock list test1 | grep ' 1 ' + LOCKER=$(rbd lock list test1 | tail -n 1 | awk '{print $1;}') + rbd lock remove test1 id $LOCKER + rbd lock list test1 | wc -l | grep '^0$' + + rbd lock add test1 id --shared tag + rbd lock list test1 | grep ' 1 ' + rbd lock add test1 id --shared tag + rbd lock list test1 | grep ' 2 ' + rbd lock add test1 id2 --shared tag + rbd lock list test1 | grep ' 3 ' + rbd lock list test1 | tail -n 1 | awk '{print $2, $1;}' | xargs rbd lock remove test1 + if rbd info test1 | grep -qE "features:.*exclusive" + then + # new locking functionality requires all locks to be released + while [ -n "$(rbd lock list test1)" ] + do + rbd lock list test1 | tail -n 1 | awk '{print $2, $1;}' | xargs rbd lock remove test1 + done + fi + rbd rm test1 +} + +test_pool_image_args() { + echo "testing pool and image args..." + remove_images + + ceph osd pool delete test test --yes-i-really-really-mean-it || true + ceph osd pool create test 32 + rbd pool init test + truncate -s 1 /tmp/empty /tmp/empty@snap + + rbd ls | wc -l | grep 0 + rbd create -s 1 test1 + rbd ls | grep -q test1 + rbd import --image test2 /tmp/empty + rbd ls | grep -q test2 + rbd --dest test3 import /tmp/empty + rbd ls | grep -q test3 + rbd import /tmp/empty foo + rbd ls | grep -q foo + + # should fail due to "destination snapname specified" + rbd import --dest test/empty@snap /tmp/empty && exit 1 || true + rbd import /tmp/empty test/empty@snap && exit 1 || true + rbd import --image test/empty@snap /tmp/empty && exit 1 || true + rbd import /tmp/empty@snap && exit 1 || true + + rbd ls test | wc -l | grep 0 + rbd import /tmp/empty test/test1 + rbd ls test | grep -q test1 + rbd -p test import /tmp/empty test2 + rbd ls test | grep -q test2 + rbd --image test3 -p test import /tmp/empty + rbd ls test | grep -q test3 + rbd --image test4 -p test import /tmp/empty + rbd ls test | grep -q test4 + rbd --dest test5 -p test import /tmp/empty + rbd ls test | grep -q test5 + rbd --dest test6 --dest-pool test import /tmp/empty + rbd ls test | grep -q test6 + rbd --image test7 --dest-pool test import /tmp/empty + rbd ls test | grep -q test7 + rbd --image test/test8 import /tmp/empty + rbd ls test | grep -q test8 + rbd --dest test/test9 import /tmp/empty + rbd ls test | grep -q test9 + rbd import --pool test /tmp/empty + rbd ls test | grep -q empty + + # copy with no explicit pool goes to pool rbd + rbd copy test/test9 test10 + rbd ls test | grep -qv test10 + rbd ls | grep -q test10 + rbd copy test/test9 test/test10 + rbd ls test | grep -q test10 + rbd copy --pool test test10 --dest-pool test test11 + rbd ls test | grep -q test11 + rbd copy --dest-pool rbd --pool test test11 test12 + rbd ls | grep test12 + rbd ls test | grep -qv test12 + + rm -f /tmp/empty /tmp/empty@snap + ceph osd pool delete test test --yes-i-really-really-mean-it + + for f in foo test1 test10 test12 test2 test3 ; do + rbd rm $f + done +} + +test_clone() { + echo "testing clone..." + remove_images + rbd create test1 $RBD_CREATE_ARGS -s 1 + rbd snap create test1@s1 + rbd snap protect test1@s1 + + ceph osd pool create rbd2 8 + rbd pool init rbd2 + rbd clone test1@s1 rbd2/clone + rbd -p rbd2 ls | grep clone + rbd -p rbd2 ls -l | grep clone | grep test1@s1 + rbd ls | grep -v clone + rbd flatten rbd2/clone + rbd snap create rbd2/clone@s1 + rbd snap protect rbd2/clone@s1 + rbd clone rbd2/clone@s1 clone2 + rbd ls | grep clone2 + rbd ls -l | grep clone2 | grep rbd2/clone@s1 + rbd -p rbd2 ls | grep -v clone2 + + rbd rm clone2 + rbd snap unprotect rbd2/clone@s1 + rbd snap rm rbd2/clone@s1 + rbd rm rbd2/clone + rbd snap unprotect test1@s1 + rbd snap rm test1@s1 + rbd rm test1 + ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it +} + +test_trash() { + echo "testing trash..." + remove_images + + rbd create $RBD_CREATE_ARGS -s 1 test1 + rbd create $RBD_CREATE_ARGS -s 1 test2 + rbd ls | grep test1 + rbd ls | grep test2 + rbd ls | wc -l | grep 2 + rbd ls -l | grep 'test1.*2.*' + rbd ls -l | grep 'test2.*2.*' + + rbd trash mv test1 + rbd ls | grep test2 + rbd ls | wc -l | grep 1 + rbd ls -l | grep 'test2.*2.*' + + rbd trash ls | grep test1 + rbd trash ls | wc -l | grep 1 + rbd trash ls -l | grep 'test1.*USER.*' + rbd trash ls -l | grep -v 'protected until' + + ID=`rbd trash ls | cut -d ' ' -f 1` + rbd trash rm $ID + + rbd trash mv test2 + ID=`rbd trash ls | cut -d ' ' -f 1` + rbd info --image-id $ID | grep "rbd image 'test2'" + + rbd trash restore $ID + rbd ls | grep test2 + rbd ls | wc -l | grep 1 + rbd ls -l | grep 'test2.*2.*' + + rbd trash mv test2 --expires-at "3600 sec" + rbd trash ls | grep test2 + rbd trash ls | wc -l | grep 1 + rbd trash ls -l | grep 'test2.*USER.*protected until' + + rbd trash rm $ID 2>&1 | grep 'Deferment time has not expired' + rbd trash rm --image-id $ID --force + + rbd create $RBD_CREATE_ARGS -s 1 test1 + rbd snap create test1@snap1 + rbd snap protect test1@snap1 + rbd trash mv test1 + + rbd trash ls | grep test1 + rbd trash ls | wc -l | grep 1 + rbd trash ls -l | grep 'test1.*USER.*' + rbd trash ls -l | grep -v 'protected until' + + ID=`rbd trash ls | cut -d ' ' -f 1` + rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 1 + rbd snap ls --image-id $ID | grep '.*snap1.*' + + rbd snap unprotect --image-id $ID --snap snap1 + rbd snap rm --image-id $ID --snap snap1 + rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 0 + + rbd trash restore $ID + rbd snap create test1@snap1 + rbd snap create test1@snap2 + rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 2 + rbd snap purge --image-id $ID + rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 0 + + rbd rm --rbd_move_to_trash_on_remove=true --rbd_move_to_trash_on_remove_expire_seconds=3600 test1 + rbd trash ls | grep test1 + rbd trash ls | wc -l | grep 1 + rbd trash ls -l | grep 'test1.*USER.*protected until' + rbd trash rm $ID 2>&1 | grep 'Deferment time has not expired' + rbd trash rm --image-id $ID --force + + remove_images +} + +test_purge() { + echo "testing trash purge..." + remove_images + + rbd trash ls | wc -l | grep 0 + rbd trash purge + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash ls | wc -l | grep 2 + rbd trash purge + rbd trash ls | wc -l | grep 0 + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd trash mv testimg1 --expires-at "1 hour" + rbd trash mv testimg2 --expires-at "3 hours" + rbd trash ls | wc -l | grep 2 + rbd trash purge + rbd trash ls | wc -l | grep 2 + rbd trash purge --expired-before "now + 2 hours" + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg2 + rbd trash purge --expired-before "now + 4 hours" + rbd trash ls | wc -l | grep 0 + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap # pin testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash ls | wc -l | grep 3 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg1 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID + rbd trash purge + rbd trash ls | wc -l | grep 0 + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd snap create testimg2@snap # pin testimg2 + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash ls | wc -l | grep 3 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg2 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID + rbd trash purge + rbd trash ls | wc -l | grep 0 + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd create $RBD_CREATE_ARGS --size 256 testimg2 + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg3@snap # pin testimg3 + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash ls | wc -l | grep 3 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg3 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID + rbd trash purge + rbd trash ls | wc -l | grep 0 + + # test purging a clone with a chain of parents + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap + rbd clone --rbd-default-clone-format=2 testimg1@snap testimg2 + rbd snap rm testimg1@snap + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg2@snap + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg4 + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg5 + rbd snap rm testimg2@snap + rbd snap create testimg4@snap + rbd clone --rbd-default-clone-format=2 testimg4@snap testimg6 + rbd snap rm testimg4@snap + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash mv testimg4 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 3 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash ls | grep testimg4 + rbd trash mv testimg6 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 2 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash mv testimg5 + rbd trash ls | wc -l | grep 3 + rbd trash purge + rbd trash ls | wc -l | grep 0 + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap + rbd clone --rbd-default-clone-format=2 testimg1@snap testimg2 + rbd snap rm testimg1@snap + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg3@snap # pin testimg3 + rbd snap create testimg2@snap + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg4 + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg5 + rbd snap rm testimg2@snap + rbd snap create testimg4@snap + rbd clone --rbd-default-clone-format=2 testimg4@snap testimg6 + rbd snap rm testimg4@snap + rbd trash mv testimg1 + rbd trash mv testimg2 + rbd trash mv testimg3 + rbd trash mv testimg4 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 4 + rbd trash mv testimg6 + rbd trash ls | wc -l | grep 5 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 3 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash ls | grep testimg3 + rbd trash mv testimg5 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg3 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID + rbd trash purge + rbd trash ls | wc -l | grep 0 + + # test purging a clone with a chain of auto-delete parents + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap + rbd clone --rbd-default-clone-format=2 testimg1@snap testimg2 + rbd snap rm testimg1@snap + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg2@snap + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg4 + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg5 + rbd snap rm testimg2@snap + rbd snap create testimg4@snap + rbd clone --rbd-default-clone-format=2 testimg4@snap testimg6 + rbd snap rm testimg4@snap + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg1 + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg2 + rbd trash mv testimg3 + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg4 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 3 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash ls | grep testimg4 + rbd trash mv testimg6 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 2 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash mv testimg5 + rbd trash ls | wc -l | grep 3 + rbd trash purge + rbd trash ls | wc -l | grep 0 + + rbd create $RBD_CREATE_ARGS --size 256 testimg1 + rbd snap create testimg1@snap + rbd clone --rbd-default-clone-format=2 testimg1@snap testimg2 + rbd snap rm testimg1@snap + rbd create $RBD_CREATE_ARGS --size 256 testimg3 + rbd snap create testimg3@snap # pin testimg3 + rbd snap create testimg2@snap + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg4 + rbd clone --rbd-default-clone-format=2 testimg2@snap testimg5 + rbd snap rm testimg2@snap + rbd snap create testimg4@snap + rbd clone --rbd-default-clone-format=2 testimg4@snap testimg6 + rbd snap rm testimg4@snap + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg1 + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg2 + rbd trash mv testimg3 + rbd rm --rbd_move_parent_to_trash_on_remove=true testimg4 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 4 + rbd trash mv testimg6 + rbd trash ls | wc -l | grep 5 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 3 + rbd trash ls | grep testimg1 + rbd trash ls | grep testimg2 + rbd trash ls | grep testimg3 + rbd trash mv testimg5 + rbd trash ls | wc -l | grep 4 + rbd trash purge 2>&1 | grep 'some expired images could not be removed' + rbd trash ls | wc -l | grep 1 + rbd trash ls | grep testimg3 + ID=$(rbd trash ls | awk '{ print $1 }') + rbd snap purge --image-id $ID + rbd trash purge + rbd trash ls | wc -l | grep 0 +} + +test_deep_copy_clone() { + echo "testing deep copy clone..." + remove_images + + rbd create testimg1 $RBD_CREATE_ARGS --size 256 + rbd snap create testimg1 --snap=snap1 + rbd snap protect testimg1@snap1 + rbd clone testimg1@snap1 testimg2 + rbd snap create testimg2@snap2 + rbd deep copy testimg2 testimg3 + rbd info testimg3 | grep 'size 256 MiB' + rbd info testimg3 | grep 'parent: rbd/testimg1@snap1' + rbd snap ls testimg3 | grep -v 'SNAPID' | wc -l | grep 1 + rbd snap ls testimg3 | grep '.*snap2.*' + rbd info testimg2 | grep 'features:.*deep-flatten' || rbd snap rm testimg2@snap2 + rbd info testimg3 | grep 'features:.*deep-flatten' || rbd snap rm testimg3@snap2 + rbd flatten testimg2 + rbd flatten testimg3 + rbd snap unprotect testimg1@snap1 + rbd snap purge testimg2 + rbd snap purge testimg3 + rbd rm testimg2 + rbd rm testimg3 + + rbd snap protect testimg1@snap1 + rbd clone testimg1@snap1 testimg2 + rbd snap create testimg2@snap2 + rbd deep copy --flatten testimg2 testimg3 + rbd info testimg3 | grep 'size 256 MiB' + rbd info testimg3 | grep -v 'parent:' + rbd snap ls testimg3 | grep -v 'SNAPID' | wc -l | grep 1 + rbd snap ls testimg3 | grep '.*snap2.*' + rbd info testimg2 | grep 'features:.*deep-flatten' || rbd snap rm testimg2@snap2 + rbd flatten testimg2 + rbd snap unprotect testimg1@snap1 + + remove_images +} + +test_clone_v2() { + echo "testing clone v2..." + remove_images + + rbd create $RBD_CREATE_ARGS -s 1 test1 + rbd snap create test1@1 + rbd clone --rbd-default-clone-format=1 test1@1 test2 && exit 1 || true + rbd clone --rbd-default-clone-format=2 test1@1 test2 + rbd clone --rbd-default-clone-format=2 test1@1 test3 + + rbd snap protect test1@1 + rbd clone --rbd-default-clone-format=1 test1@1 test4 + + rbd children test1@1 | sort | tr '\n' ' ' | grep -E "test2.*test3.*test4" + rbd children --descendants test1 | sort | tr '\n' ' ' | grep -E "test2.*test3.*test4" + + rbd remove test4 + rbd snap unprotect test1@1 + + rbd snap remove test1@1 + rbd snap list --all test1 | grep -E "trash \(1\) *$" + + rbd snap create test1@2 + rbd rm test1 2>&1 | grep 'image has snapshots' + + rbd snap rm test1@2 + rbd rm test1 2>&1 | grep 'linked clones' + + rbd rm test3 + rbd rm test1 2>&1 | grep 'linked clones' + + rbd flatten test2 + rbd snap list --all test1 | wc -l | grep '^0$' + rbd rm test1 + rbd rm test2 + + rbd create $RBD_CREATE_ARGS -s 1 test1 + rbd snap create test1@1 + rbd snap create test1@2 + rbd clone test1@1 test2 --rbd-default-clone-format 2 + rbd clone test1@2 test3 --rbd-default-clone-format 2 + rbd snap rm test1@1 + rbd snap rm test1@2 + expect_fail rbd rm test1 + rbd rm test1 --rbd-move-parent-to-trash-on-remove=true + rbd trash ls -a | grep test1 + rbd rm test2 + rbd trash ls -a | grep test1 + rbd rm test3 + rbd trash ls -a | expect_fail grep test1 +} + +test_thick_provision() { + echo "testing thick provision..." + remove_images + + # Try to create small and large thick-pro image and + # check actual size. (64M and 4G) + + # Small thick-pro image test + rbd create $RBD_CREATE_ARGS --thick-provision -s 64M test1 + count=0 + ret="" + while [ $count -lt 10 ] + do + rbd du|grep test1|tr -s " "|cut -d " " -f 4-5|grep '^64 MiB' && ret=$? + if [ "$ret" = "0" ] + then + break; + fi + count=`expr $count + 1` + sleep 2 + done + rbd du + if [ "$ret" != "0" ] + then + exit 1 + fi + rbd rm test1 + rbd ls | grep test1 | wc -l | grep '^0$' + + # Large thick-pro image test + rbd create $RBD_CREATE_ARGS --thick-provision -s 4G test1 + count=0 + ret="" + while [ $count -lt 10 ] + do + rbd du|grep test1|tr -s " "|cut -d " " -f 4-5|grep '^4 GiB' && ret=$? + if [ "$ret" = "0" ] + then + break; + fi + count=`expr $count + 1` + sleep 2 + done + rbd du + if [ "$ret" != "0" ] + then + exit 1 + fi + rbd rm test1 + rbd ls | grep test1 | wc -l | grep '^0$' +} + +test_namespace() { + echo "testing namespace..." + remove_images + + rbd namespace ls | wc -l | grep '^0$' + rbd namespace create rbd/test1 + rbd namespace create --pool rbd --namespace test2 + rbd namespace create --namespace test3 + expect_fail rbd namespace create rbd/test3 + + rbd namespace list | grep 'test' | wc -l | grep '^3$' + + expect_fail rbd namespace remove --pool rbd missing + + rbd create $RBD_CREATE_ARGS --size 1G rbd/test1/image1 + + # default test1 ns to test2 ns clone + rbd bench --io-type write --io-pattern rand --io-total 32M --io-size 4K rbd/test1/image1 + rbd snap create rbd/test1/image1@1 + rbd clone --rbd-default-clone-format 2 rbd/test1/image1@1 rbd/test2/image1 + rbd snap rm rbd/test1/image1@1 + cmp <(rbd export rbd/test1/image1 -) <(rbd export rbd/test2/image1 -) + rbd rm rbd/test2/image1 + + # default ns to test1 ns clone + rbd create $RBD_CREATE_ARGS --size 1G rbd/image2 + rbd bench --io-type write --io-pattern rand --io-total 32M --io-size 4K rbd/image2 + rbd snap create rbd/image2@1 + rbd clone --rbd-default-clone-format 2 rbd/image2@1 rbd/test2/image2 + rbd snap rm rbd/image2@1 + cmp <(rbd export rbd/image2 -) <(rbd export rbd/test2/image2 -) + expect_fail rbd rm rbd/image2 + rbd rm rbd/test2/image2 + rbd rm rbd/image2 + + # v1 clones are supported within the same namespace + rbd create $RBD_CREATE_ARGS --size 1G rbd/test1/image3 + rbd snap create rbd/test1/image3@1 + rbd snap protect rbd/test1/image3@1 + rbd clone --rbd-default-clone-format 1 rbd/test1/image3@1 rbd/test1/image4 + rbd rm rbd/test1/image4 + rbd snap unprotect rbd/test1/image3@1 + rbd snap rm rbd/test1/image3@1 + rbd rm rbd/test1/image3 + + rbd create $RBD_CREATE_ARGS --size 1G --namespace test1 image2 + expect_fail rbd namespace remove rbd/test1 + + rbd group create rbd/test1/group1 + rbd group image add rbd/test1/group1 rbd/test1/image1 + rbd group rm rbd/test1/group1 + + rbd trash move rbd/test1/image1 + ID=`rbd trash --namespace test1 ls | cut -d ' ' -f 1` + rbd trash rm rbd/test1/${ID} + + rbd remove rbd/test1/image2 + + rbd namespace remove --pool rbd --namespace test1 + rbd namespace remove --namespace test3 + + rbd namespace list | grep 'test' | wc -l | grep '^1$' + rbd namespace remove rbd/test2 +} + +get_migration_state() { + local image=$1 + + rbd --format xml status $image | + $XMLSTARLET sel -t -v '//status/migration/state' +} + +test_migration() { + echo "testing migration..." + remove_images + ceph osd pool create rbd2 8 + rbd pool init rbd2 + + # Convert to new format + rbd create --image-format 1 -s 128M test1 + rbd info test1 | grep 'format: 1' + rbd migration prepare test1 --image-format 2 + test "$(get_migration_state test1)" = prepared + rbd info test1 | grep 'format: 2' + rbd rm test1 && exit 1 || true + rbd migration execute test1 + test "$(get_migration_state test1)" = executed + rbd migration commit test1 + get_migration_state test1 && exit 1 || true + + # Enable layering (and some other features) + rbd info test1 | grep 'features: .*layering' && exit 1 || true + rbd migration prepare test1 --image-feature \ + layering,exclusive-lock,object-map,fast-diff,deep-flatten + rbd info test1 | grep 'features: .*layering' + rbd migration execute test1 + rbd migration commit test1 + + # Migration to other pool + rbd migration prepare test1 rbd2/test1 + test "$(get_migration_state rbd2/test1)" = prepared + rbd ls | wc -l | grep '^0$' + rbd -p rbd2 ls | grep test1 + rbd migration execute test1 + test "$(get_migration_state rbd2/test1)" = executed + rbd rm rbd2/test1 && exit 1 || true + rbd migration commit test1 + + # Migration to other namespace + rbd namespace create rbd2/ns1 + rbd namespace create rbd2/ns2 + rbd migration prepare rbd2/test1 rbd2/ns1/test1 + test "$(get_migration_state rbd2/ns1/test1)" = prepared + rbd migration execute rbd2/test1 + test "$(get_migration_state rbd2/ns1/test1)" = executed + rbd migration commit rbd2/test1 + rbd migration prepare rbd2/ns1/test1 rbd2/ns2/test1 + rbd migration execute rbd2/ns2/test1 + rbd migration commit rbd2/ns2/test1 + + # Enable data pool + rbd create -s 128M test1 + rbd migration prepare test1 --data-pool rbd2 + rbd info test1 | grep 'data_pool: rbd2' + rbd migration execute test1 + rbd migration commit test1 + + # testing trash + rbd migration prepare test1 + expect_fail rbd trash mv test1 + ID=`rbd trash ls -a | cut -d ' ' -f 1` + expect_fail rbd trash rm $ID + expect_fail rbd trash restore $ID + rbd migration abort test1 + + # Migrate parent + rbd remove test1 + dd if=/dev/urandom bs=1M count=1 | rbd --image-format 2 import - test1 + md5sum=$(rbd export test1 - | md5sum) + rbd snap create test1@snap1 + rbd snap protect test1@snap1 + rbd snap create test1@snap2 + rbd clone test1@snap1 clone_v1 --rbd_default_clone_format=1 + rbd clone test1@snap2 clone_v2 --rbd_default_clone_format=2 + rbd info clone_v1 | fgrep 'parent: rbd/test1@snap1' + rbd info clone_v2 | fgrep 'parent: rbd/test1@snap2' + rbd info clone_v2 |grep 'op_features: clone-child' + test "$(rbd export clone_v1 - | md5sum)" = "${md5sum}" + test "$(rbd export clone_v2 - | md5sum)" = "${md5sum}" + test "$(rbd children test1@snap1)" = "rbd/clone_v1" + test "$(rbd children test1@snap2)" = "rbd/clone_v2" + rbd migration prepare test1 rbd2/test2 + rbd info clone_v1 | fgrep 'parent: rbd2/test2@snap1' + rbd info clone_v2 | fgrep 'parent: rbd2/test2@snap2' + rbd info clone_v2 | fgrep 'op_features: clone-child' + test "$(rbd children rbd2/test2@snap1)" = "rbd/clone_v1" + test "$(rbd children rbd2/test2@snap2)" = "rbd/clone_v2" + rbd migration execute test1 + expect_fail rbd migration commit test1 + rbd migration commit test1 --force + test "$(rbd export clone_v1 - | md5sum)" = "${md5sum}" + test "$(rbd export clone_v2 - | md5sum)" = "${md5sum}" + rbd migration prepare rbd2/test2 test1 + rbd info clone_v1 | fgrep 'parent: rbd/test1@snap1' + rbd info clone_v2 | fgrep 'parent: rbd/test1@snap2' + rbd info clone_v2 | fgrep 'op_features: clone-child' + test "$(rbd children test1@snap1)" = "rbd/clone_v1" + test "$(rbd children test1@snap2)" = "rbd/clone_v2" + rbd migration execute test1 + expect_fail rbd migration commit test1 + rbd migration commit test1 --force + test "$(rbd export clone_v1 - | md5sum)" = "${md5sum}" + test "$(rbd export clone_v2 - | md5sum)" = "${md5sum}" + rbd remove clone_v1 + rbd remove clone_v2 + rbd snap unprotect test1@snap1 + rbd snap purge test1 + rbd rm test1 + + for format in 1 2; do + # Abort migration after successful prepare + rbd create -s 128M --image-format ${format} test2 + rbd migration prepare test2 --data-pool rbd2 + rbd bench --io-type write --io-size 1024 --io-total 1024 test2 + rbd migration abort test2 + rbd bench --io-type write --io-size 1024 --io-total 1024 test2 + rbd rm test2 + + # Abort migration after successful execute + rbd create -s 128M --image-format ${format} test2 + rbd migration prepare test2 --data-pool rbd2 + rbd bench --io-type write --io-size 1024 --io-total 1024 test2 + rbd migration execute test2 + rbd migration abort test2 + rbd bench --io-type write --io-size 1024 --io-total 1024 test2 + rbd rm test2 + + # Migration is automatically aborted if prepare failed + rbd create -s 128M --image-format ${format} test2 + rbd migration prepare test2 --data-pool INVALID_DATA_POOL && exit 1 || true + rbd bench --io-type write --io-size 1024 --io-total 1024 test2 + rbd rm test2 + + # Abort migration to other pool + rbd create -s 128M --image-format ${format} test2 + rbd migration prepare test2 rbd2/test2 + rbd bench --io-type write --io-size 1024 --io-total 1024 rbd2/test2 + rbd migration abort test2 + rbd bench --io-type write --io-size 1024 --io-total 1024 test2 + rbd rm test2 + + # The same but abort using destination image + rbd create -s 128M --image-format ${format} test2 + rbd migration prepare test2 rbd2/test2 + rbd migration abort rbd2/test2 + rbd bench --io-type write --io-size 1024 --io-total 1024 test2 + rbd rm test2 + + test $format = 1 && continue + + # Abort migration to other namespace + rbd create -s 128M --image-format ${format} test2 + rbd migration prepare test2 rbd2/ns1/test3 + rbd bench --io-type write --io-size 1024 --io-total 1024 rbd2/ns1/test3 + rbd migration abort test2 + rbd bench --io-type write --io-size 1024 --io-total 1024 test2 + rbd rm test2 + done + + remove_images + ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it +} + +test_config() { + echo "testing config..." + remove_images + + expect_fail rbd config global set osd rbd_cache true + expect_fail rbd config global set global debug_ms 10 + expect_fail rbd config global set global rbd_UNKNOWN false + expect_fail rbd config global set global rbd_cache INVALID + rbd config global set global rbd_cache false + rbd config global set client rbd_cache true + rbd config global set client.123 rbd_cache false + rbd config global get global rbd_cache | grep '^false$' + rbd config global get client rbd_cache | grep '^true$' + rbd config global get client.123 rbd_cache | grep '^false$' + expect_fail rbd config global get client.UNKNOWN rbd_cache + rbd config global list global | grep '^rbd_cache * false * global *$' + rbd config global list client | grep '^rbd_cache * true * client *$' + rbd config global list client.123 | grep '^rbd_cache * false * client.123 *$' + rbd config global list client.UNKNOWN | grep '^rbd_cache * true * client *$' + rbd config global rm client rbd_cache + expect_fail rbd config global get client rbd_cache + rbd config global list client | grep '^rbd_cache * false * global *$' + rbd config global rm client.123 rbd_cache + rbd config global rm global rbd_cache + + rbd config pool set rbd rbd_cache true + rbd config pool list rbd | grep '^rbd_cache * true * pool *$' + rbd config pool get rbd rbd_cache | grep '^true$' + + rbd create $RBD_CREATE_ARGS -s 1 test1 + + rbd config image list rbd/test1 | grep '^rbd_cache * true * pool *$' + rbd config image set rbd/test1 rbd_cache false + rbd config image list rbd/test1 | grep '^rbd_cache * false * image *$' + rbd config image get rbd/test1 rbd_cache | grep '^false$' + rbd config image remove rbd/test1 rbd_cache + expect_fail rbd config image get rbd/test1 rbd_cache + rbd config image list rbd/test1 | grep '^rbd_cache * true * pool *$' + + rbd config pool remove rbd rbd_cache + expect_fail rbd config pool get rbd rbd_cache + rbd config pool list rbd | grep '^rbd_cache * true * config *$' + + rbd rm test1 +} + +test_trash_purge_schedule() { + echo "testing trash purge schedule..." + remove_images + ceph osd pool create rbd2 8 + rbd pool init rbd2 + rbd namespace create rbd2/ns1 + + test "$(ceph rbd trash purge schedule list)" = "{}" + ceph rbd trash purge schedule status | fgrep '"scheduled": []' + + expect_fail rbd trash purge schedule ls + test "$(rbd trash purge schedule ls -R --format json)" = "[]" + + rbd trash purge schedule add -p rbd 1d 01:30 + + rbd trash purge schedule ls -p rbd | grep 'every 1d starting at 01:30' + expect_fail rbd trash purge schedule ls + rbd trash purge schedule ls -R | grep 'every 1d starting at 01:30' + rbd trash purge schedule ls -R -p rbd | grep 'every 1d starting at 01:30' + expect_fail rbd trash purge schedule ls -p rbd2 + test "$(rbd trash purge schedule ls -p rbd2 -R --format json)" = "[]" + + rbd trash purge schedule add -p rbd2/ns1 2d + test "$(rbd trash purge schedule ls -p rbd2 -R --format json)" != "[]" + rbd trash purge schedule ls -p rbd2 -R | grep 'rbd2 *ns1 *every 2d' + rbd trash purge schedule rm -p rbd2/ns1 + test "$(rbd trash purge schedule ls -p rbd2 -R --format json)" = "[]" + + for i in `seq 12`; do + test "$(rbd trash purge schedule status --format xml | + $XMLSTARLET sel -t -v '//scheduled/item/pool')" = 'rbd' && break + sleep 10 + done + rbd trash purge schedule status + test "$(rbd trash purge schedule status --format xml | + $XMLSTARLET sel -t -v '//scheduled/item/pool')" = 'rbd' + test "$(rbd trash purge schedule status -p rbd --format xml | + $XMLSTARLET sel -t -v '//scheduled/item/pool')" = 'rbd' + + rbd trash purge schedule add 2d 00:17 + rbd trash purge schedule ls | grep 'every 2d starting at 00:17' + rbd trash purge schedule ls -R | grep 'every 2d starting at 00:17' + expect_fail rbd trash purge schedule ls -p rbd2 + rbd trash purge schedule ls -p rbd2 -R | grep 'every 2d starting at 00:17' + rbd trash purge schedule ls -p rbd2/ns1 -R | grep 'every 2d starting at 00:17' + test "$(rbd trash purge schedule ls -R -p rbd2/ns1 --format xml | + $XMLSTARLET sel -t -v '//schedules/schedule/pool')" = "-" + test "$(rbd trash purge schedule ls -R -p rbd2/ns1 --format xml | + $XMLSTARLET sel -t -v '//schedules/schedule/namespace')" = "-" + test "$(rbd trash purge schedule ls -R -p rbd2/ns1 --format xml | + $XMLSTARLET sel -t -v '//schedules/schedule/items/item/start_time')" = "00:17:00" + + for i in `seq 12`; do + rbd trash purge schedule status --format xml | + $XMLSTARLET sel -t -v '//scheduled/item/pool' | grep 'rbd2' && break + sleep 10 + done + rbd trash purge schedule status + rbd trash purge schedule status --format xml | + $XMLSTARLET sel -t -v '//scheduled/item/pool' | grep 'rbd2' + echo $(rbd trash purge schedule status --format xml | + $XMLSTARLET sel -t -v '//scheduled/item/pool') | grep 'rbd rbd2 rbd2' + test "$(rbd trash purge schedule status -p rbd --format xml | + $XMLSTARLET sel -t -v '//scheduled/item/pool')" = 'rbd' + test "$(echo $(rbd trash purge schedule status -p rbd2 --format xml | + $XMLSTARLET sel -t -v '//scheduled/item/pool'))" = 'rbd2 rbd2' + + test "$(echo $(rbd trash purge schedule ls -R --format xml | + $XMLSTARLET sel -t -v '//schedules/schedule/items'))" = "2d00:17:00 1d01:30:00" + + rbd trash purge schedule add 1d + rbd trash purge schedule ls | grep 'every 2d starting at 00:17' + rbd trash purge schedule ls | grep 'every 1d' + + rbd trash purge schedule ls -R --format xml | + $XMLSTARLET sel -t -v '//schedules/schedule/items' | grep '2d00:17' + + rbd trash purge schedule rm 1d + rbd trash purge schedule ls | grep 'every 2d starting at 00:17' + rbd trash purge schedule rm 2d 00:17 + expect_fail rbd trash purge schedule ls + + for p in rbd2 rbd2/ns1; do + rbd create $RBD_CREATE_ARGS -s 1 rbd2/ns1/test1 + rbd trash mv rbd2/ns1/test1 + rbd trash ls rbd2/ns1 | wc -l | grep '^1$' + + rbd trash purge schedule add -p $p 1m + rbd trash purge schedule list -p rbd2 -R | grep 'every 1m' + rbd trash purge schedule list -p rbd2/ns1 -R | grep 'every 1m' + + for i in `seq 12`; do + rbd trash ls rbd2/ns1 | wc -l | grep '^1$' || break + sleep 10 + done + rbd trash ls rbd2/ns1 | wc -l | grep '^0$' + + # repeat with kicked in schedule, see https://tracker.ceph.com/issues/53915 + rbd trash purge schedule list -p rbd2 -R | grep 'every 1m' + rbd trash purge schedule list -p rbd2/ns1 -R | grep 'every 1m' + + rbd trash purge schedule status | grep 'rbd2 *ns1' + rbd trash purge schedule status -p rbd2 | grep 'rbd2 *ns1' + rbd trash purge schedule status -p rbd2/ns1 | grep 'rbd2 *ns1' + + rbd trash purge schedule rm -p $p 1m + done + + # Negative tests + rbd trash purge schedule add 2m + expect_fail rbd trash purge schedule add -p rbd dummy + expect_fail rbd trash purge schedule add dummy + expect_fail rbd trash purge schedule remove -p rbd dummy + expect_fail rbd trash purge schedule remove dummy + rbd trash purge schedule ls -p rbd | grep 'every 1d starting at 01:30' + rbd trash purge schedule ls | grep 'every 2m' + rbd trash purge schedule remove -p rbd 1d 01:30 + rbd trash purge schedule remove 2m + test "$(rbd trash purge schedule ls -R --format json)" = "[]" + + remove_images + ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it +} + +test_trash_purge_schedule_recovery() { + echo "testing recovery of trash_purge_schedule handler after module's RADOS client is blocklisted..." + remove_images + ceph osd pool create rbd3 8 + rbd pool init rbd3 + rbd namespace create rbd3/ns1 + + rbd trash purge schedule add -p rbd3/ns1 2d + rbd trash purge schedule ls -p rbd3 -R | grep 'rbd3 *ns1 *every 2d' + + # Fetch and blocklist the rbd_support module's RADOS client + CLIENT_ADDR=$(ceph mgr dump | jq .active_clients[] | + jq 'select(.name == "rbd_support")' | + jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add') + ceph osd blocklist add $CLIENT_ADDR + ceph osd blocklist ls | grep $CLIENT_ADDR + + # Check that you can add a trash purge schedule after a few retries + expect_fail rbd trash purge schedule add -p rbd3 10m + sleep 10 + for i in `seq 24`; do + rbd trash purge schedule add -p rbd3 10m && break + sleep 10 + done + + rbd trash purge schedule ls -p rbd3 -R | grep 'every 10m' + # Verify that the schedule present before client blocklisting is preserved + rbd trash purge schedule ls -p rbd3 -R | grep 'rbd3 *ns1 *every 2d' + + rbd trash purge schedule remove -p rbd3 10m + rbd trash purge schedule remove -p rbd3/ns1 2d + rbd trash purge schedule ls -p rbd3 -R | expect_fail grep 'every 10m' + rbd trash purge schedule ls -p rbd3 -R | expect_fail grep 'rbd3 *ns1 *every 2d' + + ceph osd pool rm rbd3 rbd3 --yes-i-really-really-mean-it + +} + +test_mirror_snapshot_schedule() { + echo "testing mirror snapshot schedule..." + remove_images + ceph osd pool create rbd2 8 + rbd pool init rbd2 + rbd namespace create rbd2/ns1 + + rbd mirror pool enable rbd2 image + rbd mirror pool enable rbd2/ns1 image + rbd mirror pool peer add rbd2 cluster1 + + test "$(ceph rbd mirror snapshot schedule list)" = "{}" + ceph rbd mirror snapshot schedule status | fgrep '"scheduled_images": []' + + expect_fail rbd mirror snapshot schedule ls + test "$(rbd mirror snapshot schedule ls -R --format json)" = "[]" + + rbd create $RBD_CREATE_ARGS -s 1 rbd2/ns1/test1 + + test "$(rbd mirror image status rbd2/ns1/test1 | + grep -c mirror.primary)" = '0' + + rbd mirror image enable rbd2/ns1/test1 snapshot + + test "$(rbd mirror image status rbd2/ns1/test1 | + grep -c mirror.primary)" = '1' + + rbd mirror snapshot schedule add -p rbd2/ns1 --image test1 1m + expect_fail rbd mirror snapshot schedule ls + rbd mirror snapshot schedule ls -R | grep 'rbd2 *ns1 *test1 *every 1m' + expect_fail rbd mirror snapshot schedule ls -p rbd2 + rbd mirror snapshot schedule ls -p rbd2 -R | grep 'rbd2 *ns1 *test1 *every 1m' + expect_fail rbd mirror snapshot schedule ls -p rbd2/ns1 + rbd mirror snapshot schedule ls -p rbd2/ns1 -R | grep 'rbd2 *ns1 *test1 *every 1m' + test "$(rbd mirror snapshot schedule ls -p rbd2/ns1 --image test1)" = 'every 1m' + + for i in `seq 12`; do + test "$(rbd mirror image status rbd2/ns1/test1 | + grep -c mirror.primary)" -gt '1' && break + sleep 10 + done + + test "$(rbd mirror image status rbd2/ns1/test1 | + grep -c mirror.primary)" -gt '1' + + # repeat with kicked in schedule, see https://tracker.ceph.com/issues/53915 + expect_fail rbd mirror snapshot schedule ls + rbd mirror snapshot schedule ls -R | grep 'rbd2 *ns1 *test1 *every 1m' + expect_fail rbd mirror snapshot schedule ls -p rbd2 + rbd mirror snapshot schedule ls -p rbd2 -R | grep 'rbd2 *ns1 *test1 *every 1m' + expect_fail rbd mirror snapshot schedule ls -p rbd2/ns1 + rbd mirror snapshot schedule ls -p rbd2/ns1 -R | grep 'rbd2 *ns1 *test1 *every 1m' + test "$(rbd mirror snapshot schedule ls -p rbd2/ns1 --image test1)" = 'every 1m' + + rbd mirror snapshot schedule status + test "$(rbd mirror snapshot schedule status --format xml | + $XMLSTARLET sel -t -v '//scheduled_images/image/image')" = 'rbd2/ns1/test1' + test "$(rbd mirror snapshot schedule status -p rbd2 --format xml | + $XMLSTARLET sel -t -v '//scheduled_images/image/image')" = 'rbd2/ns1/test1' + test "$(rbd mirror snapshot schedule status -p rbd2/ns1 --format xml | + $XMLSTARLET sel -t -v '//scheduled_images/image/image')" = 'rbd2/ns1/test1' + test "$(rbd mirror snapshot schedule status -p rbd2/ns1 --image test1 --format xml | + $XMLSTARLET sel -t -v '//scheduled_images/image/image')" = 'rbd2/ns1/test1' + + rbd mirror image demote rbd2/ns1/test1 + for i in `seq 12`; do + rbd mirror snapshot schedule status | grep 'rbd2/ns1/test1' || break + sleep 10 + done + rbd mirror snapshot schedule status | expect_fail grep 'rbd2/ns1/test1' + + rbd mirror image promote rbd2/ns1/test1 + for i in `seq 12`; do + rbd mirror snapshot schedule status | grep 'rbd2/ns1/test1' && break + sleep 10 + done + rbd mirror snapshot schedule status | grep 'rbd2/ns1/test1' + + rbd mirror snapshot schedule add 1h 00:15 + test "$(rbd mirror snapshot schedule ls)" = 'every 1h starting at 00:15:00' + rbd mirror snapshot schedule ls -R | grep 'every 1h starting at 00:15:00' + rbd mirror snapshot schedule ls -R | grep 'rbd2 *ns1 *test1 *every 1m' + expect_fail rbd mirror snapshot schedule ls -p rbd2 + rbd mirror snapshot schedule ls -p rbd2 -R | grep 'every 1h starting at 00:15:00' + rbd mirror snapshot schedule ls -p rbd2 -R | grep 'rbd2 *ns1 *test1 *every 1m' + expect_fail rbd mirror snapshot schedule ls -p rbd2/ns1 + rbd mirror snapshot schedule ls -p rbd2/ns1 -R | grep 'every 1h starting at 00:15:00' + rbd mirror snapshot schedule ls -p rbd2/ns1 -R | grep 'rbd2 *ns1 *test1 *every 1m' + test "$(rbd mirror snapshot schedule ls -p rbd2/ns1 --image test1)" = 'every 1m' + + # Negative tests + expect_fail rbd mirror snapshot schedule add dummy + expect_fail rbd mirror snapshot schedule add -p rbd2/ns1 --image test1 dummy + expect_fail rbd mirror snapshot schedule remove dummy + expect_fail rbd mirror snapshot schedule remove -p rbd2/ns1 --image test1 dummy + test "$(rbd mirror snapshot schedule ls)" = 'every 1h starting at 00:15:00' + test "$(rbd mirror snapshot schedule ls -p rbd2/ns1 --image test1)" = 'every 1m' + + rbd rm rbd2/ns1/test1 + for i in `seq 12`; do + rbd mirror snapshot schedule status | grep 'rbd2/ns1/test1' || break + sleep 10 + done + rbd mirror snapshot schedule status | expect_fail grep 'rbd2/ns1/test1' + + rbd mirror snapshot schedule remove + test "$(rbd mirror snapshot schedule ls -R --format json)" = "[]" + + remove_images + ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it +} + +test_mirror_snapshot_schedule_recovery() { + echo "testing recovery of mirror snapshot scheduler after module's RADOS client is blocklisted..." + remove_images + ceph osd pool create rbd3 8 + rbd pool init rbd3 + rbd namespace create rbd3/ns1 + + rbd mirror pool enable rbd3 image + rbd mirror pool enable rbd3/ns1 image + rbd mirror pool peer add rbd3 cluster1 + + rbd create $RBD_CREATE_ARGS -s 1 rbd3/ns1/test1 + rbd mirror image enable rbd3/ns1/test1 snapshot + test "$(rbd mirror image status rbd3/ns1/test1 | + grep -c mirror.primary)" = '1' + + rbd mirror snapshot schedule add -p rbd3/ns1 --image test1 1m + test "$(rbd mirror snapshot schedule ls -p rbd3/ns1 --image test1)" = 'every 1m' + + # Fetch and blocklist rbd_support module's RADOS client + CLIENT_ADDR=$(ceph mgr dump | jq .active_clients[] | + jq 'select(.name == "rbd_support")' | + jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add') + ceph osd blocklist add $CLIENT_ADDR + ceph osd blocklist ls | grep $CLIENT_ADDR + + # Check that you can add a mirror snapshot schedule after a few retries + expect_fail rbd mirror snapshot schedule add -p rbd3/ns1 --image test1 2m + sleep 10 + for i in `seq 24`; do + rbd mirror snapshot schedule add -p rbd3/ns1 --image test1 2m && break + sleep 10 + done + + rbd mirror snapshot schedule ls -p rbd3/ns1 --image test1 | grep 'every 2m' + # Verify that the schedule present before client blocklisting is preserved + rbd mirror snapshot schedule ls -p rbd3/ns1 --image test1 | grep 'every 1m' + + rbd mirror snapshot schedule rm -p rbd3/ns1 --image test1 2m + rbd mirror snapshot schedule rm -p rbd3/ns1 --image test1 1m + rbd mirror snapshot schedule ls -p rbd3/ns1 --image test1 | expect_fail grep 'every 2m' + rbd mirror snapshot schedule ls -p rbd3/ns1 --image test1 | expect_fail grep 'every 1m' + + rbd snap purge rbd3/ns1/test1 + rbd rm rbd3/ns1/test1 + ceph osd pool rm rbd3 rbd3 --yes-i-really-really-mean-it +} + +test_perf_image_iostat() { + echo "testing perf image iostat..." + remove_images + + ceph osd pool create rbd1 8 + rbd pool init rbd1 + rbd namespace create rbd1/ns + ceph osd pool create rbd2 8 + rbd pool init rbd2 + rbd namespace create rbd2/ns + + IMAGE_SPECS=("test1" "rbd1/test2" "rbd1/ns/test3" "rbd2/test4" "rbd2/ns/test5") + for spec in "${IMAGE_SPECS[@]}"; do + # ensure all images are created without a separate data pool + # as we filter iostat by specific pool specs below + rbd create $RBD_CREATE_ARGS --size 10G --rbd-default-data-pool '' $spec + done + + BENCH_PIDS=() + for spec in "${IMAGE_SPECS[@]}"; do + rbd bench --io-type write --io-pattern rand --io-total 10G --io-threads 1 \ + --rbd-cache false $spec >/dev/null 2>&1 & + BENCH_PIDS+=($!) + done + + # test specifying pool spec via spec syntax + test "$(rbd perf image iostat --format json rbd1 | + jq -r 'map(.image) | sort | join(" ")')" = 'test2' + test "$(rbd perf image iostat --format json rbd1/ns | + jq -r 'map(.image) | sort | join(" ")')" = 'test3' + test "$(rbd perf image iostat --format json --rbd-default-pool rbd1 /ns | + jq -r 'map(.image) | sort | join(" ")')" = 'test3' + + # test specifying pool spec via options + test "$(rbd perf image iostat --format json --pool rbd2 | + jq -r 'map(.image) | sort | join(" ")')" = 'test4' + test "$(rbd perf image iostat --format json --pool rbd2 --namespace ns | + jq -r 'map(.image) | sort | join(" ")')" = 'test5' + test "$(rbd perf image iostat --format json --rbd-default-pool rbd2 --namespace ns | + jq -r 'map(.image) | sort | join(" ")')" = 'test5' + + # test omitting pool spec (-> GLOBAL_POOL_KEY) + test "$(rbd perf image iostat --format json | + jq -r 'map(.image) | sort | join(" ")')" = 'test1 test2 test3 test4 test5' + + for pid in "${BENCH_PIDS[@]}"; do + kill $pid + done + wait + + remove_images + ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it + ceph osd pool rm rbd1 rbd1 --yes-i-really-really-mean-it +} + +test_perf_image_iostat_recovery() { + echo "testing recovery of perf handler after module's RADOS client is blocklisted..." + remove_images + + ceph osd pool create rbd3 8 + rbd pool init rbd3 + rbd namespace create rbd3/ns + + IMAGE_SPECS=("rbd3/test1" "rbd3/ns/test2") + for spec in "${IMAGE_SPECS[@]}"; do + # ensure all images are created without a separate data pool + # as we filter iostat by specific pool specs below + rbd create $RBD_CREATE_ARGS --size 10G --rbd-default-data-pool '' $spec + done + + BENCH_PIDS=() + for spec in "${IMAGE_SPECS[@]}"; do + rbd bench --io-type write --io-pattern rand --io-total 10G --io-threads 1 \ + --rbd-cache false $spec >/dev/null 2>&1 & + BENCH_PIDS+=($!) + done + + test "$(rbd perf image iostat --format json rbd3 | + jq -r 'map(.image) | sort | join(" ")')" = 'test1' + + # Fetch and blocklist the rbd_support module's RADOS client + CLIENT_ADDR=$(ceph mgr dump | jq .active_clients[] | + jq 'select(.name == "rbd_support")' | + jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add') + ceph osd blocklist add $CLIENT_ADDR + ceph osd blocklist ls | grep $CLIENT_ADDR + + expect_fail rbd perf image iostat --format json rbd3/ns + sleep 10 + for i in `seq 24`; do + test "$(rbd perf image iostat --format json rbd3/ns | + jq -r 'map(.image) | sort | join(" ")')" = 'test2' && break + sleep 10 + done + + for pid in "${BENCH_PIDS[@]}"; do + kill $pid + done + wait + + remove_images + ceph osd pool rm rbd3 rbd3 --yes-i-really-really-mean-it +} + +test_mirror_pool_peer_bootstrap_create() { + echo "testing mirror pool peer bootstrap create..." + remove_images + + ceph osd pool create rbd1 8 + rbd pool init rbd1 + rbd mirror pool enable rbd1 image + ceph osd pool create rbd2 8 + rbd pool init rbd2 + rbd mirror pool enable rbd2 pool + + readarray -t MON_ADDRS < <(ceph mon dump | + sed -n 's/^[0-9]: \(.*\) mon\.[a-z]$/\1/p') + + # check that all monitors make it to the token even if only one + # valid monitor is specified + BAD_MON_ADDR="1.2.3.4:6789" + MON_HOST="${MON_ADDRS[0]},$BAD_MON_ADDR" + TOKEN="$(rbd mirror pool peer bootstrap create \ + --mon-host "$MON_HOST" rbd1 | base64 -d)" + TOKEN_FSID="$(jq -r '.fsid' <<< "$TOKEN")" + TOKEN_CLIENT_ID="$(jq -r '.client_id' <<< "$TOKEN")" + TOKEN_KEY="$(jq -r '.key' <<< "$TOKEN")" + TOKEN_MON_HOST="$(jq -r '.mon_host' <<< "$TOKEN")" + + test "$TOKEN_FSID" = "$(ceph fsid)" + test "$TOKEN_KEY" = "$(ceph auth get-key client.$TOKEN_CLIENT_ID)" + for addr in "${MON_ADDRS[@]}"; do + fgrep "$addr" <<< "$TOKEN_MON_HOST" + done + expect_fail fgrep "$BAD_MON_ADDR" <<< "$TOKEN_MON_HOST" + + # check that the token does not change, including across pools + test "$(rbd mirror pool peer bootstrap create \ + --mon-host "$MON_HOST" rbd1 | base64 -d)" = "$TOKEN" + test "$(rbd mirror pool peer bootstrap create \ + rbd1 | base64 -d)" = "$TOKEN" + test "$(rbd mirror pool peer bootstrap create \ + --mon-host "$MON_HOST" rbd2 | base64 -d)" = "$TOKEN" + test "$(rbd mirror pool peer bootstrap create \ + rbd2 | base64 -d)" = "$TOKEN" + + ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it + ceph osd pool rm rbd1 rbd1 --yes-i-really-really-mean-it +} + +test_tasks_removed_pool() { + echo "testing removing pool under running tasks..." + remove_images + + ceph osd pool create rbd2 8 + rbd pool init rbd2 + + rbd create $RBD_CREATE_ARGS --size 1G foo + rbd snap create foo@snap + rbd snap protect foo@snap + rbd clone foo@snap bar + + rbd create $RBD_CREATE_ARGS --size 1G rbd2/dummy + rbd bench --io-type write --io-pattern seq --io-size 1M --io-total 1G rbd2/dummy + rbd snap create rbd2/dummy@snap + rbd snap protect rbd2/dummy@snap + for i in {1..5}; do + rbd clone rbd2/dummy@snap rbd2/dummy$i + done + + # queue flattens on a few dummy images and remove that pool + test "$(ceph rbd task list)" = "[]" + for i in {1..5}; do + ceph rbd task add flatten rbd2/dummy$i + done + ceph osd pool delete rbd2 rbd2 --yes-i-really-really-mean-it + test "$(ceph rbd task list)" != "[]" + + # queue flatten on another image and check that it completes + rbd info bar | grep 'parent: ' + expect_fail rbd snap unprotect foo@snap + ceph rbd task add flatten bar + for i in {1..12}; do + rbd info bar | grep 'parent: ' || break + sleep 10 + done + rbd info bar | expect_fail grep 'parent: ' + rbd snap unprotect foo@snap + + # check that flattens disrupted by pool removal are cleaned up + for i in {1..12}; do + test "$(ceph rbd task list)" = "[]" && break + sleep 10 + done + test "$(ceph rbd task list)" = "[]" + + remove_images +} + +test_tasks_recovery() { + echo "testing task handler recovery after module's RADOS client is blocklisted..." + remove_images + + ceph osd pool create rbd2 8 + rbd pool init rbd2 + + rbd create $RBD_CREATE_ARGS --size 1G rbd2/img1 + rbd bench --io-type write --io-pattern seq --io-size 1M --io-total 1G rbd2/img1 + rbd snap create rbd2/img1@snap + rbd snap protect rbd2/img1@snap + rbd clone rbd2/img1@snap rbd2/clone1 + + # Fetch and blocklist rbd_support module's RADOS client + CLIENT_ADDR=$(ceph mgr dump | jq .active_clients[] | + jq 'select(.name == "rbd_support")' | + jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add') + ceph osd blocklist add $CLIENT_ADDR + ceph osd blocklist ls | grep $CLIENT_ADDR + + expect_fail ceph rbd task add flatten rbd2/clone1 + sleep 10 + for i in `seq 24`; do + ceph rbd task add flatten rbd2/clone1 && break + sleep 10 + done + test "$(ceph rbd task list)" != "[]" + + for i in {1..12}; do + rbd info rbd2/clone1 | grep 'parent: ' || break + sleep 10 + done + rbd info rbd2/clone1 | expect_fail grep 'parent: ' + rbd snap unprotect rbd2/img1@snap + + test "$(ceph rbd task list)" = "[]" + ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it +} + +test_pool_image_args +test_rename +test_ls +test_remove +test_migration +test_config +RBD_CREATE_ARGS="" +test_others +test_locking +test_thick_provision +RBD_CREATE_ARGS="--image-format 2" +test_others +test_locking +test_clone +test_trash +test_purge +test_deep_copy_clone +test_clone_v2 +test_thick_provision +test_namespace +test_trash_purge_schedule +test_trash_purge_schedule_recovery +test_mirror_snapshot_schedule +test_mirror_snapshot_schedule_recovery +test_perf_image_iostat +test_perf_image_iostat_recovery +test_mirror_pool_peer_bootstrap_create +test_tasks_removed_pool +test_tasks_recovery + +echo OK diff --git a/qa/workunits/rbd/cli_migration.sh b/qa/workunits/rbd/cli_migration.sh new file mode 100755 index 000000000..be8e031fd --- /dev/null +++ b/qa/workunits/rbd/cli_migration.sh @@ -0,0 +1,357 @@ +#!/usr/bin/env bash +set -ex + +. $(dirname $0)/../../standalone/ceph-helpers.sh + +TEMPDIR= +IMAGE1=image1 +IMAGE2=image2 +IMAGE3=image3 +IMAGES="${IMAGE1} ${IMAGE2} ${IMAGE3}" + +cleanup() { + cleanup_tempdir + remove_images +} + +setup_tempdir() { + TEMPDIR=`mktemp -d` +} + +cleanup_tempdir() { + rm -rf ${TEMPDIR} +} + +create_base_image() { + local image=$1 + + rbd create --size 1G ${image} + rbd bench --io-type write --io-pattern rand --io-size=4K --io-total 256M ${image} + rbd snap create ${image}@1 + rbd bench --io-type write --io-pattern rand --io-size=4K --io-total 64M ${image} + rbd snap create ${image}@2 + rbd bench --io-type write --io-pattern rand --io-size=4K --io-total 128M ${image} +} + +export_raw_image() { + local image=$1 + + rm -rf "${TEMPDIR}/${image}" + rbd export ${image} "${TEMPDIR}/${image}" +} + +export_base_image() { + local image=$1 + + export_raw_image "${image}" + export_raw_image "${image}@1" + export_raw_image "${image}@2" +} + +remove_image() { + local image=$1 + + (rbd migration abort $image || true) >/dev/null 2>&1 + (rbd snap purge $image || true) >/dev/null 2>&1 + (rbd rm $image || true) >/dev/null 2>&1 +} + +remove_images() { + for image in ${IMAGES} + do + remove_image ${image} + done +} + +show_diff() +{ + local file1=$1 + local file2=$2 + + xxd "${file1}" > "${file1}.xxd" + xxd "${file2}" > "${file2}.xxd" + sdiff -s "${file1}.xxd" "${file2}.xxd" | head -n 64 + rm -f "${file1}.xxd" "${file2}.xxd" +} + +compare_images() { + local src_image=$1 + local dst_image=$2 + local ret=0 + + export_raw_image ${dst_image} + if ! cmp "${TEMPDIR}/${src_image}" "${TEMPDIR}/${dst_image}" + then + show_diff "${TEMPDIR}/${src_image}" "${TEMPDIR}/${dst_image}" + ret=1 + fi + return ${ret} +} + +test_import_native_format() { + local base_image=$1 + local dest_image=$2 + + rbd migration prepare --import-only "rbd/${base_image}@2" ${dest_image} + rbd migration abort ${dest_image} + + local pool_id=$(ceph osd pool ls detail --format xml | xmlstarlet sel -t -v "//pools/pool[pool_name='rbd']/pool_id") + cat > ${TEMPDIR}/spec.json <<EOF +{ + "type": "native", + "pool_id": ${pool_id}, + "pool_namespace": "", + "image_name": "${base_image}", + "snap_name": "2" +} +EOF + cat ${TEMPDIR}/spec.json + + rbd migration prepare --import-only \ + --source-spec-path ${TEMPDIR}/spec.json ${dest_image} + + compare_images "${base_image}@1" "${dest_image}@1" + compare_images "${base_image}@2" "${dest_image}@2" + + rbd migration abort ${dest_image} + + rbd migration prepare --import-only \ + --source-spec-path ${TEMPDIR}/spec.json ${dest_image} + rbd migration execute ${dest_image} + + compare_images "${base_image}@1" "${dest_image}@1" + compare_images "${base_image}@2" "${dest_image}@2" + + rbd migration abort ${dest_image} + + rbd migration prepare --import-only \ + --source-spec "{\"type\": \"native\", \"pool_id\": "${pool_id}", \"image_name\": \"${base_image}\", \"snap_name\": \"2\"}" \ + ${dest_image} + rbd migration abort ${dest_image} + + rbd migration prepare --import-only \ + --source-spec "{\"type\": \"native\", \"pool_name\": \"rbd\", \"image_name\": \"${base_image}\", \"snap_name\": \"2\"}" \ + ${dest_image} + rbd migration execute ${dest_image} + rbd migration commit ${dest_image} + + compare_images "${base_image}@1" "${dest_image}@1" + compare_images "${base_image}@2" "${dest_image}@2" + + remove_image "${dest_image}" +} + +test_import_qcow_format() { + local base_image=$1 + local dest_image=$2 + + if ! qemu-img convert -f raw -O qcow rbd:rbd/${base_image} ${TEMPDIR}/${base_image}.qcow; then + echo "skipping QCOW test" + return 0 + fi + qemu-img info -f qcow ${TEMPDIR}/${base_image}.qcow + + cat > ${TEMPDIR}/spec.json <<EOF +{ + "type": "qcow", + "stream": { + "type": "file", + "file_path": "${TEMPDIR}/${base_image}.qcow" + } +} +EOF + cat ${TEMPDIR}/spec.json + + set +e + rbd migration prepare --import-only \ + --source-spec-path ${TEMPDIR}/spec.json ${dest_image} + local error_code=$? + set -e + + if [ $error_code -eq 95 ]; then + echo "skipping QCOW test (librbd support disabled)" + return 0 + fi + test $error_code -eq 0 + + compare_images "${base_image}" "${dest_image}" + + rbd migration abort ${dest_image} + + rbd migration prepare --import-only \ + --source-spec-path ${TEMPDIR}/spec.json ${dest_image} + + compare_images "${base_image}" "${dest_image}" + + rbd migration execute ${dest_image} + + compare_images "${base_image}" "${dest_image}" + + rbd migration commit ${dest_image} + + compare_images "${base_image}" "${dest_image}" + + remove_image "${dest_image}" +} + +test_import_qcow2_format() { + local base_image=$1 + local dest_image=$2 + + # create new image via qemu-img and its bench tool since we cannot + # import snapshot deltas into QCOW2 + qemu-img create -f qcow2 ${TEMPDIR}/${base_image}.qcow2 1G + + qemu-img bench -f qcow2 -w -c 65536 -d 16 --pattern 65 -s 4096 \ + -S $((($RANDOM % 262144) * 4096)) ${TEMPDIR}/${base_image}.qcow2 + qemu-img convert -f qcow2 -O raw ${TEMPDIR}/${base_image}.qcow2 \ + "${TEMPDIR}/${base_image}@snap1" + qemu-img snapshot -c "snap1" ${TEMPDIR}/${base_image}.qcow2 + + qemu-img bench -f qcow2 -w -c 16384 -d 16 --pattern 66 -s 4096 \ + -S $((($RANDOM % 262144) * 4096)) ${TEMPDIR}/${base_image}.qcow2 + qemu-img convert -f qcow2 -O raw ${TEMPDIR}/${base_image}.qcow2 \ + "${TEMPDIR}/${base_image}@snap2" + qemu-img snapshot -c "snap2" ${TEMPDIR}/${base_image}.qcow2 + + qemu-img bench -f qcow2 -w -c 32768 -d 16 --pattern 67 -s 4096 \ + -S $((($RANDOM % 262144) * 4096)) ${TEMPDIR}/${base_image}.qcow2 + qemu-img convert -f qcow2 -O raw ${TEMPDIR}/${base_image}.qcow2 \ + ${TEMPDIR}/${base_image} + + qemu-img info -f qcow2 ${TEMPDIR}/${base_image}.qcow2 + + cat > ${TEMPDIR}/spec.json <<EOF +{ + "type": "qcow", + "stream": { + "type": "file", + "file_path": "${TEMPDIR}/${base_image}.qcow2" + } +} +EOF + cat ${TEMPDIR}/spec.json + + rbd migration prepare --import-only \ + --source-spec-path ${TEMPDIR}/spec.json ${dest_image} + + compare_images "${base_image}@snap1" "${dest_image}@snap1" + compare_images "${base_image}@snap2" "${dest_image}@snap2" + compare_images "${base_image}" "${dest_image}" + + rbd migration abort ${dest_image} + + rbd migration prepare --import-only \ + --source-spec-path ${TEMPDIR}/spec.json ${dest_image} + + compare_images "${base_image}@snap1" "${dest_image}@snap1" + compare_images "${base_image}@snap2" "${dest_image}@snap2" + compare_images "${base_image}" "${dest_image}" + + rbd migration execute ${dest_image} + + compare_images "${base_image}@snap1" "${dest_image}@snap1" + compare_images "${base_image}@snap2" "${dest_image}@snap2" + compare_images "${base_image}" "${dest_image}" + + rbd migration commit ${dest_image} + + compare_images "${base_image}@snap1" "${dest_image}@snap1" + compare_images "${base_image}@snap2" "${dest_image}@snap2" + compare_images "${base_image}" "${dest_image}" + + remove_image "${dest_image}" +} + +test_import_raw_format() { + local base_image=$1 + local dest_image=$2 + + cat > ${TEMPDIR}/spec.json <<EOF +{ + "type": "raw", + "stream": { + "type": "file", + "file_path": "${TEMPDIR}/${base_image}" + } +} +EOF + cat ${TEMPDIR}/spec.json + + cat ${TEMPDIR}/spec.json | rbd migration prepare --import-only \ + --source-spec-path - ${dest_image} + compare_images ${base_image} ${dest_image} + rbd migration abort ${dest_image} + + rbd migration prepare --import-only \ + --source-spec-path ${TEMPDIR}/spec.json ${dest_image} + rbd migration execute ${dest_image} + rbd migration commit ${dest_image} + + compare_images ${base_image} ${dest_image} + + remove_image "${dest_image}" + + cat > ${TEMPDIR}/spec.json <<EOF +{ + "type": "raw", + "stream": { + "type": "file", + "file_path": "${TEMPDIR}/${base_image}" + }, + "snapshots": [{ + "type": "raw", + "name": "snap1", + "stream": { + "type": "file", + "file_path": "${TEMPDIR}/${base_image}@1" + } + }, { + "type": "raw", + "name": "snap2", + "stream": { + "type": "file", + "file_path": "${TEMPDIR}/${base_image}@2" + } + }] +} +EOF + cat ${TEMPDIR}/spec.json + + rbd migration prepare --import-only \ + --source-spec-path ${TEMPDIR}/spec.json ${dest_image} + + rbd snap create ${dest_image}@head + rbd bench --io-type write --io-pattern rand --io-size=32K --io-total=32M ${dest_image} + + compare_images "${base_image}" "${dest_image}@head" + compare_images "${base_image}@1" "${dest_image}@snap1" + compare_images "${base_image}@2" "${dest_image}@snap2" + compare_images "${base_image}" "${dest_image}@head" + + rbd migration execute ${dest_image} + + compare_images "${base_image}@1" "${dest_image}@snap1" + compare_images "${base_image}@2" "${dest_image}@snap2" + compare_images "${base_image}" "${dest_image}@head" + + rbd migration commit ${dest_image} + + remove_image "${dest_image}" +} + +# make sure rbd pool is EMPTY.. this is a test script!! +rbd ls 2>&1 | wc -l | grep -v '^0$' && echo "nonempty rbd pool, aborting! run this script on an empty test cluster only." && exit 1 + +setup_tempdir +trap 'cleanup $?' INT TERM EXIT + +create_base_image ${IMAGE1} +export_base_image ${IMAGE1} + +test_import_native_format ${IMAGE1} ${IMAGE2} +test_import_qcow_format ${IMAGE1} ${IMAGE2} +test_import_qcow2_format ${IMAGE2} ${IMAGE3} +test_import_raw_format ${IMAGE1} ${IMAGE2} + +echo OK diff --git a/qa/workunits/rbd/concurrent.sh b/qa/workunits/rbd/concurrent.sh new file mode 100755 index 000000000..abaad75f5 --- /dev/null +++ b/qa/workunits/rbd/concurrent.sh @@ -0,0 +1,375 @@ +#!/usr/bin/env bash + +# Copyright (C) 2013 Inktank Storage, Inc. +# +# This is free software; see the source for copying conditions. +# There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# This is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as +# published by the Free Software Foundation version 2. + +# Alex Elder <elder@inktank.com> +# January 29, 2013 + +################################################################ + +# The purpose of this test is to exercise paths through the rbd +# code, making sure no bad pointer references or invalid reference +# count operations occur in the face of concurrent activity. +# +# Each pass of the test creates an rbd image, maps it, and writes +# some data into the image. It also reads some data from all of the +# other images that exist at the time the pass executes. Finally, +# the image is unmapped and removed. The image removal completes in +# the background. +# +# An iteration of the test consists of performing some number of +# passes, initating each pass as a background job, and finally +# sleeping for a variable delay. The delay is initially a specified +# value, but each iteration shortens that proportionally, such that +# the last iteration will not delay at all. +# +# The result exercises concurrent creates and deletes of rbd images, +# writes to new images, reads from both written and unwritten image +# data (including reads concurrent with writes), and attempts to +# unmap images being read. + +# Usage: concurrent [-i <iter>] [-c <count>] [-d <delay>] +# +# Exit status: +# 0: success +# 1: usage error +# 2: other runtime error +# 99: argument count error (programming error) +# 100: getopt error (internal error) + +################################################################ + +set -ex + +# Default flag values; RBD_CONCURRENT_ITER names are intended +# to be used in yaml scripts to pass in alternate values, e.g.: +# env: +# RBD_CONCURRENT_ITER: 20 +# RBD_CONCURRENT_COUNT: 5 +# RBD_CONCURRENT_DELAY: 3 +ITER_DEFAULT=${RBD_CONCURRENT_ITER:-100} +COUNT_DEFAULT=${RBD_CONCURRENT_COUNT:-5} +DELAY_DEFAULT=${RBD_CONCURRENT_DELAY:-5} # seconds + +CEPH_SECRET_FILE=${CEPH_SECRET_FILE:-} +CEPH_ID=${CEPH_ID:-admin} +SECRET_ARGS="" +if [ "${CEPH_SECRET_FILE}" ]; then + SECRET_ARGS="--secret $CEPH_SECRET_FILE" +fi + +################################################################ + +function setup() { + ID_MAX_DIR=$(mktemp -d /tmp/image_max_id.XXXXX) + ID_COUNT_DIR=$(mktemp -d /tmp/image_ids.XXXXXX) + NAMES_DIR=$(mktemp -d /tmp/image_names.XXXXXX) + SOURCE_DATA=$(mktemp /tmp/source_data.XXXXXX) + + # Use urandom to generate SOURCE_DATA + dd if=/dev/urandom of=${SOURCE_DATA} bs=2048 count=66 \ + >/dev/null 2>&1 + + # List of rbd id's *not* created by this script + export INITIAL_RBD_IDS=$(ls /sys/bus/rbd/devices) + + # Set up some environment for normal teuthology test setup. + # This really should not be necessary but I found it was. + + export CEPH_ARGS=" --name client.0" +} + +function cleanup() { + [ ! "${ID_MAX_DIR}" ] && return + local id + local image + + # Unmap mapped devices + for id in $(rbd_ids); do + image=$(cat "/sys/bus/rbd/devices/${id}/name") + rbd_unmap_image "${id}" + rbd_destroy_image "${image}" + done + # Get any leftover images + for image in $(rbd ls 2>/dev/null); do + rbd_destroy_image "${image}" + done + wait + sync + rm -f "${SOURCE_DATA}" + [ -d "${NAMES_DIR}" ] && rmdir "${NAMES_DIR}" + echo "Max concurrent rbd image count was $(get_max "${ID_COUNT_DIR}")" + rm -rf "${ID_COUNT_DIR}" + echo "Max rbd image id was $(get_max "${ID_MAX_DIR}")" + rm -rf "${ID_MAX_DIR}" +} + +function get_max() { + [ $# -eq 1 ] || exit 99 + local dir="$1" + + ls -U "${dir}" | sort -n | tail -1 +} + +trap cleanup HUP INT QUIT + +# print a usage message and quit +# +# if a message is supplied, print that first, and then exit +# with non-zero status +function usage() { + if [ $# -gt 0 ]; then + echo "" >&2 + echo "$@" >&2 + fi + + echo "" >&2 + echo "Usage: ${PROGNAME} <options> <tests>" >&2 + echo "" >&2 + echo " options:" >&2 + echo " -h or --help" >&2 + echo " show this message" >&2 + echo " -i or --iterations" >&2 + echo " iteration count (1 or more)" >&2 + echo " -c or --count" >&2 + echo " images created per iteration (1 or more)" >&2 + echo " -d or --delay" >&2 + echo " maximum delay between iterations" >&2 + echo "" >&2 + echo " defaults:" >&2 + echo " iterations: ${ITER_DEFAULT}" + echo " count: ${COUNT_DEFAULT}" + echo " delay: ${DELAY_DEFAULT} (seconds)" + echo "" >&2 + + [ $# -gt 0 ] && exit 1 + + exit 0 # This is used for a --help +} + +# parse command line arguments +function parseargs() { + ITER="${ITER_DEFAULT}" + COUNT="${COUNT_DEFAULT}" + DELAY="${DELAY_DEFAULT}" + + # Short option flags + SHORT_OPTS="" + SHORT_OPTS="${SHORT_OPTS},h" + SHORT_OPTS="${SHORT_OPTS},i:" + SHORT_OPTS="${SHORT_OPTS},c:" + SHORT_OPTS="${SHORT_OPTS},d:" + + # Short option flags + LONG_OPTS="" + LONG_OPTS="${LONG_OPTS},help" + LONG_OPTS="${LONG_OPTS},iterations:" + LONG_OPTS="${LONG_OPTS},count:" + LONG_OPTS="${LONG_OPTS},delay:" + + TEMP=$(getopt --name "${PROGNAME}" \ + --options "${SHORT_OPTS}" \ + --longoptions "${LONG_OPTS}" \ + -- "$@") + eval set -- "$TEMP" + + while [ "$1" != "--" ]; do + case "$1" in + -h|--help) + usage + ;; + -i|--iterations) + ITER="$2" + [ "${ITER}" -lt 1 ] && + usage "bad iterations value" + shift + ;; + -c|--count) + COUNT="$2" + [ "${COUNT}" -lt 1 ] && + usage "bad count value" + shift + ;; + -d|--delay) + DELAY="$2" + shift + ;; + *) + exit 100 # Internal error + ;; + esac + shift + done + shift +} + +function rbd_ids() { + [ $# -eq 0 ] || exit 99 + local ids + local i + + [ -d /sys/bus/rbd ] || return + ids=" $(echo $(ls /sys/bus/rbd/devices)) " + for i in ${INITIAL_RBD_IDS}; do + ids=${ids/ ${i} / } + done + echo ${ids} +} + +function update_maxes() { + local ids="$@" + local last_id + # These aren't 100% safe against concurrent updates but it + # should be pretty close + count=$(echo ${ids} | wc -w) + touch "${ID_COUNT_DIR}/${count}" + last_id=${ids% } + last_id=${last_id##* } + touch "${ID_MAX_DIR}/${last_id}" +} + +function rbd_create_image() { + [ $# -eq 0 ] || exit 99 + local image=$(basename $(mktemp "${NAMES_DIR}/image.XXXXXX")) + + rbd create "${image}" --size=1024 + echo "${image}" +} + +function rbd_image_id() { + [ $# -eq 1 ] || exit 99 + local image="$1" + + grep -l "${image}" /sys/bus/rbd/devices/*/name 2>/dev/null | + cut -d / -f 6 +} + +function rbd_map_image() { + [ $# -eq 1 ] || exit 99 + local image="$1" + local id + + sudo rbd map "${image}" --user "${CEPH_ID}" ${SECRET_ARGS} \ + > /dev/null 2>&1 + + id=$(rbd_image_id "${image}") + echo "${id}" +} + +function rbd_write_image() { + [ $# -eq 1 ] || exit 99 + local id="$1" + + # Offset and size here are meant to ensure beginning and end + # cross both (4K or 64K) page and (4MB) rbd object boundaries. + # It assumes the SOURCE_DATA file has size 66 * 2048 bytes + dd if="${SOURCE_DATA}" of="/dev/rbd${id}" bs=2048 seek=2015 \ + > /dev/null 2>&1 +} + +# All starting and ending offsets here are selected so they are not +# aligned on a (4 KB or 64 KB) page boundary +function rbd_read_image() { + [ $# -eq 1 ] || exit 99 + local id="$1" + + # First read starting and ending at an offset before any + # written data. The osd zero-fills data read from an + # existing rbd object, but before any previously-written + # data. + dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=3 \ + > /dev/null 2>&1 + # Next read starting at an offset before any written data, + # but ending at an offset that includes data that's been + # written. The osd zero-fills unwritten data at the + # beginning of a read. + dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=1983 \ + > /dev/null 2>&1 + # Read the data at offset 2015 * 2048 bytes (where it was + # written) and make sure it matches the original data. + cmp --quiet "${SOURCE_DATA}" "/dev/rbd${id}" 0 4126720 || + echo "MISMATCH!!!" + # Now read starting within the pre-written data, but ending + # beyond it. The rbd client zero-fills the unwritten + # portion at the end of a read. + dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=2079 \ + > /dev/null 2>&1 + # Now read starting from an unwritten range within a written + # rbd object. The rbd client zero-fills this. + dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=2115 \ + > /dev/null 2>&1 + # Finally read from an unwritten region which would reside + # in a different (non-existent) osd object. The osd client + # zero-fills unwritten data when the target object doesn't + # exist. + dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=4098 \ + > /dev/null 2>&1 +} + +function rbd_unmap_image() { + [ $# -eq 1 ] || exit 99 + local id="$1" + + sudo rbd unmap "/dev/rbd${id}" +} + +function rbd_destroy_image() { + [ $# -eq 1 ] || exit 99 + local image="$1" + + # Don't wait for it to complete, to increase concurrency + rbd rm "${image}" >/dev/null 2>&1 & + rm -f "${NAMES_DIR}/${image}" +} + +function one_pass() { + [ $# -eq 0 ] || exit 99 + local image + local id + local ids + local i + + image=$(rbd_create_image) + id=$(rbd_map_image "${image}") + ids=$(rbd_ids) + update_maxes "${ids}" + for i in ${rbd_ids}; do + if [ "${i}" -eq "${id}" ]; then + rbd_write_image "${i}" + else + rbd_read_image "${i}" + fi + done + rbd_unmap_image "${id}" + rbd_destroy_image "${image}" +} + +################################################################ + +parseargs "$@" + +setup + +for iter in $(seq 1 "${ITER}"); do + for count in $(seq 1 "${COUNT}"); do + one_pass & + done + # Sleep longer at first, overlap iterations more later. + # Use awk to get sub-second granularity (see sleep(1)). + sleep $(echo "${DELAY}" "${iter}" "${ITER}" | + awk '{ printf("%.2f\n", $1 - $1 * $2 / $3);}') + +done +wait + +cleanup + +exit 0 diff --git a/qa/workunits/rbd/crimson/test_crimson_librbd.sh b/qa/workunits/rbd/crimson/test_crimson_librbd.sh new file mode 100755 index 000000000..fb308de41 --- /dev/null +++ b/qa/workunits/rbd/crimson/test_crimson_librbd.sh @@ -0,0 +1,35 @@ +#!/bin/sh -e + +if [ -n "${VALGRIND}" ]; then + valgrind ${VALGRIND} --suppressions=${TESTDIR}/valgrind.supp \ + --error-exitcode=1 ceph_test_librbd +else + # Run test cases indivually to allow better selection + # of ongoing Crimson development. + # Disabled test groups are tracked here: + # https://tracker.ceph.com/issues/58791 + ceph_test_librbd --gtest_filter='TestLibRBD.*' + ceph_test_librbd --gtest_filter='EncryptedFlattenTest/0.*' + ceph_test_librbd --gtest_filter='EncryptedFlattenTest/1.*' + ceph_test_librbd --gtest_filter='EncryptedFlattenTest/2.*' + ceph_test_librbd --gtest_filter='EncryptedFlattenTest/3.*' + ceph_test_librbd --gtest_filter='EncryptedFlattenTest/4.*' + ceph_test_librbd --gtest_filter='EncryptedFlattenTest/5.*' + ceph_test_librbd --gtest_filter='EncryptedFlattenTest/6.*' + ceph_test_librbd --gtest_filter='EncryptedFlattenTest/7.*' + # ceph_test_librbd --gtest_filter='DiffIterateTest/0.*' + # ceph_test_librbd --gtest_filter='DiffIterateTest/1.*' + ceph_test_librbd --gtest_filter='TestImageWatcher.*' + ceph_test_librbd --gtest_filter='TestInternal.*' + ceph_test_librbd --gtest_filter='TestMirroring.*' + # ceph_test_librbd --gtest_filter='TestDeepCopy.*' + ceph_test_librbd --gtest_filter='TestGroup.*' + # ceph_test_librbd --gtest_filter='TestMigration.*' + ceph_test_librbd --gtest_filter='TestMirroringWatcher.*' + ceph_test_librbd --gtest_filter='TestObjectMap.*' + ceph_test_librbd --gtest_filter='TestOperations.*' + ceph_test_librbd --gtest_filter='TestTrash.*' + ceph_test_librbd --gtest_filter='TestJournalEntries.*' + ceph_test_librbd --gtest_filter='TestJournalReplay.*' +fi +exit 0 diff --git a/qa/workunits/rbd/diff.sh b/qa/workunits/rbd/diff.sh new file mode 100755 index 000000000..fbd6e0642 --- /dev/null +++ b/qa/workunits/rbd/diff.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -ex + +function cleanup() { + rbd snap purge foo || : + rbd rm foo || : + rbd snap purge foo.copy || : + rbd rm foo.copy || : + rbd snap purge foo.copy2 || : + rbd rm foo.copy2 || : + rm -f foo.diff foo.out +} + +cleanup + +rbd create foo --size 1000 +rbd bench --io-type write foo --io-size 4096 --io-threads 5 --io-total 4096000 --io-pattern rand + +#rbd cp foo foo.copy +rbd create foo.copy --size 1000 +rbd export-diff foo - | rbd import-diff - foo.copy + +rbd snap create foo --snap=two +rbd bench --io-type write foo --io-size 4096 --io-threads 5 --io-total 4096000 --io-pattern rand +rbd snap create foo --snap=three +rbd snap create foo.copy --snap=two + +rbd export-diff foo@two --from-snap three foo.diff && exit 1 || true # wrong snap order +rm -f foo.diff + +rbd export-diff foo@three --from-snap two foo.diff +rbd import-diff foo.diff foo.copy +rbd import-diff foo.diff foo.copy && exit 1 || true # this should fail with EEXIST on the end snap +rbd snap ls foo.copy | grep three + +rbd create foo.copy2 --size 1000 +rbd import-diff foo.diff foo.copy2 && exit 1 || true # this should fail bc the start snap dne + +rbd export foo foo.out +orig=`md5sum foo.out | awk '{print $1}'` +rm foo.out +rbd export foo.copy foo.out +copy=`md5sum foo.out | awk '{print $1}'` + +if [ "$orig" != "$copy" ]; then + echo does not match + exit 1 +fi + +cleanup + +echo OK + diff --git a/qa/workunits/rbd/diff_continuous.sh b/qa/workunits/rbd/diff_continuous.sh new file mode 100755 index 000000000..fd1785e07 --- /dev/null +++ b/qa/workunits/rbd/diff_continuous.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +set -ex +set -o pipefail + +function untar_workload() { + local i + for ((i = 0; i < 10; i++)); do + pv -L 10M linux-5.4.tar.gz > "${MOUNT}/linux-5.4.tar.gz" + tar -C "${MOUNT}" -xzf "${MOUNT}/linux-5.4.tar.gz" + sync "${MOUNT}" + rm -rf "${MOUNT}"/linux-5.4* + done +} + +function check_object_map() { + local spec="$1" + + rbd object-map check "${spec}" + + local flags + flags="$(rbd info "${spec}" | grep 'flags: ')" + if [[ "${flags}" =~ object\ map\ invalid ]]; then + echo "Object map invalid at ${spec}" + exit 1 + fi + if [[ "${flags}" =~ fast\ diff\ invalid ]]; then + echo "Fast diff invalid at ${spec}" + exit 1 + fi +} + +# RBD_DEVICE_TYPE is intended to be set from yaml, default to krbd +readonly DEVICE_TYPE="${RBD_DEVICE_TYPE:-krbd}" + +BASE_UUID="$(uuidgen)" +readonly BASE_UUID + +readonly SIZE="2G" +readonly SRC="${BASE_UUID}-src" +readonly DST="${BASE_UUID}-dst" +readonly MOUNT="${BASE_UUID}-mnt" + +rbd create -s "${SIZE}" --stripe-unit 64K --stripe-count 8 \ + --image-feature exclusive-lock,object-map,fast-diff "${SRC}" +rbd create -s "${SIZE}" --object-size 512K "${DST}" + +dev="$(sudo rbd device map -t "${DEVICE_TYPE}" "${SRC}")" +sudo mkfs.ext4 "${dev}" +mkdir "${MOUNT}" +sudo mount "${dev}" "${MOUNT}" +sudo chown "$(whoami)" "${MOUNT}" + +# start untar in the background +wget https://download.ceph.com/qa/linux-5.4.tar.gz +untar_workload & +untar_pid=$! + +# export initial incremental +snap_num=1 +rbd snap create "${SRC}@snap${snap_num}" +rbd export-diff "${SRC}@snap${snap_num}" "${BASE_UUID}@snap${snap_num}.diff" + +# keep exporting successive incrementals while untar is running +while kill -0 "${untar_pid}"; do + snap_num=$((snap_num + 1)) + rbd snap create "${SRC}@snap${snap_num}" + sleep $((RANDOM % 4 + 1)) + rbd export-diff --whole-object --from-snap "snap$((snap_num - 1))" \ + "${SRC}@snap${snap_num}" "${BASE_UUID}@snap${snap_num}.diff" +done + +sudo umount "${MOUNT}" +sudo rbd device unmap -t "${DEVICE_TYPE}" "${dev}" + +if ! wait "${untar_pid}"; then + echo "untar_workload failed" + exit 1 +fi + +echo "Exported ${snap_num} incrementals" +if ((snap_num < 30)); then + echo "Too few incrementals" + exit 1 +fi + +# validate +for ((i = 1; i <= snap_num; i++)); do + rbd import-diff "${BASE_UUID}@snap${i}.diff" "${DST}" + src_sum="$(rbd export "${SRC}@snap${i}" - | md5sum | awk '{print $1}')" + dst_sum="$(rbd export "${DST}@snap${i}" - | md5sum | awk '{print $1}')" + if [[ "${src_sum}" != "${dst_sum}" ]]; then + echo "Mismatch at snap${i}: ${src_sum} != ${dst_sum}" + exit 1 + fi + check_object_map "${SRC}@snap${i}" + # FIXME: this reproduces http://tracker.ceph.com/issues/37876 + # there is no fstrim involved but "rbd import-diff" can produce + # write-zeroes requests which turn into discards under the hood + # actual: EXISTS, expected: EXISTS_CLEAN inconsistency is harmless + # from a data integrity POV and data is validated above regardless, + # so just waive it for now + #check_object_map "${DST}@snap${i}" +done + +echo OK diff --git a/qa/workunits/rbd/huge-tickets.sh b/qa/workunits/rbd/huge-tickets.sh new file mode 100755 index 000000000..22853c07a --- /dev/null +++ b/qa/workunits/rbd/huge-tickets.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# This is a test for http://tracker.ceph.com/issues/8979 and the fallout +# from triaging it. #8979 itself was random crashes on corrupted memory +# due to a buffer overflow (for tickets larger than 256 bytes), further +# inspection showed that vmalloced tickets weren't handled correctly as +# well. +# +# What we are doing here is generating three huge keyrings and feeding +# them to libceph (through 'rbd map' on a scratch image). Bad kernels +# will crash reliably either on corrupted memory somewhere or a bad page +# fault in scatterwalk_pagedone(). + +set -ex + +function generate_keyring() { + local user=$1 + local n=$2 + + ceph-authtool -C -n client.$user --cap mon 'allow *' --gen-key /tmp/keyring-$user + + set +x # don't pollute trace with echos + echo -en "\tcaps osd = \"allow rwx pool=rbd" >>/tmp/keyring-$user + for i in $(seq 1 $n); do + echo -n ", allow rwx pool=pool$i" >>/tmp/keyring-$user + done + echo "\"" >>/tmp/keyring-$user + set -x +} + +generate_keyring foo 1000 # ~25K, kmalloc +generate_keyring bar 20000 # ~500K, vmalloc +generate_keyring baz 300000 # ~8M, vmalloc + sg chaining + +rbd create --size 1 test + +for user in {foo,bar,baz}; do + ceph auth import -i /tmp/keyring-$user + DEV=$(sudo rbd map -n client.$user --keyring /tmp/keyring-$user test) + sudo rbd unmap $DEV +done diff --git a/qa/workunits/rbd/image_read.sh b/qa/workunits/rbd/image_read.sh new file mode 100755 index 000000000..ddca8356e --- /dev/null +++ b/qa/workunits/rbd/image_read.sh @@ -0,0 +1,680 @@ +#!/usr/bin/env bash + +# Copyright (C) 2013 Inktank Storage, Inc. +# +# This is free software; see the source for copying conditions. +# There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. +# +# This is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as +# published by the Free Software Foundation version 2. + +# Alex Elder <elder@inktank.com> +# April 10, 2013 + +################################################################ + +# The purpose of this test is to validate that data read from a +# mapped rbd image is what it's expected to be. +# +# By default it creates an image and fills it with some data. It +# then reads back the data at a series of offsets known to cover +# various situations (such as reading the beginning, end, or the +# entirety of an object, or doing a read that spans multiple +# objects), and stashes the results in a set of local files. +# +# It also creates and maps a snapshot of the original image after +# it's been filled, and reads back the same ranges of data from the +# snapshot. It then compares the data read back with what was read +# back from the original image, verifying they match. +# +# Clone functionality is tested as well, in which case a clone is +# made of the snapshot, and the same ranges of data are again read +# and compared with the original. In addition, a snapshot of that +# clone is created, and a clone of *that* snapshot is put through +# the same set of tests. (Clone testing can be optionally skipped.) + +################################################################ + +# Default parameter values. Environment variables, if set, will +# supercede these defaults. Such variables have names that begin +# with "IMAGE_READ_", for e.g. use IMAGE_READ_PAGE_SIZE=65536 +# to use 65536 as the page size. +set -e + +DEFAULT_VERBOSE=true +DEFAULT_TEST_CLONES=true +DEFAULT_LOCAL_FILES=false +DEFAULT_FORMAT=2 +DEFAULT_DOUBLE_ORDER=true +DEFAULT_HALF_ORDER=false +DEFAULT_PAGE_SIZE=4096 +DEFAULT_OBJECT_ORDER=22 +MIN_OBJECT_ORDER=12 # technically 9, but the rbd CLI enforces 12 +MAX_OBJECT_ORDER=32 + +RBD_FORCE_ALLOW_V1=1 + +PROGNAME=$(basename $0) + +ORIGINAL=original-$$ +SNAP1=snap1-$$ +CLONE1=clone1-$$ +SNAP2=snap2-$$ +CLONE2=clone2-$$ + +function err() { + if [ $# -gt 0 ]; then + echo "${PROGNAME}: $@" >&2 + fi + exit 2 +} + +function usage() { + if [ $# -gt 0 ]; then + echo "" >&2 + echo "${PROGNAME}: $@" >&2 + fi + echo "" >&2 + echo "Usage: ${PROGNAME} [<options>]" >&2 + echo "" >&2 + echo "options are:" >&2 + echo " -o object_order" >&2 + echo " must be ${MIN_OBJECT_ORDER}..${MAX_OBJECT_ORDER}" >&2 + echo " -p page_size (in bytes)" >&2 + echo " note: there must be at least 4 pages per object" >&2 + echo " -1" >&2 + echo " test using format 1 rbd images (default)" >&2 + echo " -2" >&2 + echo " test using format 2 rbd images" >&2 + echo " -c" >&2 + echo " also test rbd clone images (implies format 2)" >&2 + echo " -d" >&2 + echo " clone object order double its parent's (format 2)" >&2 + echo " -h" >&2 + echo " clone object order half of its parent's (format 2)" >&2 + echo " -l" >&2 + echo " use local files rather than rbd images" >&2 + echo " -v" >&2 + echo " disable reporting of what's going on" >&2 + echo "" >&2 + exit 1 +} + +function verbose() { + [ "${VERBOSE}" = true ] && echo "$@" + true # Don't let the verbose test spoil our return value +} + +function quiet() { + "$@" 2> /dev/null +} + +function boolean_toggle() { + [ $# -eq 1 ] || exit 99 + test "$1" = "true" && echo false || echo true +} + +function parseargs() { + local opts="o:p:12clv" + local lopts="order:,page_size:,local,clone,verbose" + local parsed + local clone_order_msg + + # use values from environment if available + VERBOSE="${IMAGE_READ_VERBOSE:-${DEFAULT_VERBOSE}}" + TEST_CLONES="${IMAGE_READ_TEST_CLONES:-${DEFAULT_TEST_CLONES}}" + LOCAL_FILES="${IMAGE_READ_LOCAL_FILES:-${DEFAULT_LOCAL_FILES}}" + DOUBLE_ORDER="${IMAGE_READ_DOUBLE_ORDER:-${DEFAULT_DOUBLE_ORDER}}" + HALF_ORDER="${IMAGE_READ_HALF_ORDER:-${DEFAULT_HALF_ORDER}}" + FORMAT="${IMAGE_READ_FORMAT:-${DEFAULT_FORMAT}}" + PAGE_SIZE="${IMAGE_READ_PAGE_SIZE:-${DEFAULT_PAGE_SIZE}}" + OBJECT_ORDER="${IMAGE_READ_OBJECT_ORDER:-${DEFAULT_OBJECT_ORDER}}" + + parsed=$(getopt -o "${opts}" -l "${lopts}" -n "${PROGNAME}" -- "$@") || + usage + eval set -- "${parsed}" + while true; do + case "$1" in + -v|--verbose) + VERBOSE=$(boolean_toggle "${VERBOSE}");; + -c|--clone) + TEST_CLONES=$(boolean_toggle "${TEST_CLONES}");; + -d|--double) + DOUBLE_ORDER=$(boolean_toggle "${DOUBLE_ORDER}");; + -h|--half) + HALF_ORDER=$(boolean_toggle "${HALF_ORDER}");; + -l|--local) + LOCAL_FILES=$(boolean_toggle "${LOCAL_FILES}");; + -1|-2) + FORMAT="${1:1}";; + -p|--page_size) + PAGE_SIZE="$2"; shift;; + -o|--order) + OBJECT_ORDER="$2"; shift;; + --) + shift; break;; + *) + err "getopt internal error" + esac + shift + done + [ $# -gt 0 ] && usage "excess arguments ($*)" + + if [ "${TEST_CLONES}" = true ]; then + # If we're using different object orders for clones, + # make sure the limits are updated accordingly. If + # both "half" and "double" are specified, just + # ignore them both. + if [ "${DOUBLE_ORDER}" = true ]; then + if [ "${HALF_ORDER}" = true ]; then + DOUBLE_ORDER=false + HALF_ORDER=false + else + ((MAX_OBJECT_ORDER -= 2)) + fi + elif [ "${HALF_ORDER}" = true ]; then + ((MIN_OBJECT_ORDER += 2)) + fi + fi + + [ "${OBJECT_ORDER}" -lt "${MIN_OBJECT_ORDER}" ] && + usage "object order (${OBJECT_ORDER}) must be" \ + "at least ${MIN_OBJECT_ORDER}" + [ "${OBJECT_ORDER}" -gt "${MAX_OBJECT_ORDER}" ] && + usage "object order (${OBJECT_ORDER}) must be" \ + "at most ${MAX_OBJECT_ORDER}" + + if [ "${TEST_CLONES}" = true ]; then + if [ "${DOUBLE_ORDER}" = true ]; then + ((CLONE1_ORDER = OBJECT_ORDER + 1)) + ((CLONE2_ORDER = OBJECT_ORDER + 2)) + clone_order_msg="double" + elif [ "${HALF_ORDER}" = true ]; then + ((CLONE1_ORDER = OBJECT_ORDER - 1)) + ((CLONE2_ORDER = OBJECT_ORDER - 2)) + clone_order_msg="half of" + else + CLONE1_ORDER="${OBJECT_ORDER}" + CLONE2_ORDER="${OBJECT_ORDER}" + clone_order_msg="the same as" + fi + fi + + [ "${TEST_CLONES}" != true ] || FORMAT=2 + + OBJECT_SIZE=$(echo "2 ^ ${OBJECT_ORDER}" | bc) + OBJECT_PAGES=$(echo "${OBJECT_SIZE} / ${PAGE_SIZE}" | bc) + IMAGE_SIZE=$((2 * 16 * OBJECT_SIZE / (1024 * 1024))) + [ "${IMAGE_SIZE}" -lt 1 ] && IMAGE_SIZE=1 + IMAGE_OBJECTS=$((IMAGE_SIZE * (1024 * 1024) / OBJECT_SIZE)) + + [ "${OBJECT_PAGES}" -lt 4 ] && + usage "object size (${OBJECT_SIZE}) must be" \ + "at least 4 * page size (${PAGE_SIZE})" + + echo "parameters for this run:" + echo " format ${FORMAT} images will be tested" + echo " object order is ${OBJECT_ORDER}, so" \ + "objects are ${OBJECT_SIZE} bytes" + echo " page size is ${PAGE_SIZE} bytes, so" \ + "there are are ${OBJECT_PAGES} pages in an object" + echo " derived image size is ${IMAGE_SIZE} MB, so" \ + "there are ${IMAGE_OBJECTS} objects in an image" + if [ "${TEST_CLONES}" = true ]; then + echo " clone functionality will be tested" + echo " object size for a clone will be ${clone_order_msg}" + echo " the object size of its parent image" + fi + + true # Don't let the clones test spoil our return value +} + +function image_dev_path() { + [ $# -eq 1 ] || exit 99 + local image_name="$1" + + if [ "${LOCAL_FILES}" = true ]; then + echo "${TEMP}/${image_name}" + return + fi + + echo "/dev/rbd/rbd/${image_name}" +} + +function out_data_dir() { + [ $# -lt 2 ] || exit 99 + local out_data="${TEMP}/data" + local image_name + + if [ $# -eq 1 ]; then + image_name="$1" + echo "${out_data}/${image_name}" + else + echo "${out_data}" + fi +} + +function setup() { + verbose "===== setting up =====" + TEMP=$(mktemp -d /tmp/rbd_image_read.XXXXX) + mkdir -p $(out_data_dir) + + # create and fill the original image with some data + create_image "${ORIGINAL}" + map_image "${ORIGINAL}" + fill_original + + # create a snapshot of the original + create_image_snap "${ORIGINAL}" "${SNAP1}" + map_image_snap "${ORIGINAL}" "${SNAP1}" + + if [ "${TEST_CLONES}" = true ]; then + # create a clone of the original snapshot + create_snap_clone "${ORIGINAL}" "${SNAP1}" \ + "${CLONE1}" "${CLONE1_ORDER}" + map_image "${CLONE1}" + + # create a snapshot of that clone + create_image_snap "${CLONE1}" "${SNAP2}" + map_image_snap "${CLONE1}" "${SNAP2}" + + # create a clone of that clone's snapshot + create_snap_clone "${CLONE1}" "${SNAP2}" \ + "${CLONE2}" "${CLONE2_ORDER}" + map_image "${CLONE2}" + fi +} + +function teardown() { + verbose "===== cleaning up =====" + if [ "${TEST_CLONES}" = true ]; then + unmap_image "${CLONE2}" || true + destroy_snap_clone "${CLONE1}" "${SNAP2}" "${CLONE2}" || true + + unmap_image_snap "${CLONE1}" "${SNAP2}" || true + destroy_image_snap "${CLONE1}" "${SNAP2}" || true + + unmap_image "${CLONE1}" || true + destroy_snap_clone "${ORIGINAL}" "${SNAP1}" "${CLONE1}" || true + fi + unmap_image_snap "${ORIGINAL}" "${SNAP1}" || true + destroy_image_snap "${ORIGINAL}" "${SNAP1}" || true + unmap_image "${ORIGINAL}" || true + destroy_image "${ORIGINAL}" || true + + rm -rf $(out_data_dir) + rmdir "${TEMP}" +} + +function create_image() { + [ $# -eq 1 ] || exit 99 + local image_name="$1" + local image_path + local bytes + + verbose "creating image \"${image_name}\"" + if [ "${LOCAL_FILES}" = true ]; then + image_path=$(image_dev_path "${image_name}") + bytes=$(echo "${IMAGE_SIZE} * 1024 * 1024 - 1" | bc) + quiet dd if=/dev/zero bs=1 count=1 seek="${bytes}" \ + of="${image_path}" + return + fi + + rbd create "${image_name}" --image-format "${FORMAT}" \ + --size "${IMAGE_SIZE}" --order "${OBJECT_ORDER}" \ + --image-shared +} + +function destroy_image() { + [ $# -eq 1 ] || exit 99 + local image_name="$1" + local image_path + + verbose "destroying image \"${image_name}\"" + if [ "${LOCAL_FILES}" = true ]; then + image_path=$(image_dev_path "${image_name}") + rm -f "${image_path}" + return + fi + + rbd rm "${image_name}" +} + +function map_image() { + [ $# -eq 1 ] || exit 99 + local image_name="$1" # can be image@snap too + + if [ "${LOCAL_FILES}" = true ]; then + return + fi + + sudo rbd map "${image_name}" +} + +function unmap_image() { + [ $# -eq 1 ] || exit 99 + local image_name="$1" # can be image@snap too + local image_path + + if [ "${LOCAL_FILES}" = true ]; then + return + fi + image_path=$(image_dev_path "${image_name}") + + if [ -e "${image_path}" ]; then + sudo rbd unmap "${image_path}" + fi +} + +function map_image_snap() { + [ $# -eq 2 ] || exit 99 + local image_name="$1" + local snap_name="$2" + local image_snap + + if [ "${LOCAL_FILES}" = true ]; then + return + fi + + image_snap="${image_name}@${snap_name}" + map_image "${image_snap}" +} + +function unmap_image_snap() { + [ $# -eq 2 ] || exit 99 + local image_name="$1" + local snap_name="$2" + local image_snap + + if [ "${LOCAL_FILES}" = true ]; then + return + fi + + image_snap="${image_name}@${snap_name}" + unmap_image "${image_snap}" +} + +function create_image_snap() { + [ $# -eq 2 ] || exit 99 + local image_name="$1" + local snap_name="$2" + local image_snap="${image_name}@${snap_name}" + local image_path + local snap_path + + verbose "creating snapshot \"${snap_name}\"" \ + "of image \"${image_name}\"" + if [ "${LOCAL_FILES}" = true ]; then + image_path=$(image_dev_path "${image_name}") + snap_path=$(image_dev_path "${image_snap}") + + cp "${image_path}" "${snap_path}" + return + fi + + rbd snap create "${image_snap}" +} + +function destroy_image_snap() { + [ $# -eq 2 ] || exit 99 + local image_name="$1" + local snap_name="$2" + local image_snap="${image_name}@${snap_name}" + local snap_path + + verbose "destroying snapshot \"${snap_name}\"" \ + "of image \"${image_name}\"" + if [ "${LOCAL_FILES}" = true ]; then + snap_path=$(image_dev_path "${image_snap}") + rm -rf "${snap_path}" + return + fi + + rbd snap rm "${image_snap}" +} + +function create_snap_clone() { + [ $# -eq 4 ] || exit 99 + local image_name="$1" + local snap_name="$2" + local clone_name="$3" + local clone_order="$4" + local image_snap="${image_name}@${snap_name}" + local snap_path + local clone_path + + verbose "creating clone image \"${clone_name}\"" \ + "of image snapshot \"${image_name}@${snap_name}\"" + if [ "${LOCAL_FILES}" = true ]; then + snap_path=$(image_dev_path "${image_name}@${snap_name}") + clone_path=$(image_dev_path "${clone_name}") + + cp "${snap_path}" "${clone_path}" + return + fi + + rbd snap protect "${image_snap}" + rbd clone --order "${clone_order}" --image-shared \ + "${image_snap}" "${clone_name}" +} + +function destroy_snap_clone() { + [ $# -eq 3 ] || exit 99 + local image_name="$1" + local snap_name="$2" + local clone_name="$3" + local image_snap="${image_name}@${snap_name}" + local clone_path + + verbose "destroying clone image \"${clone_name}\"" + if [ "${LOCAL_FILES}" = true ]; then + clone_path=$(image_dev_path "${clone_name}") + + rm -rf "${clone_path}" + return + fi + + rbd rm "${clone_name}" + rbd snap unprotect "${image_snap}" +} + +# function that produces "random" data with which to fill the image +function source_data() { + while quiet dd if=/bin/bash skip=$(($$ % 199)) bs="${PAGE_SIZE}"; do + : # Just do the dd + done +} + +function fill_original() { + local image_path=$(image_dev_path "${ORIGINAL}") + + verbose "filling original image" + # Fill 16 objects worth of "random" data + source_data | + quiet dd bs="${PAGE_SIZE}" count=$((16 * OBJECT_PAGES)) \ + of="${image_path}" +} + +function do_read() { + [ $# -eq 3 -o $# -eq 4 ] || exit 99 + local image_name="$1" + local offset="$2" + local length="$3" + [ "${length}" -gt 0 ] || err "do_read: length must be non-zero" + local image_path=$(image_dev_path "${image_name}") + local out_data=$(out_data_dir "${image_name}") + local range=$(printf "%06u~%04u" "${offset}" "${length}") + local out_file + + [ $# -eq 4 ] && offset=$((offset + 16 * OBJECT_PAGES)) + + verbose "reading \"${image_name}\" pages ${range}" + + out_file="${out_data}/pages_${range}" + + quiet dd bs="${PAGE_SIZE}" skip="${offset}" count="${length}" \ + if="${image_path}" of="${out_file}" +} + +function one_pass() { + [ $# -eq 1 -o $# -eq 2 ] || exit 99 + local image_name="$1" + local extended + [ $# -eq 2 ] && extended="true" + local offset + local length + + offset=0 + + # +-----------+-----------+--- + # |X:X:X...X:X| : : ... : | : + # +-----------+-----------+--- + length="${OBJECT_PAGES}" + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # ---+-----------+--- + # : |X: : ... : | : + # ---+-----------+--- + length=1 + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # ---+-----------+--- + # : | :X: ... : | : + # ---+-----------+--- + length=1 + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # ---+-----------+--- + # : | : :X...X: | : + # ---+-----------+--- + length=$((OBJECT_PAGES - 3)) + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # ---+-----------+--- + # : | : : ... :X| : + # ---+-----------+--- + length=1 + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # ---+-----------+--- + # : |X:X:X...X:X| : + # ---+-----------+--- + length="${OBJECT_PAGES}" + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + offset=$((offset + 1)) # skip 1 + + # ---+-----------+--- + # : | :X:X...X:X| : + # ---+-----------+--- + length=$((OBJECT_PAGES - 1)) + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # ---+-----------+-----------+--- + # : |X:X:X...X:X|X: : ... : | : + # ---+-----------+-----------+--- + length=$((OBJECT_PAGES + 1)) + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # ---+-----------+-----------+--- + # : | :X:X...X:X|X: : ... : | : + # ---+-----------+-----------+--- + length="${OBJECT_PAGES}" + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # ---+-----------+-----------+--- + # : | :X:X...X:X|X:X: ... : | : + # ---+-----------+-----------+--- + length=$((OBJECT_PAGES + 1)) + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # ---+-----------+-----------+--- + # : | : :X...X:X|X:X:X...X:X| : + # ---+-----------+-----------+--- + length=$((2 * OBJECT_PAGES + 2)) + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + offset=$((offset + 1)) # skip 1 + + # ---+-----------+-----------+----- + # : | :X:X...X:X|X:X:X...X:X|X: : + # ---+-----------+-----------+----- + length=$((2 * OBJECT_PAGES)) + do_read "${image_name}" "${offset}" "${length}" ${extended} + offset=$((offset + length)) + + # --+-----------+-----------+-------- + # : | :X:X...X:X|X:X:X...X:X|X:X: : + # --+-----------+-----------+-------- + length=2049 + length=$((2 * OBJECT_PAGES + 1)) + do_read "${image_name}" "${offset}" "${length}" ${extended} + # offset=$((offset + length)) +} + +function run_using() { + [ $# -eq 1 ] || exit 99 + local image_name="$1" + local out_data=$(out_data_dir "${image_name}") + + verbose "===== running using \"${image_name}\" =====" + mkdir -p "${out_data}" + one_pass "${image_name}" + one_pass "${image_name}" extended +} + +function compare() { + [ $# -eq 1 ] || exit 99 + local image_name="$1" + local out_data=$(out_data_dir "${image_name}") + local original=$(out_data_dir "${ORIGINAL}") + + verbose "===== comparing \"${image_name}\" =====" + for i in $(ls "${original}"); do + verbose compare "\"${image_name}\" \"${i}\"" + cmp "${original}/${i}" "${out_data}/${i}" + done + [ "${image_name}" = "${ORIGINAL}" ] || rm -rf "${out_data}" +} + +function doit() { + [ $# -eq 1 ] || exit 99 + local image_name="$1" + + run_using "${image_name}" + compare "${image_name}" +} + +########## Start + +parseargs "$@" + +trap teardown EXIT HUP INT +setup + +run_using "${ORIGINAL}" +doit "${ORIGINAL}@${SNAP1}" +if [ "${TEST_CLONES}" = true ]; then + doit "${CLONE1}" + doit "${CLONE1}@${SNAP2}" + doit "${CLONE2}" +fi +rm -rf $(out_data_dir "${ORIGINAL}") + +echo "Success!" + +exit 0 diff --git a/qa/workunits/rbd/import_export.sh b/qa/workunits/rbd/import_export.sh new file mode 100755 index 000000000..89e8d35cf --- /dev/null +++ b/qa/workunits/rbd/import_export.sh @@ -0,0 +1,259 @@ +#!/bin/sh -ex + +# V1 image unsupported but required for testing purposes +export RBD_FORCE_ALLOW_V1=1 + +# returns data pool for a given image +get_image_data_pool () { + image=$1 + data_pool=$(rbd info $image | grep "data_pool: " | awk -F':' '{ print $NF }') + if [ -z $data_pool ]; then + data_pool='rbd' + fi + + echo $data_pool +} + +# return list of object numbers populated in image +objects () { + image=$1 + prefix=$(rbd info $image | grep block_name_prefix | awk '{print $NF;}') + + # strip off prefix and leading zeros from objects; sort, although + # it doesn't necessarily make sense as they're hex, at least it makes + # the list repeatable and comparable + objects=$(rados ls -p $(get_image_data_pool $image) | grep $prefix | \ + sed -e 's/'$prefix'\.//' -e 's/^0*\([0-9a-f]\)/\1/' | sort -u) + echo $objects +} + +# return false if either files don't compare or their ondisk +# sizes don't compare + +compare_files_and_ondisk_sizes () { + cmp -l $1 $2 || return 1 + origsize=$(stat $1 --format %b) + exportsize=$(stat $2 --format %b) + difference=$(($exportsize - $origsize)) + difference=${difference#-} # absolute value + test $difference -ge 0 -a $difference -lt 4096 +} + +TMPDIR=/tmp/rbd_import_export_$$ +rm -rf $TMPDIR +mkdir $TMPDIR +trap "rm -rf $TMPDIR" INT TERM EXIT + +# cannot import a dir +mkdir foo.$$ +rbd import foo.$$ foo.dir && exit 1 || true # should fail +rmdir foo.$$ + +# create a sparse file +dd if=/bin/sh of=${TMPDIR}/img bs=1k count=1 seek=10 +dd if=/bin/dd of=${TMPDIR}/img bs=1k count=10 seek=100 +dd if=/bin/rm of=${TMPDIR}/img bs=1k count=100 seek=1000 +dd if=/bin/ls of=${TMPDIR}/img bs=1k seek=10000 +dd if=/bin/ln of=${TMPDIR}/img bs=1k seek=100000 +dd if=/bin/grep of=${TMPDIR}/img bs=1k seek=1000000 + +rbd rm testimg || true + +rbd import $RBD_CREATE_ARGS ${TMPDIR}/img testimg +rbd export testimg ${TMPDIR}/img2 +rbd export testimg - > ${TMPDIR}/img3 +rbd rm testimg +cmp ${TMPDIR}/img ${TMPDIR}/img2 +cmp ${TMPDIR}/img ${TMPDIR}/img3 +rm ${TMPDIR}/img2 ${TMPDIR}/img3 + +# try again, importing from stdin +rbd import $RBD_CREATE_ARGS - testimg < ${TMPDIR}/img +rbd export testimg ${TMPDIR}/img2 +rbd export testimg - > ${TMPDIR}/img3 +rbd rm testimg +cmp ${TMPDIR}/img ${TMPDIR}/img2 +cmp ${TMPDIR}/img ${TMPDIR}/img3 + +rm ${TMPDIR}/img ${TMPDIR}/img2 ${TMPDIR}/img3 + +if rbd help export | grep -q export-format; then + # try with --export-format for snapshots + dd if=/bin/dd of=${TMPDIR}/img bs=1k count=10 seek=100 + rbd import $RBD_CREATE_ARGS ${TMPDIR}/img testimg + rbd snap create testimg@snap + rbd image-meta set testimg key1 value1 + IMAGEMETA_BEFORE=`rbd image-meta list testimg` + rbd export --export-format 2 testimg ${TMPDIR}/img_v2 + rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import + rbd info testimg_import + rbd info testimg_import@snap + IMAGEMETA_AFTER=`rbd image-meta list testimg_import` + [ "$IMAGEMETA_BEFORE" = "$IMAGEMETA_AFTER" ] + + # compare the contents between testimg and testimg_import + rbd export testimg_import ${TMPDIR}/img_import + compare_files_and_ondisk_sizes ${TMPDIR}/img ${TMPDIR}/img_import + + rbd export testimg@snap ${TMPDIR}/img_snap + rbd export testimg_import@snap ${TMPDIR}/img_snap_import + compare_files_and_ondisk_sizes ${TMPDIR}/img_snap ${TMPDIR}/img_snap_import + + rm ${TMPDIR}/img_v2 + rm ${TMPDIR}/img_import + rm ${TMPDIR}/img_snap + rm ${TMPDIR}/img_snap_import + + rbd snap rm testimg_import@snap + rbd remove testimg_import + rbd snap rm testimg@snap + rbd rm testimg + + # order + rbd import --order 20 ${TMPDIR}/img testimg + rbd export --export-format 2 testimg ${TMPDIR}/img_v2 + rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import + rbd info testimg_import|grep order|awk '{print $2}'|grep 20 + + rm ${TMPDIR}/img_v2 + + rbd remove testimg_import + rbd remove testimg + + # features + rbd import --image-feature layering ${TMPDIR}/img testimg + FEATURES_BEFORE=`rbd info testimg|grep features` + rbd export --export-format 2 testimg ${TMPDIR}/img_v2 + rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import + FEATURES_AFTER=`rbd info testimg_import|grep features` + if [ "$FEATURES_BEFORE" != "$FEATURES_AFTER" ]; then + false + fi + + rm ${TMPDIR}/img_v2 + + rbd remove testimg_import + rbd remove testimg + + # stripe + rbd import --stripe-count 1000 --stripe-unit 4096 ${TMPDIR}/img testimg + rbd export --export-format 2 testimg ${TMPDIR}/img_v2 + rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import + rbd info testimg_import|grep "stripe unit"|grep -Ei '(4 KiB|4096)' + rbd info testimg_import|grep "stripe count"|awk '{print $3}'|grep 1000 + + rm ${TMPDIR}/img_v2 + + rbd remove testimg_import + rbd remove testimg + + # snap protect + rbd import --image-format=2 ${TMPDIR}/img testimg + rbd snap create testimg@snap1 + rbd snap create testimg@snap2 + rbd snap protect testimg@snap2 + rbd export --export-format 2 testimg ${TMPDIR}/snap_protect + rbd import --export-format 2 ${TMPDIR}/snap_protect testimg_import + rbd info testimg_import@snap1 | grep 'protected: False' + rbd info testimg_import@snap2 | grep 'protected: True' + + rm ${TMPDIR}/snap_protect + + rbd snap unprotect testimg@snap2 + rbd snap unprotect testimg_import@snap2 + rbd snap purge testimg + rbd snap purge testimg_import + rbd remove testimg + rbd remove testimg_import +fi + +tiered=0 +if ceph osd dump | grep ^pool | grep "'rbd'" | grep tier; then + tiered=1 +fi + +# create specifically sparse files +# 1 1M block of sparse, 1 1M block of random +dd if=/dev/urandom bs=1M seek=1 count=1 of=${TMPDIR}/sparse1 + +# 1 1M block of random, 1 1M block of sparse +dd if=/dev/urandom bs=1M count=1 of=${TMPDIR}/sparse2; truncate ${TMPDIR}/sparse2 -s 2M + +# 1M-block images; validate resulting blocks + +# 1M sparse, 1M data +rbd rm sparse1 || true +rbd import $RBD_CREATE_ARGS --order 20 ${TMPDIR}/sparse1 +rbd ls -l | grep sparse1 | grep -Ei '(2 MiB|2048k)' +[ $tiered -eq 1 -o "$(objects sparse1)" = '1' ] + +# export, compare contents and on-disk size +rbd export sparse1 ${TMPDIR}/sparse1.out +compare_files_and_ondisk_sizes ${TMPDIR}/sparse1 ${TMPDIR}/sparse1.out +rm ${TMPDIR}/sparse1.out +rbd rm sparse1 + +# 1M data, 1M sparse +rbd rm sparse2 || true +rbd import $RBD_CREATE_ARGS --order 20 ${TMPDIR}/sparse2 +rbd ls -l | grep sparse2 | grep -Ei '(2 MiB|2048k)' +[ $tiered -eq 1 -o "$(objects sparse2)" = '0' ] +rbd export sparse2 ${TMPDIR}/sparse2.out +compare_files_and_ondisk_sizes ${TMPDIR}/sparse2 ${TMPDIR}/sparse2.out +rm ${TMPDIR}/sparse2.out +rbd rm sparse2 + +# extend sparse1 to 10 1M blocks, sparse at the end +truncate ${TMPDIR}/sparse1 -s 10M +# import from stdin just for fun, verify still sparse +rbd import $RBD_CREATE_ARGS --order 20 - sparse1 < ${TMPDIR}/sparse1 +rbd ls -l | grep sparse1 | grep -Ei '(10 MiB|10240k)' +[ $tiered -eq 1 -o "$(objects sparse1)" = '1' ] +rbd export sparse1 ${TMPDIR}/sparse1.out +compare_files_and_ondisk_sizes ${TMPDIR}/sparse1 ${TMPDIR}/sparse1.out +rm ${TMPDIR}/sparse1.out +rbd rm sparse1 + +# extend sparse2 to 4M total with two more nonsparse megs +dd if=/dev/urandom bs=2M count=1 of=${TMPDIR}/sparse2 oflag=append conv=notrunc +# again from stding +rbd import $RBD_CREATE_ARGS --order 20 - sparse2 < ${TMPDIR}/sparse2 +rbd ls -l | grep sparse2 | grep -Ei '(4 MiB|4096k)' +[ $tiered -eq 1 -o "$(objects sparse2)" = '0 2 3' ] +rbd export sparse2 ${TMPDIR}/sparse2.out +compare_files_and_ondisk_sizes ${TMPDIR}/sparse2 ${TMPDIR}/sparse2.out +rm ${TMPDIR}/sparse2.out +rbd rm sparse2 + +# zeros import to a sparse image. Note: all zeros currently +# doesn't work right now due to the way we handle 'empty' fiemaps; +# the image ends up zero-filled. + +echo "partially-sparse file imports to partially-sparse image" +rbd import $RBD_CREATE_ARGS --order 20 ${TMPDIR}/sparse1 sparse +[ $tiered -eq 1 -o "$(objects sparse)" = '1' ] +rbd rm sparse + +echo "zeros import through stdin to sparse image" +# stdin +dd if=/dev/zero bs=1M count=4 | rbd import $RBD_CREATE_ARGS - sparse +[ $tiered -eq 1 -o "$(objects sparse)" = '' ] +rbd rm sparse + +echo "zeros export to sparse file" +# Must be tricky to make image "by hand" ; import won't create a zero image +rbd create $RBD_CREATE_ARGS sparse --size 4 +prefix=$(rbd info sparse | grep block_name_prefix | awk '{print $NF;}') +# drop in 0 object directly +dd if=/dev/zero bs=4M count=1 | rados -p $(get_image_data_pool sparse) \ + put ${prefix}.000000000000 - +[ $tiered -eq 1 -o "$(objects sparse)" = '0' ] +# 1 object full of zeros; export should still create 0-disk-usage file +rm ${TMPDIR}/sparse || true +rbd export sparse ${TMPDIR}/sparse +[ $(stat ${TMPDIR}/sparse --format=%b) = '0' ] +rbd rm sparse + +rm ${TMPDIR}/sparse ${TMPDIR}/sparse1 ${TMPDIR}/sparse2 ${TMPDIR}/sparse3 || true + +echo OK diff --git a/qa/workunits/rbd/issue-20295.sh b/qa/workunits/rbd/issue-20295.sh new file mode 100755 index 000000000..3d617a066 --- /dev/null +++ b/qa/workunits/rbd/issue-20295.sh @@ -0,0 +1,18 @@ +#!/bin/sh -ex + +TEST_POOL=ecpool +TEST_IMAGE=test1 +PGS=12 + +ceph osd pool create $TEST_POOL $PGS $PGS erasure +ceph osd pool application enable $TEST_POOL rbd +ceph osd pool set $TEST_POOL allow_ec_overwrites true +rbd --data-pool $TEST_POOL create --size 1024G $TEST_IMAGE +rbd bench \ + --io-type write \ + --io-size 4096 \ + --io-pattern=rand \ + --io-total 100M \ + $TEST_IMAGE + +echo "OK" diff --git a/qa/workunits/rbd/journal.sh b/qa/workunits/rbd/journal.sh new file mode 100755 index 000000000..ba89e75c9 --- /dev/null +++ b/qa/workunits/rbd/journal.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +set -e + +. $(dirname $0)/../../standalone/ceph-helpers.sh + +function list_tests() +{ + echo "AVAILABLE TESTS" + for i in $TESTS; do + echo " $i" + done +} + +function usage() +{ + echo "usage: $0 [-h|-l|-t <testname> [-t <testname>...] [--no-cleanup]]" +} + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +function save_commit_position() +{ + local journal=$1 + + rados -p rbd getomapval journal.${journal} client_ \ + $TMPDIR/${journal}.client_.omap +} + +function restore_commit_position() +{ + local journal=$1 + + rados -p rbd setomapval journal.${journal} client_ \ + < $TMPDIR/${journal}.client_.omap +} + +test_rbd_journal() +{ + local image=testrbdjournal$$ + + rbd create --image-feature exclusive-lock --image-feature journaling \ + --size 128 ${image} + local journal=$(rbd info ${image} --format=xml 2>/dev/null | + $XMLSTARLET sel -t -v "//image/journal") + test -n "${journal}" + rbd journal info ${journal} + rbd journal info --journal ${journal} + rbd journal info --image ${image} + + rbd feature disable ${image} journaling + + rbd info ${image} --format=xml 2>/dev/null | + expect_false $XMLSTARLET sel -t -v "//image/journal" + expect_false rbd journal info ${journal} + expect_false rbd journal info --image ${image} + + rbd feature enable ${image} journaling + + local journal1=$(rbd info ${image} --format=xml 2>/dev/null | + $XMLSTARLET sel -t -v "//image/journal") + test "${journal}" = "${journal1}" + + rbd journal info ${journal} + + rbd journal status ${journal} + + local count=10 + save_commit_position ${journal} + rbd bench --io-type write ${image} --io-size 4096 --io-threads 1 \ + --io-total $((4096 * count)) --io-pattern seq + rbd journal status --image ${image} | fgrep "tid=$((count - 1))" + restore_commit_position ${journal} + rbd journal status --image ${image} | fgrep "positions=[]" + local count1=$(rbd journal inspect --verbose ${journal} | + grep -c 'event_type.*AioWrite') + test "${count}" -eq "${count1}" + + rbd journal export ${journal} $TMPDIR/journal.export + local size=$(stat -c "%s" $TMPDIR/journal.export) + test "${size}" -gt 0 + + rbd export ${image} $TMPDIR/${image}.export + + local image1=${image}1 + rbd create --image-feature exclusive-lock --image-feature journaling \ + --size 128 ${image1} + journal1=$(rbd info ${image1} --format=xml 2>/dev/null | + $XMLSTARLET sel -t -v "//image/journal") + + save_commit_position ${journal1} + rbd journal import --dest ${image1} $TMPDIR/journal.export + rbd snap create ${image1}@test + restore_commit_position ${journal1} + # check that commit position is properly updated: the journal should contain + # 14 entries (2 AioFlush + 10 AioWrite + 1 SnapCreate + 1 OpFinish) and + # commit position set to tid=14 + rbd journal inspect --image ${image1} --verbose | awk ' + /AioFlush/ {a++} # match: "event_type": "AioFlush", + /AioWrite/ {w++} # match: "event_type": "AioWrite", + /SnapCreate/ {s++} # match: "event_type": "SnapCreate", + /OpFinish/ {f++} # match: "event_type": "OpFinish", + /entries inspected/ {t=$1; e=$4} # match: 14 entries inspected, 0 errors + {print} # for diagnostic + END { + if (a != 2 || w != 10 || s != 1 || f != 1 || t != 14 || e != 0) exit(1) + } + ' + + rbd export ${image1}@test $TMPDIR/${image1}.export + cmp $TMPDIR/${image}.export $TMPDIR/${image1}.export + + rbd journal reset ${journal} + + rbd journal inspect --verbose ${journal} | expect_false grep 'event_type' + + rbd snap purge ${image1} + rbd remove ${image1} + rbd remove ${image} +} + + +rbd_assert_eq() { + local image=$1 + local cmd=$2 + local param=$3 + local expected_val=$4 + + local val=$(rbd --format xml ${cmd} --image ${image} | + $XMLSTARLET sel -t -v "${param}") + test "${val}" = "${expected_val}" +} + +test_rbd_create() +{ + local image=testrbdcreate$$ + + rbd create --image-feature exclusive-lock --image-feature journaling \ + --journal-pool rbd \ + --journal-object-size 20M \ + --journal-splay-width 6 \ + --size 256 ${image} + + rbd_assert_eq ${image} 'journal info' '//journal/order' 25 + rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6 + rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd + + rbd remove ${image} +} + +test_rbd_copy() +{ + local src=testrbdcopys$$ + rbd create --size 256 ${src} + + local image=testrbdcopy$$ + rbd copy --image-feature exclusive-lock --image-feature journaling \ + --journal-pool rbd \ + --journal-object-size 20M \ + --journal-splay-width 6 \ + ${src} ${image} + + rbd remove ${src} + + rbd_assert_eq ${image} 'journal info' '//journal/order' 25 + rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6 + rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd + + rbd remove ${image} +} + +test_rbd_deep_copy() +{ + local src=testrbdcopys$$ + rbd create --size 256 ${src} + rbd snap create ${src}@snap1 + + local dest=testrbdcopy$$ + rbd deep copy --image-feature exclusive-lock --image-feature journaling \ + --journal-pool rbd \ + --journal-object-size 20M \ + --journal-splay-width 6 \ + ${src} ${dest} + + rbd snap purge ${src} + rbd remove ${src} + + rbd_assert_eq ${dest} 'journal info' '//journal/order' 25 + rbd_assert_eq ${dest} 'journal info' '//journal/splay_width' 6 + rbd_assert_eq ${dest} 'journal info' '//journal/object_pool' rbd + + rbd snap purge ${dest} + rbd remove ${dest} +} + +test_rbd_clone() +{ + local parent=testrbdclonep$$ + rbd create --image-feature layering --size 256 ${parent} + rbd snap create ${parent}@snap + rbd snap protect ${parent}@snap + + local image=testrbdclone$$ + rbd clone --image-feature layering --image-feature exclusive-lock --image-feature journaling \ + --journal-pool rbd \ + --journal-object-size 20M \ + --journal-splay-width 6 \ + ${parent}@snap ${image} + + rbd_assert_eq ${image} 'journal info' '//journal/order' 25 + rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6 + rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd + + rbd remove ${image} + rbd snap unprotect ${parent}@snap + rbd snap purge ${parent} + rbd remove ${parent} +} + +test_rbd_import() +{ + local src=testrbdimports$$ + rbd create --size 256 ${src} + + rbd export ${src} $TMPDIR/${src}.export + rbd remove ${src} + + local image=testrbdimport$$ + rbd import --image-feature exclusive-lock --image-feature journaling \ + --journal-pool rbd \ + --journal-object-size 20M \ + --journal-splay-width 6 \ + $TMPDIR/${src}.export ${image} + + rbd_assert_eq ${image} 'journal info' '//journal/order' 25 + rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6 + rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd + + rbd remove ${image} +} + +test_rbd_feature() +{ + local image=testrbdfeature$$ + + rbd create --image-feature exclusive-lock --size 256 ${image} + + rbd feature enable ${image} journaling \ + --journal-pool rbd \ + --journal-object-size 20M \ + --journal-splay-width 6 + + rbd_assert_eq ${image} 'journal info' '//journal/order' 25 + rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6 + rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd + + rbd remove ${image} +} + +TESTS+=" rbd_journal" +TESTS+=" rbd_create" +TESTS+=" rbd_copy" +TESTS+=" rbd_clone" +TESTS+=" rbd_import" +TESTS+=" rbd_feature" + +# +# "main" follows +# + +tests_to_run=() + +cleanup=true + +while [[ $# -gt 0 ]]; do + opt=$1 + + case "$opt" in + "-l" ) + do_list=1 + ;; + "--no-cleanup" ) + cleanup=false + ;; + "-t" ) + shift + if [[ -z "$1" ]]; then + echo "missing argument to '-t'" + usage ; + exit 1 + fi + tests_to_run+=" $1" + ;; + "-h" ) + usage ; + exit 0 + ;; + esac + shift +done + +if [[ $do_list -eq 1 ]]; then + list_tests ; + exit 0 +fi + +TMPDIR=/tmp/rbd_journal$$ +mkdir $TMPDIR +if $cleanup; then + trap "rm -fr $TMPDIR" 0 +fi + +if test -z "$tests_to_run" ; then + tests_to_run="$TESTS" +fi + +for i in $tests_to_run; do + set -x + test_${i} + set +x +done + +echo OK diff --git a/qa/workunits/rbd/kernel.sh b/qa/workunits/rbd/kernel.sh new file mode 100755 index 000000000..faa5760ee --- /dev/null +++ b/qa/workunits/rbd/kernel.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -ex + +CEPH_SECRET_FILE=${CEPH_SECRET_FILE:-} +CEPH_ID=${CEPH_ID:-admin} +SECRET_ARGS='' +if [ ! -z $CEPH_SECRET_FILE ]; then + SECRET_ARGS="--secret $CEPH_SECRET_FILE" +fi + +TMP_FILES="/tmp/img1 /tmp/img1.small /tmp/img1.snap1 /tmp/img1.export /tmp/img1.trunc" + +function expect_false() { + if "$@"; then return 1; else return 0; fi +} + +function get_device_dir { + local POOL=$1 + local IMAGE=$2 + local SNAP=$3 + rbd device list | tail -n +2 | egrep "\s+$POOL\s+$IMAGE\s+$SNAP\s+" | + awk '{print $1;}' +} + +function clean_up { + [ -e /dev/rbd/rbd/testimg1@snap1 ] && + sudo rbd device unmap /dev/rbd/rbd/testimg1@snap1 + if [ -e /dev/rbd/rbd/testimg1 ]; then + sudo rbd device unmap /dev/rbd/rbd/testimg1 + rbd snap purge testimg1 || true + fi + rbd ls | grep testimg1 > /dev/null && rbd rm testimg1 || true + sudo rm -f $TMP_FILES +} + +clean_up + +trap clean_up INT TERM EXIT + +# create an image +dd if=/bin/sh of=/tmp/img1 bs=1k count=1 seek=10 +dd if=/bin/dd of=/tmp/img1 bs=1k count=10 seek=100 +dd if=/bin/rm of=/tmp/img1 bs=1k count=100 seek=1000 +dd if=/bin/ls of=/tmp/img1 bs=1k seek=10000 +dd if=/bin/ln of=/tmp/img1 bs=1k seek=100000 +dd if=/dev/zero of=/tmp/img1 count=0 seek=150000 + +# import +rbd import /tmp/img1 testimg1 +sudo rbd device map testimg1 --user $CEPH_ID $SECRET_ARGS + +DEV_ID1=$(get_device_dir rbd testimg1 -) +echo "dev_id1 = $DEV_ID1" +cat /sys/bus/rbd/devices/$DEV_ID1/size +cat /sys/bus/rbd/devices/$DEV_ID1/size | grep 76800000 + +sudo dd if=/dev/rbd/rbd/testimg1 of=/tmp/img1.export +cmp /tmp/img1 /tmp/img1.export + +# snapshot +rbd snap create testimg1 --snap=snap1 +sudo rbd device map --snap=snap1 testimg1 --user $CEPH_ID $SECRET_ARGS + +DEV_ID2=$(get_device_dir rbd testimg1 snap1) +cat /sys/bus/rbd/devices/$DEV_ID2/size | grep 76800000 + +sudo dd if=/dev/rbd/rbd/testimg1@snap1 of=/tmp/img1.snap1 +cmp /tmp/img1 /tmp/img1.snap1 + +# resize +rbd resize testimg1 --size=40 --allow-shrink +cat /sys/bus/rbd/devices/$DEV_ID1/size | grep 41943040 +cat /sys/bus/rbd/devices/$DEV_ID2/size | grep 76800000 + +sudo dd if=/dev/rbd/rbd/testimg1 of=/tmp/img1.small +cp /tmp/img1 /tmp/img1.trunc +truncate -s 41943040 /tmp/img1.trunc +cmp /tmp/img1.trunc /tmp/img1.small + +# rollback expects an unlocked image +# (acquire and) release the lock as a side effect +rbd bench --io-type read --io-size 1 --io-threads 1 --io-total 1 testimg1 + +# rollback and check data again +rbd snap rollback --snap=snap1 testimg1 +cat /sys/bus/rbd/devices/$DEV_ID1/size | grep 76800000 +cat /sys/bus/rbd/devices/$DEV_ID2/size | grep 76800000 +sudo rm -f /tmp/img1.snap1 /tmp/img1.export + +sudo dd if=/dev/rbd/rbd/testimg1@snap1 of=/tmp/img1.snap1 +cmp /tmp/img1 /tmp/img1.snap1 +sudo dd if=/dev/rbd/rbd/testimg1 of=/tmp/img1.export +cmp /tmp/img1 /tmp/img1.export + +# zeros are returned if an image or a snapshot is removed +expect_false cmp -n 76800000 /dev/rbd/rbd/testimg1@snap1 /dev/zero +rbd snap rm --snap=snap1 testimg1 +cmp -n 76800000 /dev/rbd/rbd/testimg1@snap1 /dev/zero + +echo OK diff --git a/qa/workunits/rbd/krbd_data_pool.sh b/qa/workunits/rbd/krbd_data_pool.sh new file mode 100755 index 000000000..8eada88bb --- /dev/null +++ b/qa/workunits/rbd/krbd_data_pool.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash + +set -ex + +export RBD_FORCE_ALLOW_V1=1 + +function fill_image() { + local spec=$1 + + local dev + dev=$(sudo rbd map $spec) + xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 -W 0 $IMAGE_SIZE" $dev + sudo rbd unmap $dev +} + +function create_clones() { + local spec=$1 + + rbd snap create $spec@snap + rbd snap protect $spec@snap + + local pool=${spec%/*} # pool/image is assumed + local image=${spec#*/} + local child_pool + for child_pool in $pool clonesonly; do + rbd clone $spec@snap $child_pool/$pool-$image-clone1 + rbd clone $spec@snap --data-pool repdata $child_pool/$pool-$image-clone2 + rbd clone $spec@snap --data-pool ecdata $child_pool/$pool-$image-clone3 + done +} + +function trigger_copyup() { + local spec=$1 + + local dev + dev=$(sudo rbd map $spec) + local i + { + for ((i = 0; i < $NUM_OBJECTS; i++)); do + echo pwrite -b $OBJECT_SIZE -S 0x59 $((i * OBJECT_SIZE + OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2)) + done + echo fsync + echo quit + } | xfs_io $dev + sudo rbd unmap $dev +} + +function compare() { + local spec=$1 + local object=$2 + + local dev + dev=$(sudo rbd map $spec) + local i + for ((i = 0; i < $NUM_OBJECTS; i++)); do + dd if=$dev bs=$OBJECT_SIZE count=1 skip=$i | cmp $object - + done + sudo rbd unmap $dev +} + +function mkfs_and_mount() { + local spec=$1 + + local dev + dev=$(sudo rbd map $spec) + blkdiscard $dev + mkfs.ext4 -q -E nodiscard $dev + sudo mount $dev /mnt + sudo umount /mnt + sudo rbd unmap $dev +} + +function list_HEADs() { + local pool=$1 + + rados -p $pool ls | while read obj; do + if rados -p $pool stat $obj >/dev/null 2>&1; then + echo $obj + fi + done +} + +function count_data_objects() { + local spec=$1 + + local pool + pool=$(rbd info $spec | grep 'data_pool: ' | awk '{ print $NF }') + if [[ -z $pool ]]; then + pool=${spec%/*} # pool/image is assumed + fi + + local prefix + prefix=$(rbd info $spec | grep 'block_name_prefix: ' | awk '{ print $NF }') + rados -p $pool ls | grep -c $prefix +} + +function get_num_clones() { + local pool=$1 + + rados -p $pool --format=json df | + python3 -c 'import sys, json; print(json.load(sys.stdin)["pools"][0]["num_object_clones"])' +} + +ceph osd pool create repdata 24 24 +rbd pool init repdata +ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2 +ceph osd pool create ecdata 24 24 erasure teuthologyprofile +rbd pool init ecdata +ceph osd pool set ecdata allow_ec_overwrites true +ceph osd pool create rbdnonzero 24 24 +rbd pool init rbdnonzero +ceph osd pool create clonesonly 24 24 +rbd pool init clonesonly + +for pool in rbd rbdnonzero; do + rbd create --size 200 --image-format 1 $pool/img0 + rbd create --size 200 $pool/img1 + rbd create --size 200 --data-pool repdata $pool/img2 + rbd create --size 200 --data-pool ecdata $pool/img3 +done + +IMAGE_SIZE=$(rbd info --format=json img1 | python3 -c 'import sys, json; print(json.load(sys.stdin)["size"])') +OBJECT_SIZE=$(rbd info --format=json img1 | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_size"])') +NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE)) +[[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]] + +OBJECT_X=$(mktemp) # xxxx +xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $OBJECT_SIZE" $OBJECT_X + +OBJECT_XY=$(mktemp) # xxYY +xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $((OBJECT_SIZE / 2))" \ + -c "pwrite -b $OBJECT_SIZE -S 0x59 $((OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))" \ + $OBJECT_XY + +for pool in rbd rbdnonzero; do + for i in {0..3}; do + fill_image $pool/img$i + if [[ $i -ne 0 ]]; then + create_clones $pool/img$i + for child_pool in $pool clonesonly; do + for j in {1..3}; do + trigger_copyup $child_pool/$pool-img$i-clone$j + done + done + fi + done +done + +# rbd_directory, rbd_children, rbd_info + img0 header + ... +NUM_META_RBDS=$((3 + 1 + 3 * (1*2 + 3*2))) +# rbd_directory, rbd_children, rbd_info + ... +NUM_META_CLONESONLY=$((3 + 2 * 3 * (3*2))) + +[[ $(rados -p rbd ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]] +[[ $(rados -p repdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]] +[[ $(rados -p ecdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]] +[[ $(rados -p rbdnonzero ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]] +[[ $(rados -p clonesonly ls | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]] + +for pool in rbd rbdnonzero; do + for i in {0..3}; do + [[ $(count_data_objects $pool/img$i) -eq $NUM_OBJECTS ]] + if [[ $i -ne 0 ]]; then + for child_pool in $pool clonesonly; do + for j in {1..3}; do + [[ $(count_data_objects $child_pool/$pool-img$i-clone$j) -eq $NUM_OBJECTS ]] + done + done + fi + done +done + +[[ $(get_num_clones rbd) -eq 0 ]] +[[ $(get_num_clones repdata) -eq 0 ]] +[[ $(get_num_clones ecdata) -eq 0 ]] +[[ $(get_num_clones rbdnonzero) -eq 0 ]] +[[ $(get_num_clones clonesonly) -eq 0 ]] + +for pool in rbd rbdnonzero; do + for i in {0..3}; do + compare $pool/img$i $OBJECT_X + mkfs_and_mount $pool/img$i + if [[ $i -ne 0 ]]; then + for child_pool in $pool clonesonly; do + for j in {1..3}; do + compare $child_pool/$pool-img$i-clone$j $OBJECT_XY + done + done + fi + done +done + +# mkfs_and_mount should discard some objects everywhere but in clonesonly +[[ $(list_HEADs rbd | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]] +[[ $(list_HEADs repdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]] +[[ $(list_HEADs ecdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]] +[[ $(list_HEADs rbdnonzero | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]] +[[ $(list_HEADs clonesonly | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]] + +[[ $(get_num_clones rbd) -eq $NUM_OBJECTS ]] +[[ $(get_num_clones repdata) -eq $((2 * NUM_OBJECTS)) ]] +[[ $(get_num_clones ecdata) -eq $((2 * NUM_OBJECTS)) ]] +[[ $(get_num_clones rbdnonzero) -eq $NUM_OBJECTS ]] +[[ $(get_num_clones clonesonly) -eq 0 ]] + +echo OK diff --git a/qa/workunits/rbd/krbd_exclusive_option.sh b/qa/workunits/rbd/krbd_exclusive_option.sh new file mode 100755 index 000000000..f8493ce98 --- /dev/null +++ b/qa/workunits/rbd/krbd_exclusive_option.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash + +set -ex + +function expect_false() { + if "$@"; then return 1; else return 0; fi +} + +function assert_locked() { + local dev_id="${1#/dev/rbd}" + + local client_addr + client_addr="$(< $SYSFS_DIR/$dev_id/client_addr)" + + local client_id + client_id="$(< $SYSFS_DIR/$dev_id/client_id)" + # client4324 -> client.4324 + client_id="client.${client_id#client}" + + local watch_cookie + watch_cookie="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID | + grep $client_id | cut -d ' ' -f 3 | cut -d '=' -f 2)" + [[ $(echo -n "$watch_cookie" | grep -c '^') -eq 1 ]] + + local actual + actual="$(rados -p rbd --format=json lock info rbd_header.$IMAGE_ID rbd_lock | + python3 -m json.tool --sort-keys)" + + local expected + expected="$(cat <<EOF | python3 -m json.tool --sort-keys +{ + "lockers": [ + { + "addr": "$client_addr", + "cookie": "auto $watch_cookie", + "description": "", + "expiration": "0.000000", + "name": "$client_id" + } + ], + "name": "rbd_lock", + "tag": "internal", + "type": "exclusive" +} +EOF + )" + + [ "$actual" = "$expected" ] +} + +function assert_unlocked() { + rados -p rbd --format=json lock info rbd_header.$IMAGE_ID rbd_lock | + grep '"lockers":\[\]' +} + +function blocklist_add() { + local dev_id="${1#/dev/rbd}" + + local client_addr + client_addr="$(< $SYSFS_DIR/$dev_id/client_addr)" + + ceph osd blocklist add $client_addr +} + +SYSFS_DIR="/sys/bus/rbd/devices" +IMAGE_NAME="exclusive-option-test" + +rbd create --size 1 --image-feature '' $IMAGE_NAME + +IMAGE_ID="$(rbd info --format=json $IMAGE_NAME | + python3 -c "import sys, json; print(json.load(sys.stdin)['block_name_prefix'].split('.')[1])")" + +DEV=$(sudo rbd map $IMAGE_NAME) +assert_unlocked +sudo rbd unmap $DEV +assert_unlocked + +expect_false sudo rbd map -o exclusive $IMAGE_NAME +assert_unlocked + +expect_false sudo rbd map -o lock_on_read $IMAGE_NAME +assert_unlocked + +rbd feature enable $IMAGE_NAME exclusive-lock +rbd snap create $IMAGE_NAME@snap + +DEV=$(sudo rbd map $IMAGE_NAME) +assert_locked $DEV +[[ $(blockdev --getro $DEV) -eq 0 ]] +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map $IMAGE_NAME@snap) +assert_unlocked +[[ $(blockdev --getro $DEV) -eq 1 ]] +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map -o ro $IMAGE_NAME) +assert_unlocked +[[ $(blockdev --getro $DEV) -eq 1 ]] +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map -o exclusive $IMAGE_NAME) +assert_locked $DEV +[[ $(blockdev --getro $DEV) -eq 0 ]] +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map -o exclusive $IMAGE_NAME@snap) +assert_unlocked +[[ $(blockdev --getro $DEV) -eq 1 ]] +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map -o exclusive,ro $IMAGE_NAME) +assert_unlocked +[[ $(blockdev --getro $DEV) -eq 1 ]] +sudo rbd unmap $DEV +assert_unlocked + +# alternate syntax +DEV=$(sudo rbd map --exclusive --read-only $IMAGE_NAME) +assert_unlocked +[[ $(blockdev --getro $DEV) -eq 1 ]] +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map $IMAGE_NAME) +assert_locked $DEV +OTHER_DEV=$(sudo rbd map -o noshare $IMAGE_NAME) +assert_locked $OTHER_DEV +dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct +assert_locked $DEV +dd if=/dev/urandom of=$OTHER_DEV bs=4k count=10 oflag=direct +assert_locked $OTHER_DEV +sudo rbd unmap $DEV +sudo rbd unmap $OTHER_DEV +assert_unlocked + +DEV=$(sudo rbd map $IMAGE_NAME) +assert_locked $DEV +OTHER_DEV=$(sudo rbd map -o noshare,exclusive $IMAGE_NAME) +assert_locked $OTHER_DEV +dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct +expect_false dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct +assert_locked $OTHER_DEV +sudo rbd unmap $OTHER_DEV +assert_unlocked +dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct +assert_unlocked +dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct +assert_locked $DEV +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map -o lock_on_read $IMAGE_NAME) +assert_locked $DEV +OTHER_DEV=$(sudo rbd map -o noshare,exclusive $IMAGE_NAME) +assert_locked $OTHER_DEV +expect_false dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct +expect_false dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct +sudo udevadm settle +assert_locked $OTHER_DEV +sudo rbd unmap $OTHER_DEV +assert_unlocked +dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct +assert_locked $DEV +dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct +assert_locked $DEV +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map -o exclusive $IMAGE_NAME) +assert_locked $DEV +expect_false sudo rbd map -o noshare $IMAGE_NAME +assert_locked $DEV +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map -o exclusive $IMAGE_NAME) +assert_locked $DEV +expect_false sudo rbd map -o noshare,exclusive $IMAGE_NAME +assert_locked $DEV +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map $IMAGE_NAME) +assert_locked $DEV +rbd resize --size 1G $IMAGE_NAME +assert_unlocked +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map -o exclusive $IMAGE_NAME) +assert_locked $DEV +expect_false rbd resize --size 2G $IMAGE_NAME +assert_locked $DEV +sudo rbd unmap $DEV +assert_unlocked + +DEV=$(sudo rbd map $IMAGE_NAME) +assert_locked $DEV +dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct +{ sleep 10; blocklist_add $DEV; } & +PID=$! +expect_false dd if=/dev/urandom of=$DEV bs=4k count=200000 oflag=direct +wait $PID +# break lock +OTHER_DEV=$(sudo rbd map -o noshare $IMAGE_NAME) +assert_locked $OTHER_DEV +sudo rbd unmap $DEV +assert_locked $OTHER_DEV +sudo rbd unmap $OTHER_DEV +assert_unlocked + +# induce a watch error after 30 seconds +DEV=$(sudo rbd map -o exclusive,osdkeepalive=60 $IMAGE_NAME) +assert_locked $DEV +OLD_WATCHER="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID)" +sleep 40 +assert_locked $DEV +NEW_WATCHER="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID)" +# same client_id, old cookie < new cookie +[ "$(echo "$OLD_WATCHER" | cut -d ' ' -f 2)" = \ + "$(echo "$NEW_WATCHER" | cut -d ' ' -f 2)" ] +[[ $(echo "$OLD_WATCHER" | cut -d ' ' -f 3 | cut -d '=' -f 2) -lt \ + $(echo "$NEW_WATCHER" | cut -d ' ' -f 3 | cut -d '=' -f 2) ]] +sudo rbd unmap $DEV +assert_unlocked + +echo OK diff --git a/qa/workunits/rbd/krbd_fallocate.sh b/qa/workunits/rbd/krbd_fallocate.sh new file mode 100755 index 000000000..79efa1a8b --- /dev/null +++ b/qa/workunits/rbd/krbd_fallocate.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# - fallocate -z deallocates because BLKDEV_ZERO_NOUNMAP hint is ignored by +# krbd +# +# - big unaligned blkdiscard and fallocate -z/-p leave the objects in place + +set -ex + +# no blkdiscard(8) in trusty +function py_blkdiscard() { + local offset=$1 + + python3 <<EOF +import fcntl, struct +BLKDISCARD = 0x1277 +with open('$DEV', 'w') as dev: + fcntl.ioctl(dev, BLKDISCARD, struct.pack('QQ', $offset, $IMAGE_SIZE - $offset)) +EOF +} + +# fallocate(1) in trusty doesn't support -z/-p +function py_fallocate() { + local mode=$1 + local offset=$2 + + python3 <<EOF +import os, ctypes, ctypes.util +FALLOC_FL_KEEP_SIZE = 0x01 +FALLOC_FL_PUNCH_HOLE = 0x02 +FALLOC_FL_ZERO_RANGE = 0x10 +libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) +with open('$DEV', 'w') as dev: + if libc.fallocate(dev.fileno(), ctypes.c_int($mode), ctypes.c_long($offset), ctypes.c_long($IMAGE_SIZE - $offset)): + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) +EOF +} + +function allocate() { + xfs_io -c "pwrite -b $OBJECT_SIZE -W 0 $IMAGE_SIZE" $DEV + assert_allocated +} + +function assert_allocated() { + cmp <(od -xAx $DEV) - <<EOF +000000 cdcd cdcd cdcd cdcd cdcd cdcd cdcd cdcd +* +$(printf %x $IMAGE_SIZE) +EOF + [[ $(rados -p rbd ls | grep -c rbd_data.$IMAGE_ID) -eq $NUM_OBJECTS ]] +} + +function assert_zeroes() { + local num_objects_expected=$1 + + cmp <(od -xAx $DEV) - <<EOF +000000 0000 0000 0000 0000 0000 0000 0000 0000 +* +$(printf %x $IMAGE_SIZE) +EOF + [[ $(rados -p rbd ls | grep -c rbd_data.$IMAGE_ID) -eq $num_objects_expected ]] +} + +function assert_zeroes_unaligned() { + local num_objects_expected=$1 + + cmp <(od -xAx $DEV) - <<EOF +000000 cdcd cdcd cdcd cdcd cdcd cdcd cdcd cdcd +* +$(printf %x $((OBJECT_SIZE / 2))) 0000 0000 0000 0000 0000 0000 0000 0000 +* +$(printf %x $IMAGE_SIZE) +EOF + [[ $(rados -p rbd ls | grep -c rbd_data.$IMAGE_ID) -eq $num_objects_expected ]] + for ((i = 0; i < $num_objects_expected; i++)); do + rados -p rbd stat rbd_data.$IMAGE_ID.$(printf %016x $i) | egrep "(size $((OBJECT_SIZE / 2)))|(size 0)" + done +} + +IMAGE_NAME="fallocate-test" + +rbd create --size 200 $IMAGE_NAME + +IMAGE_SIZE=$(rbd info --format=json $IMAGE_NAME | python3 -c 'import sys, json; print(json.load(sys.stdin)["size"])') +OBJECT_SIZE=$(rbd info --format=json $IMAGE_NAME | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_size"])') +NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE)) +[[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]] + +IMAGE_ID="$(rbd info --format=json $IMAGE_NAME | + python3 -c "import sys, json; print(json.load(sys.stdin)['block_name_prefix'].split('.')[1])")" + +DEV=$(sudo rbd map $IMAGE_NAME) + +# make sure -ENOENT is hidden +assert_zeroes 0 +py_blkdiscard 0 +assert_zeroes 0 + +# blkdev_issue_discard +allocate +py_blkdiscard 0 +assert_zeroes 0 + +# blkdev_issue_zeroout w/ BLKDEV_ZERO_NOUNMAP +allocate +py_fallocate FALLOC_FL_ZERO_RANGE\|FALLOC_FL_KEEP_SIZE 0 +assert_zeroes 0 + +# blkdev_issue_zeroout w/ BLKDEV_ZERO_NOFALLBACK +allocate +py_fallocate FALLOC_FL_PUNCH_HOLE\|FALLOC_FL_KEEP_SIZE 0 +assert_zeroes 0 + +# unaligned blkdev_issue_discard +allocate +py_blkdiscard $((OBJECT_SIZE / 2)) +assert_zeroes_unaligned $NUM_OBJECTS + +# unaligned blkdev_issue_zeroout w/ BLKDEV_ZERO_NOUNMAP +allocate +py_fallocate FALLOC_FL_ZERO_RANGE\|FALLOC_FL_KEEP_SIZE $((OBJECT_SIZE / 2)) +assert_zeroes_unaligned $NUM_OBJECTS + +# unaligned blkdev_issue_zeroout w/ BLKDEV_ZERO_NOFALLBACK +allocate +py_fallocate FALLOC_FL_PUNCH_HOLE\|FALLOC_FL_KEEP_SIZE $((OBJECT_SIZE / 2)) +assert_zeroes_unaligned $NUM_OBJECTS + +sudo rbd unmap $DEV + +DEV=$(sudo rbd map -o notrim $IMAGE_NAME) + +# blkdev_issue_discard +allocate +py_blkdiscard 0 |& grep 'Operation not supported' +assert_allocated + +# blkdev_issue_zeroout w/ BLKDEV_ZERO_NOUNMAP +allocate +py_fallocate FALLOC_FL_ZERO_RANGE\|FALLOC_FL_KEEP_SIZE 0 +assert_zeroes $NUM_OBJECTS + +# blkdev_issue_zeroout w/ BLKDEV_ZERO_NOFALLBACK +allocate +py_fallocate FALLOC_FL_PUNCH_HOLE\|FALLOC_FL_KEEP_SIZE 0 |& grep 'Operation not supported' +assert_allocated + +sudo rbd unmap $DEV + +echo OK diff --git a/qa/workunits/rbd/krbd_huge_osdmap.sh b/qa/workunits/rbd/krbd_huge_osdmap.sh new file mode 100755 index 000000000..0a550d674 --- /dev/null +++ b/qa/workunits/rbd/krbd_huge_osdmap.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# This is a test for https://tracker.ceph.com/issues/40481. +# +# An osdmap with 60000 slots encodes to ~16M, of which the ignored portion +# is ~13M. However in-memory osdmap is larger than ~3M: in-memory osd_addr +# array for 60000 OSDs is ~8M because of sockaddr_storage. +# +# Set mon_max_osd = 60000 in ceph.conf. + +set -ex + +function expect_false() { + if "$@"; then return 1; else return 0; fi +} + +function run_test() { + local dev + + # initially tiny, grow via incrementals + dev=$(sudo rbd map img) + for max in 8 60 600 6000 60000; do + ceph osd setmaxosd $max + expect_false sudo rbd map wait_for/latest_osdmap + xfs_io -c 'pwrite -w 0 12M' $DEV + done + ceph osd getcrushmap -o /dev/stdout | ceph osd setcrushmap -i /dev/stdin + expect_false sudo rbd map wait_for/latest_osdmap + xfs_io -c 'pwrite -w 0 12M' $DEV + sudo rbd unmap $dev + + # initially huge, shrink via incrementals + dev=$(sudo rbd map img) + for max in 60000 6000 600 60 8; do + ceph osd setmaxosd $max + expect_false sudo rbd map wait_for/latest_osdmap + xfs_io -c 'pwrite -w 0 12M' $DEV + done + ceph osd getcrushmap -o /dev/stdout | ceph osd setcrushmap -i /dev/stdin + expect_false sudo rbd map wait_for/latest_osdmap + xfs_io -c 'pwrite -w 0 12M' $DEV + sudo rbd unmap $dev +} + +rbd create --size 12M img +run_test +# repeat with primary affinity (adds an extra array) +ceph osd primary-affinity osd.0 0.5 +run_test + +echo OK diff --git a/qa/workunits/rbd/krbd_latest_osdmap_on_map.sh b/qa/workunits/rbd/krbd_latest_osdmap_on_map.sh new file mode 100755 index 000000000..f70f38639 --- /dev/null +++ b/qa/workunits/rbd/krbd_latest_osdmap_on_map.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -ex + +function run_test() { + ceph osd pool create foo 12 + rbd pool init foo + rbd create --size 1 foo/img + + local dev + dev=$(sudo rbd map foo/img) + sudo rbd unmap $dev + + ceph osd pool delete foo foo --yes-i-really-really-mean-it +} + +NUM_ITER=20 + +for ((i = 0; i < $NUM_ITER; i++)); do + run_test +done + +rbd create --size 1 img +DEV=$(sudo rbd map img) +for ((i = 0; i < $NUM_ITER; i++)); do + run_test +done +sudo rbd unmap $DEV + +echo OK diff --git a/qa/workunits/rbd/krbd_namespaces.sh b/qa/workunits/rbd/krbd_namespaces.sh new file mode 100755 index 000000000..0273d8499 --- /dev/null +++ b/qa/workunits/rbd/krbd_namespaces.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +set -ex + +function get_block_name_prefix() { + rbd info --format=json $1 | python3 -c "import sys, json; print(json.load(sys.stdin)['block_name_prefix'])" +} + +function do_pwrite() { + local spec=$1 + local old_byte=$2 + local new_byte=$3 + + local dev + dev=$(sudo rbd map $spec) + cmp <(dd if=/dev/zero bs=1M count=10 | tr \\000 \\$old_byte) $dev + xfs_io -c "pwrite -b 1M -S $new_byte 0 10M" $dev + sudo rbd unmap $dev +} + +function do_cmp() { + local spec=$1 + local byte=$2 + + local dev + dev=$(sudo rbd map $spec) + cmp <(dd if=/dev/zero bs=1M count=10 | tr \\000 \\$byte) $dev + sudo rbd unmap $dev +} + +function gen_child_specs() { + local i=$1 + + local child_specs="foo/img$i-clone1 foo/img$i-clone2 foo/ns1/img$i-clone1 foo/ns1/img$i-clone2" + if [[ $i -ge 3 ]]; then + child_specs="$child_specs foo/ns2/img$i-clone1 foo/ns2/img$i-clone2" + fi + echo $child_specs +} + +ceph osd pool create foo 12 +rbd pool init foo +ceph osd pool create bar 12 +rbd pool init bar + +ceph osd set-require-min-compat-client nautilus +rbd namespace create foo/ns1 +rbd namespace create foo/ns2 + +SPECS=(foo/img1 foo/img2 foo/ns1/img3 foo/ns1/img4) + +COUNT=1 +for spec in "${SPECS[@]}"; do + if [[ $spec =~ img1|img3 ]]; then + rbd create --size 10 $spec + else + rbd create --size 10 --data-pool bar $spec + fi + do_pwrite $spec 000 $(printf %03d $COUNT) + rbd snap create $spec@snap + COUNT=$((COUNT + 1)) +done +for i in {1..4}; do + for child_spec in $(gen_child_specs $i); do + if [[ $child_spec =~ clone1 ]]; then + rbd clone ${SPECS[i - 1]}@snap $child_spec + else + rbd clone --data-pool bar ${SPECS[i - 1]}@snap $child_spec + fi + do_pwrite $child_spec $(printf %03d $i) $(printf %03d $COUNT) + COUNT=$((COUNT + 1)) + done +done + +[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img1)) -eq 3 ]] +[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img2)) -eq 3 ]] +[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img3)) -eq 3 ]] +[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img4)) -eq 3 ]] + +[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img1-clone1)) -eq 3 ]] +[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img1-clone2)) -eq 3 ]] +[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img1-clone1)) -eq 3 ]] +[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img1-clone2)) -eq 3 ]] + +[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img2-clone1)) -eq 3 ]] +[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img2-clone2)) -eq 3 ]] +[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img2-clone1)) -eq 3 ]] +[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img2-clone2)) -eq 3 ]] + +[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img3-clone1)) -eq 3 ]] +[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img3-clone2)) -eq 3 ]] +[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img3-clone1)) -eq 3 ]] +[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img3-clone2)) -eq 3 ]] +[[ $(rados -p foo -N ns2 ls | grep -c $(get_block_name_prefix foo/ns2/img3-clone1)) -eq 3 ]] +[[ $(rados -p bar -N ns2 ls | grep -c $(get_block_name_prefix foo/ns2/img3-clone2)) -eq 3 ]] + +[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img4-clone1)) -eq 3 ]] +[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img4-clone2)) -eq 3 ]] +[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img4-clone1)) -eq 3 ]] +[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img4-clone2)) -eq 3 ]] +[[ $(rados -p foo -N ns2 ls | grep -c $(get_block_name_prefix foo/ns2/img4-clone1)) -eq 3 ]] +[[ $(rados -p bar -N ns2 ls | grep -c $(get_block_name_prefix foo/ns2/img4-clone2)) -eq 3 ]] + +COUNT=1 +for spec in "${SPECS[@]}"; do + do_cmp $spec $(printf %03d $COUNT) + COUNT=$((COUNT + 1)) +done +for i in {1..4}; do + for child_spec in $(gen_child_specs $i); do + do_cmp $child_spec $(printf %03d $COUNT) + COUNT=$((COUNT + 1)) + done +done + +echo OK diff --git a/qa/workunits/rbd/krbd_rxbounce.sh b/qa/workunits/rbd/krbd_rxbounce.sh new file mode 100755 index 000000000..ad00e3f96 --- /dev/null +++ b/qa/workunits/rbd/krbd_rxbounce.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +set -ex + +rbd create --size 256 img + +IMAGE_SIZE=$(rbd info --format=json img | python3 -c 'import sys, json; print(json.load(sys.stdin)["size"])') +OBJECT_SIZE=$(rbd info --format=json img | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_size"])') +NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE)) +[[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]] +OP_SIZE=16384 + +DEV=$(sudo rbd map img) +{ + for ((i = 0; i < $NUM_OBJECTS; i++)); do + echo pwrite -b $OP_SIZE -S $i $((i * OBJECT_SIZE)) $OP_SIZE + done + echo fsync + echo quit +} | xfs_io $DEV +sudo rbd unmap $DEV + +g++ -xc++ -o racereads - -lpthread <<EOF +#include <assert.h> +#include <fcntl.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <thread> +#include <vector> + +const int object_size = $OBJECT_SIZE; +const int num_objects = $NUM_OBJECTS; +const int read_len = $OP_SIZE; +const int num_reads = 1024; + +int main() { + int fd = open("$DEV", O_DIRECT | O_RDONLY); + assert(fd >= 0); + + void *buf; + int r = posix_memalign(&buf, 512, read_len); + assert(r == 0); + + std::vector<std::thread> threads; + for (int i = 0; i < num_objects; i++) { + threads.emplace_back( + [fd, buf, read_off = static_cast<off_t>(i) * object_size]() { + for (int i = 0; i < num_reads; i++) { + auto len = pread(fd, buf, read_len, read_off); + assert(len == read_len); + } + }); + } + + for (auto &t : threads) { + t.join(); + } +} +EOF + +DEV=$(sudo rbd map -o ms_mode=legacy img) +sudo dmesg -C +./racereads +[[ $(dmesg | grep -c 'libceph: osd.* bad crc/signature') -gt 100 ]] +sudo rbd unmap $DEV + +DEV=$(sudo rbd map -o ms_mode=legacy,rxbounce img) +sudo dmesg -C +./racereads +[[ $(dmesg | grep -c 'libceph: osd.* bad crc/signature') -eq 0 ]] +sudo rbd unmap $DEV + +DEV=$(sudo rbd map -o ms_mode=crc img) +sudo dmesg -C +./racereads +[[ $(dmesg | grep -c 'libceph: osd.* integrity error') -gt 100 ]] +sudo rbd unmap $DEV + +DEV=$(sudo rbd map -o ms_mode=crc,rxbounce img) +sudo dmesg -C +./racereads +[[ $(dmesg | grep -c 'libceph: osd.* integrity error') -eq 0 ]] +sudo rbd unmap $DEV + +# rxbounce is a no-op for secure mode +DEV=$(sudo rbd map -o ms_mode=secure img) +sudo dmesg -C +./racereads +[[ $(dmesg | grep -c 'libceph: osd.* integrity error') -eq 0 ]] +sudo rbd unmap $DEV + +DEV=$(sudo rbd map -o ms_mode=secure,rxbounce img) +sudo dmesg -C +./racereads +[[ $(dmesg | grep -c 'libceph: osd.* integrity error') -eq 0 ]] +sudo rbd unmap $DEV + +rbd rm img + +echo OK diff --git a/qa/workunits/rbd/krbd_stable_writes.sh b/qa/workunits/rbd/krbd_stable_writes.sh new file mode 100755 index 000000000..d00e5fd04 --- /dev/null +++ b/qa/workunits/rbd/krbd_stable_writes.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +set -ex + +function assert_dm() { + local name=$1 + local val=$2 + + local devno + devno=$(sudo dmsetup info -c --noheadings -o Major,Minor $name) + grep -q $val /sys/dev/block/$devno/queue/stable_writes +} + +function dmsetup_reload() { + local name=$1 + + local table + table=$(</dev/stdin) + + sudo dmsetup suspend $name + echo "$table" | sudo dmsetup reload $name + sudo dmsetup resume $name +} + +IMAGE_NAME="stable-writes-test" + +rbd create --size 1 $IMAGE_NAME +DEV=$(sudo rbd map $IMAGE_NAME) + +fallocate -l 1M loopfile +LOOP_DEV=$(sudo losetup -f --show loopfile) + +[[ $(blockdev --getsize64 $DEV) -eq 1048576 ]] +grep -q 1 /sys/block/${DEV#/dev/}/queue/stable_writes + +rbd resize --size 2 $IMAGE_NAME +[[ $(blockdev --getsize64 $DEV) -eq 2097152 ]] +grep -q 1 /sys/block/${DEV#/dev/}/queue/stable_writes + +cat <<EOF | sudo dmsetup create tbl +0 1024 linear $LOOP_DEV 0 +EOF +assert_dm tbl 0 +sudo dmsetup remove tbl + +cat <<EOF | sudo dmsetup create tbl +0 1024 linear $DEV 0 +EOF +assert_dm tbl 1 +sudo dmsetup remove tbl + +cat <<EOF | sudo dmsetup create tbl +0 1024 linear $LOOP_DEV 0 +1024 2048 error +EOF +assert_dm tbl 0 +sudo dmsetup remove tbl + +cat <<EOF | sudo dmsetup create tbl +0 1024 linear $DEV 0 +1024 2048 error +EOF +assert_dm tbl 1 +sudo dmsetup remove tbl + +cat <<EOF | sudo dmsetup create tbl +0 1024 linear $LOOP_DEV 0 +1024 2048 linear $DEV 0 +EOF +assert_dm tbl 1 +sudo dmsetup remove tbl + +cat <<EOF | sudo dmsetup create tbl +0 1024 linear $DEV 0 +1024 2048 linear $LOOP_DEV 0 +EOF +assert_dm tbl 1 +sudo dmsetup remove tbl + +cat <<EOF | sudo dmsetup create tbl +0 1024 linear $LOOP_DEV 0 +EOF +assert_dm tbl 0 +cat <<EOF | dmsetup_reload tbl +0 1024 linear $LOOP_DEV 0 +1024 2048 linear $DEV 0 +EOF +assert_dm tbl 1 +cat <<EOF | dmsetup_reload tbl +0 1024 linear $LOOP_DEV 0 +EOF +assert_dm tbl 0 +sudo dmsetup remove tbl + +cat <<EOF | sudo dmsetup create tbl +0 1024 linear $DEV 0 +EOF +assert_dm tbl 1 +cat <<EOF | dmsetup_reload tbl +0 1024 linear $DEV 0 +1024 2048 linear $LOOP_DEV 0 +EOF +assert_dm tbl 1 +cat <<EOF | dmsetup_reload tbl +0 1024 linear $DEV 0 +EOF +assert_dm tbl 1 +sudo dmsetup remove tbl + +cat <<EOF | sudo dmsetup create tbl +0 1024 linear $DEV 0 +EOF +assert_dm tbl 1 +cat <<EOF | dmsetup_reload tbl +0 1024 linear $DEV 0 +1024 2048 linear $LOOP_DEV 0 +EOF +assert_dm tbl 1 +cat <<EOF | dmsetup_reload tbl +0 1024 error +1024 2048 linear $LOOP_DEV 0 +EOF +assert_dm tbl 0 +cat <<EOF | dmsetup_reload tbl +0 1024 linear $DEV 0 +1024 2048 linear $LOOP_DEV 0 +EOF +assert_dm tbl 1 +cat <<EOF | dmsetup_reload tbl +0 1024 linear $DEV 0 +EOF +assert_dm tbl 1 +sudo dmsetup remove tbl + +sudo losetup -d $LOOP_DEV +rm loopfile + +sudo rbd unmap $DEV +rbd rm $IMAGE_NAME + +echo OK diff --git a/qa/workunits/rbd/krbd_udev_enumerate.sh b/qa/workunits/rbd/krbd_udev_enumerate.sh new file mode 100755 index 000000000..494f958f8 --- /dev/null +++ b/qa/workunits/rbd/krbd_udev_enumerate.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# This is a test for https://tracker.ceph.com/issues/41036, but it also +# triggers https://tracker.ceph.com/issues/41404 in some environments. + +set -ex + +function assert_exit_codes() { + declare -a pids=($@) + + for pid in ${pids[@]}; do + wait $pid + done +} + +function run_map() { + declare -a pids + + for i in {1..300}; do + sudo rbd map img$i & + pids+=($!) + done + + assert_exit_codes ${pids[@]} + [[ $(rbd showmapped | wc -l) -eq 301 ]] +} + +function run_unmap_by_dev() { + declare -a pids + + run_map + for i in {0..299}; do + sudo rbd unmap /dev/rbd$i & + pids+=($!) + done + + assert_exit_codes ${pids[@]} + [[ $(rbd showmapped | wc -l) -eq 0 ]] +} + +function run_unmap_by_spec() { + declare -a pids + + run_map + for i in {1..300}; do + sudo rbd unmap img$i & + pids+=($!) + done + + assert_exit_codes ${pids[@]} + [[ $(rbd showmapped | wc -l) -eq 0 ]] +} + +# Can't test with exclusive-lock, don't bother enabling deep-flatten. +# See https://tracker.ceph.com/issues/42492. +for i in {1..300}; do + rbd create --size 1 --image-feature '' img$i +done + +for i in {1..30}; do + echo Iteration $i + run_unmap_by_dev + run_unmap_by_spec +done + +echo OK diff --git a/qa/workunits/rbd/krbd_udev_netlink_enobufs.sh b/qa/workunits/rbd/krbd_udev_netlink_enobufs.sh new file mode 100755 index 000000000..7c9c53a2f --- /dev/null +++ b/qa/workunits/rbd/krbd_udev_netlink_enobufs.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# This is a test for https://tracker.ceph.com/issues/41404, verifying that udev +# events are properly reaped while the image is being (un)mapped in the kernel. +# UDEV_BUF_SIZE is 1M (giving us a 2M socket receive buffer), but modprobe + +# modprobe -r generate ~28M worth of "block" events. + +set -ex + +rbd create --size 1 img + +ceph osd pause +sudo rbd map img & +PID=$! +sudo modprobe scsi_debug max_luns=16 add_host=16 num_parts=1 num_tgts=16 +sudo udevadm settle +sudo modprobe -r scsi_debug +[[ $(rbd showmapped | wc -l) -eq 0 ]] +ceph osd unpause +wait $PID +[[ $(rbd showmapped | wc -l) -eq 2 ]] +sudo rbd unmap img + +echo OK diff --git a/qa/workunits/rbd/krbd_udev_netns.sh b/qa/workunits/rbd/krbd_udev_netns.sh new file mode 100755 index 000000000..e746a682e --- /dev/null +++ b/qa/workunits/rbd/krbd_udev_netns.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +set -ex + +sudo ip netns add ns1 +sudo ip link add veth1-ext type veth peer name veth1-int +sudo ip link set veth1-int netns ns1 + +sudo ip netns exec ns1 ip link set dev lo up +sudo ip netns exec ns1 ip addr add 192.168.1.2/24 dev veth1-int +sudo ip netns exec ns1 ip link set veth1-int up +sudo ip netns exec ns1 ip route add default via 192.168.1.1 + +sudo ip addr add 192.168.1.1/24 dev veth1-ext +sudo ip link set veth1-ext up + +# Enable forwarding between the namespace and the default route +# interface and set up NAT. In case of multiple default routes, +# just pick the first one. +if [[ $(sysctl -n net.ipv4.ip_forward) -eq 0 ]]; then + sudo iptables -P FORWARD DROP + sudo sysctl -w net.ipv4.ip_forward=1 +fi +IFACE="$(ip route list 0.0.0.0/0 | head -n 1 | cut -d ' ' -f 5)" +sudo iptables -A FORWARD -i veth1-ext -o "$IFACE" -j ACCEPT +sudo iptables -A FORWARD -i "$IFACE" -o veth1-ext -j ACCEPT +sudo iptables -t nat -A POSTROUTING -s 192.168.1.2 -o "$IFACE" -j MASQUERADE + +rbd create --size 300 img + +DEV="$(sudo rbd map img)" +mkfs.ext4 "$DEV" +sudo mount "$DEV" /mnt +sudo umount /mnt +sudo rbd unmap "$DEV" + +sudo ip netns exec ns1 bash <<'EOF' + +set -ex + +DEV="/dev/rbd/rbd/img" +[[ ! -e "$DEV" ]] + +# In a network namespace, "rbd map" maps the device and hangs waiting +# for udev add uevents. udev runs as usual (in particular creating the +# symlink which is used here because the device node is never printed), +# but the uevents it sends out never come because they don't cross +# network namespace boundaries. +set +e +timeout 30s rbd map img +RET=$? +set -e +[[ $RET -eq 124 ]] +[[ -L "$DEV" ]] +mkfs.ext4 -F "$DEV" +mount "$DEV" /mnt +umount /mnt + +# In a network namespace, "rbd unmap" unmaps the device and hangs +# waiting for udev remove uevents. udev runs as usual (removing the +# symlink), but the uevents it sends out never come because they don't +# cross network namespace boundaries. +set +e +timeout 30s rbd unmap "$DEV" +RET=$? +set -e +[[ $RET -eq 124 ]] +[[ ! -e "$DEV" ]] + +# Skip waiting for udev uevents with "-o noudev". +DEV="$(rbd map -o noudev img)" +mkfs.ext4 -F "$DEV" +mount "$DEV" /mnt +umount /mnt +rbd unmap -o noudev "$DEV" + +EOF + +rbd rm img + +sudo iptables -t nat -D POSTROUTING -s 192.168.1.2 -o "$IFACE" -j MASQUERADE +sudo iptables -D FORWARD -i "$IFACE" -o veth1-ext -j ACCEPT +sudo iptables -D FORWARD -i veth1-ext -o "$IFACE" -j ACCEPT +sudo ip netns delete ns1 + +echo OK diff --git a/qa/workunits/rbd/krbd_udev_symlinks.sh b/qa/workunits/rbd/krbd_udev_symlinks.sh new file mode 100755 index 000000000..271476527 --- /dev/null +++ b/qa/workunits/rbd/krbd_udev_symlinks.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +set -ex + +SPECS=( +rbd/img1 +rbd/img2 +rbd/img2@snap1 +rbd/img3 +rbd/img3@snap1 +rbd/img3@snap2 +rbd/ns1/img1 +rbd/ns1/img2 +rbd/ns1/img2@snap1 +rbd/ns1/img3 +rbd/ns1/img3@snap1 +rbd/ns1/img3@snap2 +rbd/ns2/img1 +rbd/ns2/img2 +rbd/ns2/img2@snap1 +rbd/ns2/img3 +rbd/ns2/img3@snap1 +rbd/ns2/img3@snap2 +custom/img1 +custom/img1@snap1 +custom/img2 +custom/img2@snap1 +custom/img2@snap2 +custom/img3 +custom/ns1/img1 +custom/ns1/img1@snap1 +custom/ns1/img2 +custom/ns1/img2@snap1 +custom/ns1/img2@snap2 +custom/ns1/img3 +custom/ns2/img1 +custom/ns2/img1@snap1 +custom/ns2/img2 +custom/ns2/img2@snap1 +custom/ns2/img2@snap2 +custom/ns2/img3 +) + +ceph osd pool create custom 8 +rbd pool init custom + +ceph osd set-require-min-compat-client nautilus +rbd namespace create rbd/ns1 +rbd namespace create rbd/ns2 +rbd namespace create custom/ns1 +rbd namespace create custom/ns2 + +# create in order, images before snapshots +for spec in "${SPECS[@]}"; do + if [[ "$spec" =~ snap ]]; then + rbd snap create "$spec" + else + rbd create --size 10 "$spec" + DEV="$(sudo rbd map "$spec")" + sudo sfdisk "$DEV" <<EOF +unit: sectors +${DEV}p1 : start= 2048, size= 2, type=83 +${DEV}p2 : start= 4096, size= 2, type=83 +EOF + sudo rbd unmap "$DEV" + fi +done + +[[ ! -e /dev/rbd ]] + +# map in random order +COUNT=${#SPECS[@]} +read -r -a INDEXES < <(python3 <<EOF +import random +l = list(range($COUNT)) +random.shuffle(l) +print(*l) +EOF +) + +DEVS=() +for idx in "${INDEXES[@]}"; do + DEVS+=("$(sudo rbd map "${SPECS[idx]}")") +done + +[[ $(rbd showmapped | wc -l) -eq $((COUNT + 1)) ]] + +for ((i = 0; i < COUNT; i++)); do + [[ "$(readlink -e "/dev/rbd/${SPECS[INDEXES[i]]}")" == "${DEVS[i]}" ]] + [[ "$(readlink -e "/dev/rbd/${SPECS[INDEXES[i]]}-part1")" == "${DEVS[i]}p1" ]] + [[ "$(readlink -e "/dev/rbd/${SPECS[INDEXES[i]]}-part2")" == "${DEVS[i]}p2" ]] +done + +for idx in "${INDEXES[@]}"; do + sudo rbd unmap "/dev/rbd/${SPECS[idx]}" +done + +[[ ! -e /dev/rbd ]] + +# remove in reverse order, snapshots before images +for ((i = COUNT - 1; i >= 0; i--)); do + if [[ "${SPECS[i]}" =~ snap ]]; then + rbd snap rm "${SPECS[i]}" + else + rbd rm "${SPECS[i]}" + fi +done + +rbd namespace rm custom/ns2 +rbd namespace rm custom/ns1 +rbd namespace rm rbd/ns2 +rbd namespace rm rbd/ns1 + +ceph osd pool delete custom custom --yes-i-really-really-mean-it + +echo OK diff --git a/qa/workunits/rbd/krbd_wac.sh b/qa/workunits/rbd/krbd_wac.sh new file mode 100755 index 000000000..134460409 --- /dev/null +++ b/qa/workunits/rbd/krbd_wac.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -ex + +wget http://download.ceph.com/qa/wac.c +gcc -o wac wac.c + +rbd create --size 300 img +DEV=$(sudo rbd map img) + +sudo mkfs.ext4 $DEV +sudo mount $DEV /mnt +set +e +sudo timeout 5m ./wac -l 65536 -n 64 -r /mnt/wac-test +RET=$? +set -e +[[ $RET -eq 124 ]] +sudo killall -w wac || true # wac forks +sudo umount /mnt + +sudo wipefs -a $DEV +sudo vgcreate vg_img $DEV +sudo lvcreate -L 256M -n lv_img vg_img +udevadm settle +sudo mkfs.ext4 /dev/mapper/vg_img-lv_img +sudo mount /dev/mapper/vg_img-lv_img /mnt +set +e +sudo timeout 5m ./wac -l 65536 -n 64 -r /mnt/wac-test +RET=$? +set -e +[[ $RET -eq 124 ]] +sudo killall -w wac || true # wac forks +sudo umount /mnt +sudo vgremove -f vg_img +sudo pvremove $DEV + +sudo rbd unmap $DEV +rbd rm img + +echo OK diff --git a/qa/workunits/rbd/krbd_watch_errors.sh b/qa/workunits/rbd/krbd_watch_errors.sh new file mode 100755 index 000000000..f650d2a74 --- /dev/null +++ b/qa/workunits/rbd/krbd_watch_errors.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -ex +set -o pipefail + +function refresh_loop() { + local dev_id="$1" + + set +x + + local i + for ((i = 1; ; i++)); do + echo 1 | sudo tee "${SYSFS_DIR}/${dev_id}/refresh" > /dev/null + if ((i % 100 == 0)); then + echo "Refreshed ${i} times" + fi + done +} + +readonly SYSFS_DIR="/sys/bus/rbd/devices" +readonly IMAGE_NAME="watch-errors-test" + +rbd create -s 1G --image-feature exclusive-lock "${IMAGE_NAME}" + +# induce a watch error every 30 seconds +dev="$(sudo rbd device map -o osdkeepalive=60 "${IMAGE_NAME}")" +dev_id="${dev#/dev/rbd}" + +# constantly refresh, not just on watch errors +refresh_loop "${dev_id}" & +refresh_pid=$! + +sudo dmesg -C + +# test that none of the above triggers a deadlock with a workload +fio --name test --filename="${dev}" --ioengine=libaio --direct=1 \ + --rw=randwrite --norandommap --randrepeat=0 --bs=512 --iodepth=128 \ + --time_based --runtime=1h --eta=never + +num_errors="$(dmesg | grep -c "rbd${dev_id}: encountered watch error")" +echo "Recorded ${num_errors} watch errors" + +kill "${refresh_pid}" +wait + +sudo rbd device unmap "${dev}" + +if ((num_errors < 60)); then + echo "Too few watch errors" + exit 1 +fi + +echo OK diff --git a/qa/workunits/rbd/luks-encryption.sh b/qa/workunits/rbd/luks-encryption.sh new file mode 100755 index 000000000..5d3cc68cd --- /dev/null +++ b/qa/workunits/rbd/luks-encryption.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash +set -ex + +CEPH_ID=${CEPH_ID:-admin} +TMP_FILES="/tmp/passphrase /tmp/passphrase2 /tmp/testdata1 /tmp/testdata2 /tmp/cmpdata" + +_sudo() +{ + local cmd + + if [ `id -u` -eq 0 ] + then + "$@" + return $? + fi + + # Look for the command in the user path. If it fails run it as is, + # supposing it is in sudo path. + cmd=`which $1 2>/dev/null` || cmd=$1 + shift + sudo -nE "${cmd}" "$@" +} + +function drop_caches { + sudo sync + echo 3 | sudo tee /proc/sys/vm/drop_caches +} + +function expect_false() { + if "$@"; then return 1; else return 0; fi +} + +function test_encryption_format() { + local format=$1 + clean_up_cryptsetup + + # format + rbd encryption format testimg $format /tmp/passphrase + drop_caches + + # open encryption with cryptsetup + sudo cryptsetup open $RAW_DEV --type luks cryptsetupdev -d /tmp/passphrase + sudo chmod 666 /dev/mapper/cryptsetupdev + + # open encryption with librbd + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg -t nbd -o encryption-passphrase-file=/tmp/passphrase) + sudo chmod 666 $LIBRBD_DEV + + # write via librbd && compare + dd if=/tmp/testdata1 of=$LIBRBD_DEV oflag=direct bs=1M + dd if=/dev/mapper/cryptsetupdev of=/tmp/cmpdata iflag=direct bs=4M count=4 + cmp -n 16MB /tmp/cmpdata /tmp/testdata1 + + # write via cryptsetup && compare + dd if=/tmp/testdata2 of=/dev/mapper/cryptsetupdev oflag=direct bs=1M + dd if=$LIBRBD_DEV of=/tmp/cmpdata iflag=direct bs=4M count=4 + cmp -n 16MB /tmp/cmpdata /tmp/testdata2 + + # FIXME: encryption-aware flatten/resize misbehave if proxied to + # RAW_DEV mapping (i.e. if RAW_DEV mapping ows the lock) + # (acquire and) release the lock as a side effect + rbd bench --io-type read --io-size 1 --io-threads 1 --io-total 1 testimg + + # check that encryption-aware resize compensates LUKS header overhead + (( $(sudo blockdev --getsize64 $LIBRBD_DEV) < (32 << 20) )) + expect_false rbd resize --size 32M testimg + rbd resize --size 32M --encryption-passphrase-file /tmp/passphrase testimg + (( $(sudo blockdev --getsize64 $LIBRBD_DEV) == (32 << 20) )) + + _sudo rbd device unmap -t nbd $LIBRBD_DEV +} + +function test_clone_encryption() { + clean_up_cryptsetup + + # write 1MB plaintext + dd if=/tmp/testdata1 of=$RAW_DEV oflag=direct bs=1M count=1 + + # clone (luks1) + rbd snap create testimg@snap + rbd snap protect testimg@snap + rbd clone testimg@snap testimg1 + rbd encryption format testimg1 luks1 /tmp/passphrase + + # open encryption with librbd, write one more MB, close + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg1 -t nbd -o encryption-format=luks1,encryption-passphrase-file=/tmp/passphrase) + sudo chmod 666 $LIBRBD_DEV + dd if=$LIBRBD_DEV of=/tmp/cmpdata iflag=direct bs=1M count=1 + cmp -n 1MB /tmp/cmpdata /tmp/testdata1 + dd if=/tmp/testdata1 of=$LIBRBD_DEV seek=1 skip=1 oflag=direct bs=1M count=1 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # second clone (luks2) + rbd snap create testimg1@snap + rbd snap protect testimg1@snap + rbd clone testimg1@snap testimg2 + rbd encryption format testimg2 luks2 /tmp/passphrase2 + + # open encryption with librbd, write one more MB, close + LIBRBD_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd -o encryption-format=luks2,encryption-passphrase-file=/tmp/passphrase2,encryption-format=luks1,encryption-passphrase-file=/tmp/passphrase) + sudo chmod 666 $LIBRBD_DEV + dd if=$LIBRBD_DEV of=/tmp/cmpdata iflag=direct bs=1M count=2 + cmp -n 2MB /tmp/cmpdata /tmp/testdata1 + dd if=/tmp/testdata1 of=$LIBRBD_DEV seek=2 skip=2 oflag=direct bs=1M count=1 + _sudo rbd device unmap -t nbd $LIBRBD_DEV + + # flatten + expect_false rbd flatten testimg2 --encryption-format luks1 --encryption-format luks2 --encryption-passphrase-file /tmp/passphrase2 --encryption-passphrase-file /tmp/passphrase + rbd flatten testimg2 --encryption-format luks2 --encryption-format luks1 --encryption-passphrase-file /tmp/passphrase2 --encryption-passphrase-file /tmp/passphrase + + # verify with cryptsetup + RAW_FLAT_DEV=$(_sudo rbd -p rbd map testimg2 -t nbd) + sudo cryptsetup open $RAW_FLAT_DEV --type luks cryptsetupdev -d /tmp/passphrase2 + sudo chmod 666 /dev/mapper/cryptsetupdev + dd if=/dev/mapper/cryptsetupdev of=/tmp/cmpdata iflag=direct bs=1M count=3 + cmp -n 3MB /tmp/cmpdata /tmp/testdata1 + _sudo rbd device unmap -t nbd $RAW_FLAT_DEV +} + +function test_clone_and_load_with_a_single_passphrase { + local expectedfail=$1 + + # clone and format + rbd snap create testimg@snap + rbd snap protect testimg@snap + rbd clone testimg@snap testimg1 + rbd encryption format testimg1 luks2 /tmp/passphrase2 + + if [ "$expectedfail" = "true" ] + then + expect_false rbd flatten testimg1 --encryption-passphrase-file /tmp/passphrase2 + rbd flatten testimg1 --encryption-passphrase-file /tmp/passphrase2 --encryption-passphrase-file /tmp/passphrase + else + rbd flatten testimg1 --encryption-passphrase-file /tmp/passphrase2 + fi + + rbd remove testimg1 + rbd snap unprotect testimg@snap + rbd snap remove testimg@snap +} + +function test_plaintext_detection { + # 16k LUKS header + sudo cryptsetup -q luksFormat --type luks2 --luks2-metadata-size 16k $RAW_DEV /tmp/passphrase + test_clone_and_load_with_a_single_passphrase true + + # 4m LUKS header + sudo cryptsetup -q luksFormat --type luks2 --luks2-metadata-size 4m $RAW_DEV /tmp/passphrase + test_clone_and_load_with_a_single_passphrase true + + # no luks header + dd if=/dev/zero of=$RAW_DEV oflag=direct bs=4M count=8 + test_clone_and_load_with_a_single_passphrase false +} + +function get_nbd_device_paths { + rbd device list -t nbd | tail -n +2 | egrep "\s+rbd\s+testimg" | awk '{print $5;}' +} + +function clean_up_cryptsetup() { + ls /dev/mapper/cryptsetupdev && sudo cryptsetup close cryptsetupdev || true +} + +function clean_up { + sudo rm -f $TMP_FILES + clean_up_cryptsetup + for device in $(get_nbd_device_paths); do + _sudo rbd device unmap -t nbd $device + done + + rbd remove testimg2 || true + rbd snap unprotect testimg1@snap || true + rbd snap remove testimg1@snap || true + rbd remove testimg1 || true + rbd snap unprotect testimg@snap || true + rbd snap remove testimg@snap || true + rbd remove testimg || true +} + +if [[ $(uname) != "Linux" ]]; then + echo "LUKS encryption tests only supported on Linux" + exit 0 +fi + + +if [[ $(($(ceph-conf --name client.${CEPH_ID} --show-config-value rbd_default_features) & 64)) != 0 ]]; then + echo "LUKS encryption tests not supported alongside image journaling feature" + exit 0 +fi + +clean_up + +trap clean_up INT TERM EXIT + +# generate test data +dd if=/dev/urandom of=/tmp/testdata1 bs=4M count=4 +dd if=/dev/urandom of=/tmp/testdata2 bs=4M count=4 + +# create passphrase files +printf "pass\0word\n" > /tmp/passphrase +printf "\t password2 " > /tmp/passphrase2 + +# create an image +rbd create testimg --size=32M + +# map raw data to nbd device +RAW_DEV=$(_sudo rbd -p rbd map testimg -t nbd) +sudo chmod 666 $RAW_DEV + +test_plaintext_detection + +test_encryption_format luks1 +test_encryption_format luks2 + +test_clone_encryption + +echo OK diff --git a/qa/workunits/rbd/map-snapshot-io.sh b/qa/workunits/rbd/map-snapshot-io.sh new file mode 100755 index 000000000..a69d84829 --- /dev/null +++ b/qa/workunits/rbd/map-snapshot-io.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# http://tracker.ceph.com/issues/3964 + +set -ex + +rbd create image -s 100 +DEV=$(sudo rbd map image) +dd if=/dev/zero of=$DEV oflag=direct count=10 +rbd snap create image@s1 +dd if=/dev/zero of=$DEV oflag=direct count=10 # used to fail +rbd snap rm image@s1 +dd if=/dev/zero of=$DEV oflag=direct count=10 +sudo rbd unmap $DEV +rbd rm image + +echo OK diff --git a/qa/workunits/rbd/map-unmap.sh b/qa/workunits/rbd/map-unmap.sh new file mode 100755 index 000000000..99863849e --- /dev/null +++ b/qa/workunits/rbd/map-unmap.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -ex + +RUN_TIME=300 # approximate duration of run (seconds) + +[ $# -eq 1 ] && RUN_TIME="$1" + +IMAGE_NAME="image-$$" +IMAGE_SIZE="1024" # MB + +function get_time() { + date '+%s' +} + +function times_up() { + local end_time="$1" + + test $(get_time) -ge "${end_time}" +} + +function map_unmap() { + [ $# -eq 1 ] || exit 99 + local image_name="$1" + + local dev + dev="$(sudo rbd map "${image_name}")" + sudo rbd unmap "${dev}" +} + +#### Start + +rbd create "${IMAGE_NAME}" --size="${IMAGE_SIZE}" + +COUNT=0 +START_TIME=$(get_time) +END_TIME=$(expr $(get_time) + ${RUN_TIME}) +while ! times_up "${END_TIME}"; do + map_unmap "${IMAGE_NAME}" + COUNT=$(expr $COUNT + 1) +done +ELAPSED=$(expr "$(get_time)" - "${START_TIME}") + +rbd rm "${IMAGE_NAME}" + +echo "${COUNT} iterations completed in ${ELAPSED} seconds" diff --git a/qa/workunits/rbd/merge_diff.sh b/qa/workunits/rbd/merge_diff.sh new file mode 100755 index 000000000..eb8597304 --- /dev/null +++ b/qa/workunits/rbd/merge_diff.sh @@ -0,0 +1,477 @@ +#!/usr/bin/env bash +set -ex + +export RBD_FORCE_ALLOW_V1=1 + +pool=rbd +gen=$pool/gen +out=$pool/out +testno=1 + +mkdir -p merge_diff_test +pushd merge_diff_test + +function expect_false() +{ + if "$@"; then return 1; else return 0; fi +} + +function clear_all() +{ + fusermount -u mnt || true + + rbd snap purge --no-progress $gen || true + rbd rm --no-progress $gen || true + rbd snap purge --no-progress $out || true + rbd rm --no-progress $out || true + + rm -rf diffs || true +} + +function rebuild() +{ + clear_all + echo Starting test $testno + ((testno++)) + if [[ "$2" -lt "$1" ]] && [[ "$3" -gt "1" ]]; then + rbd create $gen --size 100 --object-size $1 --stripe-unit $2 --stripe-count $3 --image-format $4 + else + rbd create $gen --size 100 --object-size $1 --image-format $4 + fi + rbd create $out --size 1 --object-size 524288 + mkdir -p mnt diffs + # lttng has atexit handlers that need to be fork/clone aware + LD_PRELOAD=liblttng-ust-fork.so.0 rbd-fuse -p $pool mnt +} + +function write() +{ + dd if=/dev/urandom of=mnt/gen bs=1M conv=notrunc seek=$1 count=$2 +} + +function snap() +{ + rbd snap create $gen@$1 +} + +function resize() +{ + rbd resize --no-progress $gen --size $1 --allow-shrink +} + +function export_diff() +{ + if [ $2 == "head" ]; then + target="$gen" + else + target="$gen@$2" + fi + if [ $1 == "null" ]; then + rbd export-diff --no-progress $target diffs/$1.$2 + else + rbd export-diff --no-progress $target --from-snap $1 diffs/$1.$2 + fi +} + +function merge_diff() +{ + rbd merge-diff diffs/$1.$2 diffs/$2.$3 diffs/$1.$3 +} + +function check() +{ + rbd import-diff --no-progress diffs/$1.$2 $out || return -1 + if [ "$2" == "head" ]; then + sum1=`rbd export $gen - | md5sum` + else + sum1=`rbd export $gen@$2 - | md5sum` + fi + sum2=`rbd export $out - | md5sum` + if [ "$sum1" != "$sum2" ]; then + exit -1 + fi + if [ "$2" != "head" ]; then + rbd snap ls $out | awk '{print $2}' | grep "^$2\$" || return -1 + fi +} + +#test f/t header +rebuild 4194304 4194304 1 2 +write 0 1 +snap a +write 1 1 +export_diff null a +export_diff a head +merge_diff null a head +check null head + +rebuild 4194304 4194304 1 2 +write 0 1 +snap a +write 1 1 +snap b +write 2 1 +export_diff null a +export_diff a b +export_diff b head +merge_diff null a b +check null b + +rebuild 4194304 4194304 1 2 +write 0 1 +snap a +write 1 1 +snap b +write 2 1 +export_diff null a +export_diff a b +export_diff b head +merge_diff a b head +check null a +check a head + +rebuild 4194304 4194304 1 2 +write 0 1 +snap a +write 1 1 +snap b +write 2 1 +export_diff null a +export_diff a b +export_diff b head +rbd merge-diff diffs/null.a diffs/a.b - | rbd merge-diff - diffs/b.head - > diffs/null.head +check null head + +#data test +rebuild 4194304 4194304 1 2 +write 4 2 +snap s101 +write 0 3 +write 8 2 +snap s102 +export_diff null s101 +export_diff s101 s102 +merge_diff null s101 s102 +check null s102 + +rebuild 4194304 4194304 1 2 +write 0 3 +write 2 5 +write 8 2 +snap s201 +write 0 2 +write 6 3 +snap s202 +export_diff null s201 +export_diff s201 s202 +merge_diff null s201 s202 +check null s202 + +rebuild 4194304 4194304 1 2 +write 0 4 +write 12 6 +snap s301 +write 0 6 +write 10 5 +write 16 4 +snap s302 +export_diff null s301 +export_diff s301 s302 +merge_diff null s301 s302 +check null s302 + +rebuild 4194304 4194304 1 2 +write 0 12 +write 14 2 +write 18 2 +snap s401 +write 1 2 +write 5 6 +write 13 3 +write 18 2 +snap s402 +export_diff null s401 +export_diff s401 s402 +merge_diff null s401 s402 +check null s402 + +rebuild 4194304 4194304 1 2 +write 2 4 +write 10 12 +write 27 6 +write 36 4 +snap s501 +write 0 24 +write 28 4 +write 36 4 +snap s502 +export_diff null s501 +export_diff s501 s502 +merge_diff null s501 s502 +check null s502 + +rebuild 4194304 4194304 1 2 +write 0 8 +resize 5 +snap r1 +resize 20 +write 12 8 +snap r2 +resize 8 +write 4 4 +snap r3 +export_diff null r1 +export_diff r1 r2 +export_diff r2 r3 +merge_diff null r1 r2 +merge_diff null r2 r3 +check null r3 + +rebuild 4194304 4194304 1 2 +write 0 8 +resize 5 +snap r1 +resize 20 +write 12 8 +snap r2 +resize 8 +write 4 4 +snap r3 +resize 10 +snap r4 +export_diff null r1 +export_diff r1 r2 +export_diff r2 r3 +export_diff r3 r4 +merge_diff null r1 r2 +merge_diff null r2 r3 +merge_diff null r3 r4 +check null r4 + +# merge diff doesn't yet support fancy striping +# rebuild 4194304 65536 8 2 +# write 0 32 +# snap r1 +# write 16 32 +# snap r2 +# export_diff null r1 +# export_diff r1 r2 +# expect_false merge_diff null r1 r2 + +rebuild 4194304 4194304 1 2 +write 0 1 +write 2 1 +write 4 1 +write 6 1 +snap s1 +write 1 1 +write 3 1 +write 5 1 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 1 1 +write 3 1 +write 5 1 +snap s1 +write 0 1 +write 2 1 +write 4 1 +write 6 1 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 0 3 +write 6 3 +write 12 3 +snap s1 +write 1 1 +write 7 1 +write 13 1 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 0 3 +write 6 3 +write 12 3 +snap s1 +write 0 1 +write 6 1 +write 12 1 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 0 3 +write 6 3 +write 12 3 +snap s1 +write 2 1 +write 8 1 +write 14 1 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 1 1 +write 7 1 +write 13 1 +snap s1 +write 0 3 +write 6 3 +write 12 3 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 0 1 +write 6 1 +write 12 1 +snap s1 +write 0 3 +write 6 3 +write 12 3 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 2 1 +write 8 1 +write 14 1 +snap s1 +write 0 3 +write 6 3 +write 12 3 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 0 3 +write 6 3 +write 12 3 +snap s1 +write 0 3 +write 6 3 +write 12 3 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 2 4 +write 8 4 +write 14 4 +snap s1 +write 0 3 +write 6 3 +write 12 3 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 0 4 +write 6 4 +write 12 4 +snap s1 +write 0 3 +write 6 3 +write 12 3 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 0 6 +write 6 6 +write 12 6 +snap s1 +write 0 3 +write 6 3 +write 12 3 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 3 6 +write 9 6 +write 15 6 +snap s1 +write 0 3 +write 6 3 +write 12 3 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 0 8 +snap s1 +resize 2 +resize 100 +snap s2 +export_diff null s1 +export_diff s1 s2 +merge_diff null s1 s2 +check null s2 + +rebuild 4194304 4194304 1 2 +write 0 8 +snap s1 +resize 2 +resize 100 +snap s2 +write 20 2 +snap s3 +export_diff null s1 +export_diff s1 s2 +export_diff s2 s3 +merge_diff s1 s2 s3 +check null s1 +check s1 s3 + +#addme + +clear_all +popd +rm -rf merge_diff_test + +echo OK diff --git a/qa/workunits/rbd/notify_master.sh b/qa/workunits/rbd/notify_master.sh new file mode 100755 index 000000000..99ccd74db --- /dev/null +++ b/qa/workunits/rbd/notify_master.sh @@ -0,0 +1,5 @@ +#!/bin/sh -ex + +relpath=$(dirname $0)/../../../src/test/librbd +python3 $relpath/test_notify.py master +exit 0 diff --git a/qa/workunits/rbd/notify_slave.sh b/qa/workunits/rbd/notify_slave.sh new file mode 100755 index 000000000..7f49a0c7d --- /dev/null +++ b/qa/workunits/rbd/notify_slave.sh @@ -0,0 +1,5 @@ +#!/bin/sh -ex + +relpath=$(dirname $0)/../../../src/test/librbd +python3 $relpath/test_notify.py slave +exit 0 diff --git a/qa/workunits/rbd/permissions.sh b/qa/workunits/rbd/permissions.sh new file mode 100755 index 000000000..f8a9aaa71 --- /dev/null +++ b/qa/workunits/rbd/permissions.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +set -ex + +IMAGE_FEATURES="layering,exclusive-lock,object-map,fast-diff" + +clone_v2_enabled() { + image_spec=$1 + rbd info $image_spec | grep "clone-parent" +} + +create_pools() { + ceph osd pool create images 32 + rbd pool init images + ceph osd pool create volumes 32 + rbd pool init volumes +} + +delete_pools() { + (ceph osd pool delete images images --yes-i-really-really-mean-it || true) >/dev/null 2>&1 + (ceph osd pool delete volumes volumes --yes-i-really-really-mean-it || true) >/dev/null 2>&1 + +} + +recreate_pools() { + delete_pools + create_pools +} + +delete_users() { + (ceph auth del client.volumes || true) >/dev/null 2>&1 + (ceph auth del client.images || true) >/dev/null 2>&1 + + (ceph auth del client.snap_none || true) >/dev/null 2>&1 + (ceph auth del client.snap_all || true) >/dev/null 2>&1 + (ceph auth del client.snap_pool || true) >/dev/null 2>&1 + (ceph auth del client.snap_profile_all || true) >/dev/null 2>&1 + (ceph auth del client.snap_profile_pool || true) >/dev/null 2>&1 + + (ceph auth del client.mon_write || true) >/dev/null 2>&1 +} + +create_users() { + ceph auth get-or-create client.volumes \ + mon 'profile rbd' \ + osd 'profile rbd pool=volumes, profile rbd-read-only pool=images' \ + mgr 'profile rbd pool=volumes, profile rbd-read-only pool=images' >> $KEYRING + ceph auth get-or-create client.images mon 'profile rbd' osd 'profile rbd pool=images' >> $KEYRING + + ceph auth get-or-create client.snap_none mon 'allow r' >> $KEYRING + ceph auth get-or-create client.snap_all mon 'allow r' osd 'allow w' >> $KEYRING + ceph auth get-or-create client.snap_pool mon 'allow r' osd 'allow w pool=images' >> $KEYRING + ceph auth get-or-create client.snap_profile_all mon 'allow r' osd 'profile rbd' >> $KEYRING + ceph auth get-or-create client.snap_profile_pool mon 'allow r' osd 'profile rbd pool=images' >> $KEYRING + + ceph auth get-or-create client.mon_write mon 'allow *' >> $KEYRING +} + +expect() { + + set +e + + local expected_ret=$1 + local ret + + shift + cmd=$@ + + eval $cmd + ret=$? + + set -e + + if [[ $ret -ne $expected_ret ]]; then + echo "ERROR: running \'$cmd\': expected $expected_ret got $ret" + return 1 + fi + + return 0 +} + +test_images_access() { + rbd -k $KEYRING --id images create --image-format 2 --image-feature $IMAGE_FEATURES -s 1 images/foo + rbd -k $KEYRING --id images snap create images/foo@snap + rbd -k $KEYRING --id images snap protect images/foo@snap + rbd -k $KEYRING --id images snap unprotect images/foo@snap + rbd -k $KEYRING --id images snap protect images/foo@snap + rbd -k $KEYRING --id images export images/foo@snap - >/dev/null + expect 16 rbd -k $KEYRING --id images snap rm images/foo@snap + + rbd -k $KEYRING --id volumes clone --image-feature $IMAGE_FEATURES images/foo@snap volumes/child + + if ! clone_v2_enabled images/foo; then + expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap + fi + + expect 1 rbd -k $KEYRING --id volumes snap unprotect images/foo@snap + expect 1 rbd -k $KEYRING --id images flatten volumes/child + rbd -k $KEYRING --id volumes flatten volumes/child + expect 1 rbd -k $KEYRING --id volumes snap unprotect images/foo@snap + rbd -k $KEYRING --id images snap unprotect images/foo@snap + + expect 39 rbd -k $KEYRING --id images rm images/foo + rbd -k $KEYRING --id images snap rm images/foo@snap + rbd -k $KEYRING --id images rm images/foo + rbd -k $KEYRING --id volumes rm volumes/child +} + +test_volumes_access() { + rbd -k $KEYRING --id images create --image-format 2 --image-feature $IMAGE_FEATURES -s 1 images/foo + rbd -k $KEYRING --id images snap create images/foo@snap + rbd -k $KEYRING --id images snap protect images/foo@snap + + # commands that work with read-only access + rbd -k $KEYRING --id volumes info images/foo@snap + rbd -k $KEYRING --id volumes snap ls images/foo + rbd -k $KEYRING --id volumes export images/foo - >/dev/null + rbd -k $KEYRING --id volumes cp images/foo volumes/foo_copy + rbd -k $KEYRING --id volumes rm volumes/foo_copy + rbd -k $KEYRING --id volumes children images/foo@snap + rbd -k $KEYRING --id volumes lock list images/foo + + # commands that fail with read-only access + expect 1 rbd -k $KEYRING --id volumes resize -s 2 images/foo --allow-shrink + expect 1 rbd -k $KEYRING --id volumes snap create images/foo@2 + expect 1 rbd -k $KEYRING --id volumes snap rollback images/foo@snap + expect 1 rbd -k $KEYRING --id volumes snap remove images/foo@snap + expect 1 rbd -k $KEYRING --id volumes snap purge images/foo + expect 1 rbd -k $KEYRING --id volumes snap unprotect images/foo@snap + expect 1 rbd -k $KEYRING --id volumes flatten images/foo + expect 1 rbd -k $KEYRING --id volumes lock add images/foo test + expect 1 rbd -k $KEYRING --id volumes lock remove images/foo test locker + expect 1 rbd -k $KEYRING --id volumes ls rbd + + # create clone and snapshot + rbd -k $KEYRING --id volumes clone --image-feature $IMAGE_FEATURES images/foo@snap volumes/child + rbd -k $KEYRING --id volumes snap create volumes/child@snap1 + rbd -k $KEYRING --id volumes snap protect volumes/child@snap1 + rbd -k $KEYRING --id volumes snap create volumes/child@snap2 + + # make sure original snapshot stays protected + if clone_v2_enabled images/foo; then + rbd -k $KEYRING --id volumes flatten volumes/child + rbd -k $KEYRING --id volumes snap rm volumes/child@snap2 + rbd -k $KEYRING --id volumes snap unprotect volumes/child@snap1 + else + expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap + rbd -k $KEYRING --id volumes flatten volumes/child + expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap + rbd -k $KEYRING --id volumes snap rm volumes/child@snap2 + expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap + expect 2 rbd -k $KEYRING --id volumes snap rm volumes/child@snap2 + rbd -k $KEYRING --id volumes snap unprotect volumes/child@snap1 + expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap + fi + + # clean up + rbd -k $KEYRING --id volumes snap rm volumes/child@snap1 + rbd -k $KEYRING --id images snap unprotect images/foo@snap + rbd -k $KEYRING --id images snap rm images/foo@snap + rbd -k $KEYRING --id images rm images/foo + rbd -k $KEYRING --id volumes rm volumes/child +} + +create_self_managed_snapshot() { + ID=$1 + POOL=$2 + + cat << EOF | CEPH_ARGS="-k $KEYRING" python3 +import rados + +with rados.Rados(conffile="", rados_id="${ID}") as cluster: + ioctx = cluster.open_ioctx("${POOL}") + + snap_id = ioctx.create_self_managed_snap() + print ("Created snap id {}".format(snap_id)) +EOF +} + +remove_self_managed_snapshot() { + ID=$1 + POOL=$2 + + cat << EOF | CEPH_ARGS="-k $KEYRING" python3 +import rados + +with rados.Rados(conffile="", rados_id="mon_write") as cluster1, \ + rados.Rados(conffile="", rados_id="${ID}") as cluster2: + ioctx1 = cluster1.open_ioctx("${POOL}") + + snap_id = ioctx1.create_self_managed_snap() + print ("Created snap id {}".format(snap_id)) + + ioctx2 = cluster2.open_ioctx("${POOL}") + + ioctx2.remove_self_managed_snap(snap_id) + print ("Removed snap id {}".format(snap_id)) +EOF +} + +test_remove_self_managed_snapshots() { + # Ensure users cannot create self-managed snapshots w/o permissions + expect 1 create_self_managed_snapshot snap_none images + expect 1 create_self_managed_snapshot snap_none volumes + + create_self_managed_snapshot snap_all images + create_self_managed_snapshot snap_all volumes + + create_self_managed_snapshot snap_pool images + expect 1 create_self_managed_snapshot snap_pool volumes + + create_self_managed_snapshot snap_profile_all images + create_self_managed_snapshot snap_profile_all volumes + + create_self_managed_snapshot snap_profile_pool images + expect 1 create_self_managed_snapshot snap_profile_pool volumes + + # Ensure users cannot delete self-managed snapshots w/o permissions + expect 1 remove_self_managed_snapshot snap_none images + expect 1 remove_self_managed_snapshot snap_none volumes + + remove_self_managed_snapshot snap_all images + remove_self_managed_snapshot snap_all volumes + + remove_self_managed_snapshot snap_pool images + expect 1 remove_self_managed_snapshot snap_pool volumes + + remove_self_managed_snapshot snap_profile_all images + remove_self_managed_snapshot snap_profile_all volumes + + remove_self_managed_snapshot snap_profile_pool images + expect 1 remove_self_managed_snapshot snap_profile_pool volumes +} + +test_rbd_support() { + # read-only commands should work on both pools + ceph -k $KEYRING --id volumes rbd perf image stats volumes + ceph -k $KEYRING --id volumes rbd perf image stats images + + # read/write commands should only work on 'volumes' + rbd -k $KEYRING --id volumes create --image-format 2 --image-feature $IMAGE_FEATURES -s 1 volumes/foo + ceph -k $KEYRING --id volumes rbd task add remove volumes/foo + expect 13 ceph -k $KEYRING --id volumes rbd task add remove images/foo +} + +cleanup() { + rm -f $KEYRING +} + +KEYRING=$(mktemp) +trap cleanup EXIT ERR HUP INT QUIT + +delete_users +create_users + +recreate_pools +test_images_access + +recreate_pools +test_volumes_access + +test_remove_self_managed_snapshots + +test_rbd_support + +delete_pools +delete_users + +echo OK +exit 0 diff --git a/qa/workunits/rbd/qemu-iotests.sh b/qa/workunits/rbd/qemu-iotests.sh new file mode 100755 index 000000000..a2e9e0600 --- /dev/null +++ b/qa/workunits/rbd/qemu-iotests.sh @@ -0,0 +1,47 @@ +#!/bin/sh -ex + +# Run qemu-iotests against rbd. These are block-level tests that go +# through qemu but do not involve running a full vm. Note that these +# require the admin ceph user, as there's no way to pass the ceph user +# to qemu-iotests currently. + +testlist='001 002 003 004 005 008 009 010 011 021 025 032 033' + +git clone https://github.com/qemu/qemu.git +cd qemu + + +if grep -iqE '(bionic|focal|jammy|platform:el9)' /etc/os-release; then + git checkout v2.11.0 +elif grep -iqE '(xenial|platform:el8)' /etc/os-release; then + git checkout v2.3.0 +else + # use v2.2.0-rc3 (last released version that handles all the tests + git checkout 2528043f1f299e0e88cb026f1ca7c40bbb4e1f80 +fi + +cd tests/qemu-iotests +# qemu-iotests expects a binary called just 'qemu' to be available +if [ -x '/usr/bin/qemu-system-x86_64' ] +then + QEMU='/usr/bin/qemu-system-x86_64' +else + QEMU='/usr/libexec/qemu-kvm' +fi + +# Bionic (v2.11.0) tests expect all tools in current directory +ln -s $QEMU qemu +ln -s /usr/bin/qemu-img +ln -s /usr/bin/qemu-io +ln -s /usr/bin/qemu-nbd + +# this is normally generated by configure, but has nothing but a python +# binary definition, which we don't care about. for some reason it is +# not present on trusty. +touch common.env + +# TEST_DIR is the pool for rbd +TEST_DIR=rbd ./check -rbd $testlist + +cd ../../.. +rm -rf qemu diff --git a/qa/workunits/rbd/qemu_dynamic_features.sh b/qa/workunits/rbd/qemu_dynamic_features.sh new file mode 100755 index 000000000..70e9fbb3c --- /dev/null +++ b/qa/workunits/rbd/qemu_dynamic_features.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -x + +if [[ -z "${IMAGE_NAME}" ]]; then + echo image name must be provided + exit 1 +fi + +is_qemu_running() { + rbd status ${IMAGE_NAME} | grep -v "Watchers: none" +} + +wait_for_qemu() { + while ! is_qemu_running ; do + echo "*** Waiting for QEMU" + sleep 30 + done +} + +wait_for_qemu +rbd feature disable ${IMAGE_NAME} journaling +rbd feature disable ${IMAGE_NAME} object-map +rbd feature disable ${IMAGE_NAME} exclusive-lock + +while is_qemu_running ; do + echo "*** Enabling all features" + rbd feature enable ${IMAGE_NAME} exclusive-lock || break + rbd feature enable ${IMAGE_NAME} journaling || break + rbd feature enable ${IMAGE_NAME} object-map || break + if is_qemu_running ; then + sleep 60 + fi + + echo "*** Disabling all features" + rbd feature disable ${IMAGE_NAME} journaling || break + rbd feature disable ${IMAGE_NAME} object-map || break + rbd feature disable ${IMAGE_NAME} exclusive-lock || break + if is_qemu_running ; then + sleep 60 + fi +done + +if is_qemu_running ; then + echo "RBD command failed on alive QEMU" + exit 1 +fi diff --git a/qa/workunits/rbd/qemu_rebuild_object_map.sh b/qa/workunits/rbd/qemu_rebuild_object_map.sh new file mode 100755 index 000000000..2647dcdcd --- /dev/null +++ b/qa/workunits/rbd/qemu_rebuild_object_map.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -ex + +if [[ -z "${IMAGE_NAME}" ]]; then + echo image name must be provided + exit 1 +fi + +is_qemu_running() { + rbd status ${IMAGE_NAME} | grep -v "Watchers: none" +} + +wait_for_qemu() { + while ! is_qemu_running ; do + echo "*** Waiting for QEMU" + sleep 30 + done +} + +wait_for_qemu +rbd feature disable ${IMAGE_NAME} journaling || true +rbd feature disable ${IMAGE_NAME} fast-diff || true +rbd feature disable ${IMAGE_NAME} object-map || true +rbd feature disable ${IMAGE_NAME} exclusive-lock || true + +rbd feature enable ${IMAGE_NAME} exclusive-lock +rbd feature enable ${IMAGE_NAME} object-map + +while is_qemu_running ; do + echo "*** Rebuilding object map" + rbd object-map rebuild ${IMAGE_NAME} + + if is_qemu_running ; then + sleep 60 + fi +done + diff --git a/qa/workunits/rbd/qos.sh b/qa/workunits/rbd/qos.sh new file mode 100755 index 000000000..feb1d5144 --- /dev/null +++ b/qa/workunits/rbd/qos.sh @@ -0,0 +1,90 @@ +#!/bin/sh -ex + +POOL=rbd +IMAGE=test$$ +IMAGE_SIZE=1G +TOLERANCE_PRCNT=10 + +rbd_bench() { + local image=$1 + local type=$2 + local total=$3 + local qos_type=$4 + local qos_limit=$5 + local iops_var_name=$6 + local bps_var_name=$7 + local timeout=$8 + local timeout_cmd="" + + if [ -n "${timeout}" ]; then + timeout_cmd="timeout --preserve-status ${timeout}" + fi + + # parse `rbd bench` output for string like this: + # elapsed: 25 ops: 2560 ops/sec: 100.08 bytes/sec: 409.13 MiB + iops_bps=$(${timeout_cmd} rbd bench "${image}" \ + --io-type ${type} --io-size 4K \ + --io-total ${total} --rbd-cache=false \ + --rbd_qos_${qos_type}_limit ${qos_limit} | + awk '/elapsed:.* GiB/ {print int($6) ":" int($8) * 1024 * 1024 * 1024} + /elapsed:.* MiB/ {print int($6) ":" int($8) * 1024 * 1024} + /elapsed:.* KiB/ {print int($6) ":" int($8) * 1024} + /elapsed:.* B/ {print int($6) ":" int($8)}') + eval ${iops_var_name}=${iops_bps%:*} + eval ${bps_var_name}=${iops_bps#*:} +} + +rbd create "${POOL}/${IMAGE}" -s ${IMAGE_SIZE} +rbd bench "${POOL}/${IMAGE}" --io-type write --io-size 4M --io-total ${IMAGE_SIZE} + +rbd_bench "${POOL}/${IMAGE}" write ${IMAGE_SIZE} iops 0 iops bps 60 +iops_unlimited=$iops +bps_unlimited=$bps + +test "${iops_unlimited}" -ge 20 || exit 0 + +io_total=$((bps_unlimited * 30)) + +rbd_bench "${POOL}/${IMAGE}" write ${io_total} iops $((iops_unlimited / 2)) iops bps +test "${iops}" -le $((iops_unlimited / 2 * (100 + TOLERANCE_PRCNT) / 100)) + +rbd_bench "${POOL}/${IMAGE}" write ${io_total} write_iops $((iops_unlimited / 2)) iops bps +test "${iops}" -le $((iops_unlimited / 2 * (100 + TOLERANCE_PRCNT) / 100)) + +rbd_bench "${POOL}/${IMAGE}" write ${io_total} bps $((bps_unlimited / 2)) iops bps +test "${bps}" -le $((bps_unlimited / 2 * (100 + TOLERANCE_PRCNT) / 100)) + +rbd_bench "${POOL}/${IMAGE}" write ${io_total} write_bps $((bps_unlimited / 2)) iops bps +test "${bps}" -le $((bps_unlimited / 2 * (100 + TOLERANCE_PRCNT) / 100)) + +rbd_bench "${POOL}/${IMAGE}" read ${io_total} iops 0 iops bps +iops_unlimited=$iops +bps_unlimited=$bps + +test "${iops_unlimited}" -ge 20 || exit 0 + +io_total=$((bps_unlimited * 30)) + +rbd_bench "${POOL}/${IMAGE}" read ${io_total} iops $((iops_unlimited / 2)) iops bps +test "${iops}" -le $((iops_unlimited / 2 * (100 + TOLERANCE_PRCNT) / 100)) + +rbd_bench "${POOL}/${IMAGE}" read ${io_total} read_iops $((iops_unlimited / 2)) iops bps +test "${iops}" -le $((iops_unlimited / 2 * (100 + TOLERANCE_PRCNT) / 100)) + +rbd_bench "${POOL}/${IMAGE}" read ${io_total} bps $((bps_unlimited / 2)) iops bps +test "${bps}" -le $((bps_unlimited / 2 * (100 + TOLERANCE_PRCNT) / 100)) + +rbd_bench "${POOL}/${IMAGE}" read ${io_total} read_bps $((bps_unlimited / 2)) iops bps +test "${bps}" -le $((bps_unlimited / 2 * (100 + TOLERANCE_PRCNT) / 100)) + +# test a config override is applied +rbd config image set "${POOL}/${IMAGE}" rbd_qos_iops_limit $((iops_unlimited / 4)) +rbd_bench "${POOL}/${IMAGE}" read ${io_total} iops $((iops_unlimited / 2)) iops bps +test "${iops}" -le $((iops_unlimited / 4 * (100 + TOLERANCE_PRCNT) / 100)) +rbd config image remove "${POOL}/${IMAGE}" rbd_qos_iops_limit +rbd_bench "${POOL}/${IMAGE}" read ${io_total} iops $((iops_unlimited / 2)) iops bps +test "${iops}" -le $((iops_unlimited / 2 * (100 + TOLERANCE_PRCNT) / 100)) + +rbd rm "${POOL}/${IMAGE}" + +echo OK diff --git a/qa/workunits/rbd/rbd-ggate.sh b/qa/workunits/rbd/rbd-ggate.sh new file mode 100755 index 000000000..1bf89da38 --- /dev/null +++ b/qa/workunits/rbd/rbd-ggate.sh @@ -0,0 +1,239 @@ +#!/bin/sh -ex + +POOL=testrbdggate$$ +NS=ns +IMAGE=test +SIZE=64 +DATA= +DEV= + +if which xmlstarlet > /dev/null 2>&1; then + XMLSTARLET=xmlstarlet +elif which xml > /dev/null 2>&1; then + XMLSTARLET=xml +else + echo "Missing xmlstarlet binary!" + exit 1 +fi + +if [ `uname -K` -ge 1200078 ] ; then + RBD_GGATE_RESIZE_SUPPORTED=1 +fi + +_sudo() +{ + local cmd + + if [ `id -u` -eq 0 ] + then + "$@" + return $? + fi + + # Look for the command in the user path. If it fails run it as is, + # supposing it is in sudo path. + cmd=`which $1 2>/dev/null` || cmd=$1 + shift + sudo -nE "${cmd}" "$@" +} + +check_geom_gate() +{ + # See if geom_date is load, or can be loaded. + # Otherwise the tests can not run + if ! kldstat -q -n geom_gate ; then + # See if we can load it + if ! _sudo kldload geom_gate ; then + echo Not able to load geom_gate + echo check /var/log/messages as to why + exit 1 + fi + fi +} + +setup() +{ + local ns x + + if [ -e CMakeCache.txt ]; then + # running under cmake build dir + + CEPH_SRC=$(readlink -f $(dirname $0)/../../../src) + CEPH_ROOT=${PWD} + CEPH_BIN=${CEPH_ROOT}/bin + + export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH} + export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind:${CEPH_ROOT}/lib/cython_modules/lib.3 + PATH=${CEPH_BIN}:${PATH} + fi + + _sudo echo test sudo + check_geom_gate + + trap cleanup INT TERM EXIT + TEMPDIR=`mktemp -d` + DATA=${TEMPDIR}/data + dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE} + ceph osd pool create ${POOL} 32 + + rbd namespace create ${POOL}/${NS} + for ns in '' ${NS}; do + rbd --dest-pool ${POOL} --dest-namespace "${ns}" --no-progress import \ + ${DATA} ${IMAGE} + done +} + +cleanup() +{ + local ns s + + set +e + rm -Rf ${TEMPDIR} + if [ -n "${DEV}" ] + then + _sudo rbd-ggate unmap ${DEV} + fi + + ceph osd pool delete ${POOL} ${POOL} --yes-i-really-really-mean-it +} + +expect_false() +{ + if "$@"; then return 1; else return 0; fi +} + +# +# main +# + +setup + +echo exit status test +expect_false rbd-ggate +expect_false rbd-ggate INVALIDCMD +if [ `id -u` -ne 0 ] +then + expect_false rbd-ggate map ${IMAGE} +fi +expect_false _sudo rbd-ggate map INVALIDIMAGE + +echo map test using the first unused device +DEV=`_sudo rbd-ggate map ${POOL}/${IMAGE}` +rbd-ggate list | grep " ${DEV} *$" + +echo map test specifying the device +expect_false _sudo rbd-ggate --device ${DEV} map ${POOL}/${IMAGE} +dev1=${DEV} +_sudo rbd-ggate unmap ${DEV} +rbd-ggate list | expect_false grep " ${DEV} *$" +DEV= +# XXX: race possible when the device is reused by other process +DEV=`_sudo rbd-ggate --device ${dev1} map ${POOL}/${IMAGE}` +[ "${DEV}" = "${dev1}" ] +rbd-ggate list | grep " ${DEV} *$" + +echo list format test +expect_false _sudo rbd-ggate --format INVALID list +rbd-ggate --format json --pretty-format list +rbd-ggate --format xml list + +echo read test +[ "`dd if=${DATA} bs=1M | md5`" = "`_sudo dd if=${DEV} bs=1M | md5`" ] + +echo write test +dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE} +_sudo dd if=${DATA} of=${DEV} bs=1M +_sudo sync +[ "`dd if=${DATA} bs=1M | md5`" = "`rbd -p ${POOL} --no-progress export ${IMAGE} - | md5`" ] + +echo trim test +provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .` +used=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .` +[ "${used}" -eq "${provisioned}" ] +_sudo newfs -E ${DEV} +_sudo sync +provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .` +used=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .` +[ "${used}" -lt "${provisioned}" ] + +echo resize test +devname=$(basename ${DEV}) +size=$(geom gate list ${devname} | awk '$1 ~ /Mediasize:/ {print $2}') +test -n "${size}" +rbd resize ${POOL}/${IMAGE} --size $((SIZE * 2))M +rbd info ${POOL}/${IMAGE} +if [ -z "$RBD_GGATE_RESIZE_SUPPORTED" ]; then + # when resizing is not supported: + # resizing the underlying image for a GEOM ggate will stop the + # ggate process servicing the device. So we can resize and test + # the disappearance of the device + rbd-ggate list | expect_false grep " ${DEV} *$" +else + rbd-ggate list | grep " ${DEV} *$" + size2=$(geom gate list ${devname} | awk '$1 ~ /Mediasize:/ {print $2}') + test -n "${size2}" + test ${size2} -eq $((size * 2)) + dd if=/dev/urandom of=${DATA} bs=1M count=$((SIZE * 2)) + _sudo dd if=${DATA} of=${DEV} bs=1M + _sudo sync + [ "`dd if=${DATA} bs=1M | md5`" = "`rbd -p ${POOL} --no-progress export ${IMAGE} - | md5`" ] + rbd resize ${POOL}/${IMAGE} --allow-shrink --size ${SIZE}M + rbd info ${POOL}/${IMAGE} + size2=$(geom gate list ${devname} | awk '$1 ~ /Mediasize:/ {print $2}') + test -n "${size2}" + test ${size2} -eq ${size} + truncate -s ${SIZE}M ${DATA} + [ "`dd if=${DATA} bs=1M | md5`" = "`rbd -p ${POOL} --no-progress export ${IMAGE} - | md5`" ] + _sudo rbd-ggate unmap ${DEV} +fi +DEV= + +echo read-only option test +DEV=`_sudo rbd-ggate map --read-only ${POOL}/${IMAGE}` +devname=$(basename ${DEV}) +rbd-ggate list | grep " ${DEV} *$" +access=$(geom gate list ${devname} | awk '$1 == "access:" {print $2}') +test "${access}" = "read-only" +_sudo dd if=${DEV} of=/dev/null bs=1M +expect_false _sudo dd if=${DATA} of=${DEV} bs=1M +_sudo rbd-ggate unmap ${DEV} + +echo exclusive option test +DEV=`_sudo rbd-ggate map --exclusive ${POOL}/${IMAGE}` +rbd-ggate list | grep " ${DEV} *$" +_sudo dd if=${DATA} of=${DEV} bs=1M +_sudo sync +expect_false timeout 10 \ + rbd -p ${POOL} bench ${IMAGE} --io-type=write --io-size=1024 --io-total=1024 +_sudo rbd-ggate unmap ${DEV} +DEV= +rbd bench -p ${POOL} ${IMAGE} --io-type=write --io-size=1024 --io-total=1024 + +echo unmap by image name test +DEV=`_sudo rbd-ggate map ${POOL}/${IMAGE}` +rbd-ggate list | grep " ${DEV} *$" +_sudo rbd-ggate unmap "${POOL}/${IMAGE}" +rbd-ggate list | expect_false grep " ${DEV} *$" +DEV= + +echo map/unmap snap test +rbd snap create ${POOL}/${IMAGE}@snap +DEV=`_sudo rbd-ggate map ${POOL}/${IMAGE}@snap` +rbd-ggate list | grep " ${DEV} *$" +_sudo rbd-ggate unmap "${POOL}/${IMAGE}@snap" +rbd-ggate list | expect_false grep " ${DEV} *$" +DEV= + +echo map/unmap namespace test +rbd snap create ${POOL}/${NS}/${IMAGE}@snap +DEV=`_sudo rbd-ggate map ${POOL}/${NS}/${IMAGE}@snap` +rbd-ggate list | grep " ${DEV} *$" +_sudo rbd-ggate unmap "${POOL}/${NS}/${IMAGE}@snap" +rbd-ggate list | expect_false grep "${DEV} $" +DEV= + +echo OK diff --git a/qa/workunits/rbd/rbd-nbd.sh b/qa/workunits/rbd/rbd-nbd.sh new file mode 100755 index 000000000..bc89e9be5 --- /dev/null +++ b/qa/workunits/rbd/rbd-nbd.sh @@ -0,0 +1,500 @@ +#!/usr/bin/env bash +set -ex + +. $(dirname $0)/../../standalone/ceph-helpers.sh + +POOL=rbd +ANOTHER_POOL=new_default_pool$$ +NS=ns +IMAGE=testrbdnbd$$ +SIZE=64 +DATA= +DEV= + +_sudo() +{ + local cmd + + if [ `id -u` -eq 0 ] + then + "$@" + return $? + fi + + # Look for the command in the user path. If it fails run it as is, + # supposing it is in sudo path. + cmd=`which $1 2>/dev/null` || cmd=$1 + shift + sudo -nE "${cmd}" "$@" +} + +setup() +{ + local ns x + + if [ -e CMakeCache.txt ]; then + # running under cmake build dir + + CEPH_SRC=$(readlink -f $(dirname $0)/../../../src) + CEPH_ROOT=${PWD} + CEPH_BIN=${CEPH_ROOT}/bin + + export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH} + export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind:${CEPH_ROOT}/lib/cython_modules/lib.3 + PATH=${CEPH_BIN}:${PATH} + fi + + _sudo echo test sudo + + trap cleanup INT TERM EXIT + TEMPDIR=`mktemp -d` + DATA=${TEMPDIR}/data + dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE} + + rbd namespace create ${POOL}/${NS} + + for ns in '' ${NS}; do + rbd --dest-pool ${POOL} --dest-namespace "${ns}" --no-progress import \ + ${DATA} ${IMAGE} + done + + # create another pool + ceph osd pool create ${ANOTHER_POOL} 8 + rbd pool init ${ANOTHER_POOL} +} + +function cleanup() +{ + local ns s + + set +e + + mount | fgrep ${TEMPDIR}/mnt && _sudo umount -f ${TEMPDIR}/mnt + + rm -Rf ${TEMPDIR} + if [ -n "${DEV}" ] + then + _sudo rbd device --device-type nbd unmap ${DEV} + fi + + for ns in '' ${NS}; do + if rbd -p ${POOL} --namespace "${ns}" status ${IMAGE} 2>/dev/null; then + for s in 0.5 1 2 4 8 16 32; do + sleep $s + rbd -p ${POOL} --namespace "${ns}" status ${IMAGE} | + grep 'Watchers: none' && break + done + rbd -p ${POOL} --namespace "${ns}" snap purge ${IMAGE} + rbd -p ${POOL} --namespace "${ns}" remove ${IMAGE} + fi + done + rbd namespace remove ${POOL}/${NS} + + # cleanup/reset default pool + rbd config global rm global rbd_default_pool + ceph osd pool delete ${ANOTHER_POOL} ${ANOTHER_POOL} --yes-i-really-really-mean-it +} + +function expect_false() +{ + if "$@"; then return 1; else return 0; fi +} + +function get_pid() +{ + local pool=$1 + local ns=$2 + + PID=$(rbd device --device-type nbd --format xml list | $XMLSTARLET sel -t -v \ + "//devices/device[pool='${pool}'][namespace='${ns}'][image='${IMAGE}'][device='${DEV}']/id") + test -n "${PID}" || return 1 + ps -p ${PID} -C rbd-nbd +} + +unmap_device() +{ + local args=$1 + local pid=$2 + + _sudo rbd device --device-type nbd unmap ${args} + rbd device --device-type nbd list | expect_false grep "^${pid}\\b" || return 1 + ps -C rbd-nbd | expect_false grep "^ *${pid}\\b" || return 1 + + # workaround possible race between unmap and following map + sleep 0.5 +} + +# +# main +# + +setup + +# exit status test +expect_false rbd-nbd +expect_false rbd-nbd INVALIDCMD +if [ `id -u` -ne 0 ] +then + expect_false rbd device --device-type nbd map ${IMAGE} +fi +expect_false _sudo rbd device --device-type nbd map INVALIDIMAGE +expect_false _sudo rbd-nbd --device INVALIDDEV map ${IMAGE} + +# list format test +expect_false rbd device --device-type nbd --format INVALID list +rbd device --device-type nbd --format json --pretty-format list +rbd device --device-type nbd --format xml list + +# map test using the first unused device +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} +# map test specifying the device +expect_false _sudo rbd-nbd --device ${DEV} map ${POOL}/${IMAGE} +dev1=${DEV} +unmap_device ${DEV} ${PID} +DEV= +# XXX: race possible when the device is reused by other process +DEV=`_sudo rbd-nbd --device ${dev1} map ${POOL}/${IMAGE}` +[ "${DEV}" = "${dev1}" ] +rbd device --device-type nbd list | grep "${IMAGE}" +get_pid ${POOL} + +# read test +[ "`dd if=${DATA} bs=1M | md5sum`" = "`_sudo dd if=${DEV} bs=1M | md5sum`" ] + +# write test +dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE} +_sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct +[ "`dd if=${DATA} bs=1M | md5sum`" = "`rbd -p ${POOL} --no-progress export ${IMAGE} - | md5sum`" ] +unmap_device ${DEV} ${PID} + +# notrim test +DEV=`_sudo rbd device --device-type nbd --options notrim map ${POOL}/${IMAGE}` +get_pid ${POOL} +provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .` +used=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .` +[ "${used}" -eq "${provisioned}" ] +# should fail discard as at time of mapping notrim was used +expect_false _sudo blkdiscard ${DEV} +sync +provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .` +used=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .` +[ "${used}" -eq "${provisioned}" ] +unmap_device ${DEV} ${PID} + +# trim test +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} +provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .` +used=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .` +[ "${used}" -eq "${provisioned}" ] +# should honor discard as at time of mapping trim was considered by default +_sudo blkdiscard ${DEV} +sync +provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .` +used=`rbd -p ${POOL} --format xml du ${IMAGE} | + $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .` +[ "${used}" -lt "${provisioned}" ] + +# resize test +devname=$(basename ${DEV}) +blocks=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions) +test -n "${blocks}" +rbd resize ${POOL}/${IMAGE} --size $((SIZE * 2))M +rbd info ${POOL}/${IMAGE} +blocks2=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions) +test -n "${blocks2}" +test ${blocks2} -eq $((blocks * 2)) +rbd resize ${POOL}/${IMAGE} --allow-shrink --size ${SIZE}M +blocks2=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions) +test -n "${blocks2}" +test ${blocks2} -eq ${blocks} + +# read-only option test +unmap_device ${DEV} ${PID} +DEV=`_sudo rbd --device-type nbd map --read-only ${POOL}/${IMAGE}` +PID=$(rbd device --device-type nbd list | awk -v pool=${POOL} -v img=${IMAGE} -v dev=${DEV} \ + '$2 == pool && $3 == img && $5 == dev {print $1}') +test -n "${PID}" +ps -p ${PID} -C rbd-nbd + +_sudo dd if=${DEV} of=/dev/null bs=1M +expect_false _sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct +unmap_device ${DEV} ${PID} + +# exclusive option test +DEV=`_sudo rbd --device-type nbd map --exclusive ${POOL}/${IMAGE}` +get_pid ${POOL} + +_sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct +expect_false timeout 10 \ + rbd bench ${IMAGE} --io-type write --io-size=1024 --io-total=1024 +unmap_device ${DEV} ${PID} +DEV= +rbd bench ${IMAGE} --io-type write --io-size=1024 --io-total=1024 + +# unmap by image name test +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} +unmap_device ${IMAGE} ${PID} +DEV= + +# map/unmap snap test +rbd snap create ${POOL}/${IMAGE}@snap +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}@snap` +get_pid ${POOL} +unmap_device "${IMAGE}@snap" ${PID} +DEV= + +# map/unmap snap test with --snap-id +SNAPID=`rbd snap ls ${POOL}/${IMAGE} | awk '$2 == "snap" {print $1}'` +DEV=`_sudo rbd device --device-type nbd map --snap-id ${SNAPID} ${POOL}/${IMAGE}` +get_pid ${POOL} +unmap_device "--snap-id ${SNAPID} ${IMAGE}" ${PID} +DEV= + +# map/unmap namespace test +rbd snap create ${POOL}/${NS}/${IMAGE}@snap +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${NS}/${IMAGE}@snap` +get_pid ${POOL} ${NS} +unmap_device "${POOL}/${NS}/${IMAGE}@snap" ${PID} +DEV= + +# map/unmap namespace test with --snap-id +SNAPID=`rbd snap ls ${POOL}/${NS}/${IMAGE} | awk '$2 == "snap" {print $1}'` +DEV=`_sudo rbd device --device-type nbd map --snap-id ${SNAPID} ${POOL}/${NS}/${IMAGE}` +get_pid ${POOL} ${NS} +unmap_device "--snap-id ${SNAPID} ${POOL}/${NS}/${IMAGE}" ${PID} +DEV= + +# map/unmap namespace using options test +DEV=`_sudo rbd device --device-type nbd map --pool ${POOL} --namespace ${NS} --image ${IMAGE}` +get_pid ${POOL} ${NS} +unmap_device "--pool ${POOL} --namespace ${NS} --image ${IMAGE}" ${PID} +DEV=`_sudo rbd device --device-type nbd map --pool ${POOL} --namespace ${NS} --image ${IMAGE} --snap snap` +get_pid ${POOL} ${NS} +unmap_device "--pool ${POOL} --namespace ${NS} --image ${IMAGE} --snap snap" ${PID} +DEV= + +# unmap by image name test 2 +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} +pid=$PID +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${NS}/${IMAGE}` +get_pid ${POOL} ${NS} +unmap_device ${POOL}/${NS}/${IMAGE} ${PID} +DEV= +unmap_device ${POOL}/${IMAGE} ${pid} + +# map/unmap test with just image name and expect image to come from default pool +if [ "${POOL}" = "rbd" ];then + DEV=`_sudo rbd device --device-type nbd map ${IMAGE}` + get_pid ${POOL} + unmap_device ${IMAGE} ${PID} + DEV= +fi + +# map/unmap test with just image name after changing default pool +rbd config global set global rbd_default_pool ${ANOTHER_POOL} +rbd create --size 10M ${IMAGE} +DEV=`_sudo rbd device --device-type nbd map ${IMAGE}` +get_pid ${ANOTHER_POOL} +unmap_device ${IMAGE} ${PID} +DEV= + +# reset +rbd config global rm global rbd_default_pool + +# auto unmap test +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} +_sudo kill ${PID} +for i in `seq 10`; do + rbd device --device-type nbd list | expect_false grep "^${PID} *${POOL} *${IMAGE}" && break + sleep 1 +done +rbd device --device-type nbd list | expect_false grep "^${PID} *${POOL} *${IMAGE}" + +# quiesce test +QUIESCE_HOOK=${TEMPDIR}/quiesce.sh +DEV=`_sudo rbd device --device-type nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} ${POOL}/${IMAGE}` +get_pid ${POOL} + +# test it fails if the hook does not exists +test ! -e ${QUIESCE_HOOK} +expect_false rbd snap create ${POOL}/${IMAGE}@quiesce1 +_sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct + +# test the hook is executed +touch ${QUIESCE_HOOK} +chmod +x ${QUIESCE_HOOK} +cat > ${QUIESCE_HOOK} <<EOF +#/bin/sh +echo "test the hook is executed" >&2 +echo \$1 > ${TEMPDIR}/\$2 +EOF +rbd snap create ${POOL}/${IMAGE}@quiesce1 +_sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct +test "$(cat ${TEMPDIR}/quiesce)" = ${DEV} +test "$(cat ${TEMPDIR}/unquiesce)" = ${DEV} + +# test snap create fails if the hook fails +touch ${QUIESCE_HOOK} +chmod +x ${QUIESCE_HOOK} +cat > ${QUIESCE_HOOK} <<EOF +#/bin/sh +echo "test snap create fails if the hook fails" >&2 +exit 22 +EOF +expect_false rbd snap create ${POOL}/${IMAGE}@quiesce2 +_sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct + +# test the hook is slow +cat > ${QUIESCE_HOOK} <<EOF +#/bin/sh +echo "test the hook is slow" >&2 +sleep 7 +EOF +rbd snap create ${POOL}/${IMAGE}@quiesce2 +_sudo dd if=${DATA} of=${DEV} bs=1M count=1 oflag=direct + +# test rbd-nbd_quiesce hook that comes with distribution +unmap_device ${DEV} ${PID} +LOG_FILE=${TEMPDIR}/rbd-nbd.log +if [ -n "${CEPH_SRC}" ]; then + QUIESCE_HOOK=${CEPH_SRC}/tools/rbd_nbd/rbd-nbd_quiesce + DEV=`_sudo rbd device --device-type nbd map --quiesce --quiesce-hook ${QUIESCE_HOOK} \ + ${POOL}/${IMAGE} --log-file=${LOG_FILE}` +else + DEV=`_sudo rbd device --device-type nbd map --quiesce ${POOL}/${IMAGE} --log-file=${LOG_FILE}` +fi +get_pid ${POOL} +_sudo mkfs ${DEV} +mkdir ${TEMPDIR}/mnt +_sudo mount ${DEV} ${TEMPDIR}/mnt +rbd snap create ${POOL}/${IMAGE}@quiesce3 +_sudo dd if=${DATA} of=${TEMPDIR}/mnt/test bs=1M count=1 oflag=direct +_sudo umount ${TEMPDIR}/mnt +unmap_device ${DEV} ${PID} +DEV= +cat ${LOG_FILE} +expect_false grep 'quiesce failed' ${LOG_FILE} + +# test detach/attach +OUT=`_sudo rbd device --device-type nbd --options try-netlink,show-cookie map ${POOL}/${IMAGE}` +read DEV COOKIE <<< "${OUT}" +get_pid ${POOL} +_sudo mount ${DEV} ${TEMPDIR}/mnt +_sudo rbd device detach ${POOL}/${IMAGE} --device-type nbd +expect_false get_pid ${POOL} +expect_false _sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd +if [ -n "${COOKIE}" ]; then + _sudo rbd device attach --device ${DEV} --cookie ${COOKIE} ${POOL}/${IMAGE} --device-type nbd +else + _sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd --force +fi +get_pid ${POOL} +_sudo rbd device detach ${DEV} --device-type nbd +expect_false get_pid ${POOL} +if [ -n "${COOKIE}" ]; then + _sudo rbd device attach --device ${DEV} --cookie ${COOKIE} ${POOL}/${IMAGE} --device-type nbd +else + _sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd --force +fi +get_pid ${POOL} +ls ${TEMPDIR}/mnt/ +dd if=${TEMPDIR}/mnt/test of=/dev/null bs=1M count=1 +_sudo dd if=${DATA} of=${TEMPDIR}/mnt/test1 bs=1M count=1 oflag=direct +_sudo umount ${TEMPDIR}/mnt +unmap_device ${DEV} ${PID} +# if kernel supports cookies +if [ -n "${COOKIE}" ]; then + OUT=`_sudo rbd device --device-type nbd --show-cookie --cookie "abc de" --options try-netlink map ${POOL}/${IMAGE}` + read DEV ANOTHER_COOKIE <<< "${OUT}" + get_pid ${POOL} + test "${ANOTHER_COOKIE}" = "abc de" + unmap_device ${DEV} ${PID} +fi +DEV= + +# test detach/attach with --snap-id +SNAPID=`rbd snap ls ${POOL}/${IMAGE} | awk '$2 == "snap" {print $1}'` +OUT=`_sudo rbd device --device-type nbd --options try-netlink,show-cookie map --snap-id ${SNAPID} ${POOL}/${IMAGE}` +read DEV COOKIE <<< "${OUT}" +get_pid ${POOL} +_sudo rbd device detach ${POOL}/${IMAGE} --snap-id ${SNAPID} --device-type nbd +expect_false get_pid ${POOL} +expect_false _sudo rbd device attach --device ${DEV} --snap-id ${SNAPID} ${POOL}/${IMAGE} --device-type nbd +if [ -n "${COOKIE}" ]; then + _sudo rbd device attach --device ${DEV} --cookie ${COOKIE} --snap-id ${SNAPID} ${POOL}/${IMAGE} --device-type nbd +else + _sudo rbd device attach --device ${DEV} --snap-id ${SNAPID} ${POOL}/${IMAGE} --device-type nbd --force +fi +get_pid ${POOL} +_sudo rbd device detach ${DEV} --device-type nbd +expect_false get_pid ${POOL} +DEV= + +# test discard granularity with journaling +rbd config image set ${POOL}/${IMAGE} rbd_discard_granularity_bytes 4096 +rbd feature enable ${POOL}/${IMAGE} journaling +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} +# since a discard will now be pruned to only whole blocks (0..4095, 4096..8191) +# let us test all the cases around those alignments. 512 is the smallest +# possible block blkdiscard allows us to use. Thus the test checks +# 512 before, on the alignment, 512 after. +_sudo blkdiscard --offset 0 --length $((4096-512)) ${DEV} +_sudo blkdiscard --offset 0 --length 4096 ${DEV} +_sudo blkdiscard --offset 0 --length $((4096+512)) ${DEV} +_sudo blkdiscard --offset 512 --length $((8192-1024)) ${DEV} +_sudo blkdiscard --offset 512 --length $((8192-512)) ${DEV} +_sudo blkdiscard --offset 512 --length 8192 ${DEV} +# wait for commit log to be empty, 10 seconds should be well enough +tries=0 +queue_length=`rbd journal inspect --pool ${POOL} --image ${IMAGE} | awk '/entries inspected/ {print $1}'` +while [ ${tries} -lt 10 ] && [ ${queue_length} -gt 0 ]; do + rbd journal inspect --pool ${POOL} --image ${IMAGE} --verbose + sleep 1 + queue_length=`rbd journal inspect --pool ${POOL} --image ${IMAGE} | awk '/entries inspected/ {print $1}'` + tries=$((tries+1)) +done +[ ${queue_length} -eq 0 ] +unmap_device ${DEV} ${PID} +DEV= +rbd feature disable ${POOL}/${IMAGE} journaling +rbd config image rm ${POOL}/${IMAGE} rbd_discard_granularity_bytes + +# test that disabling a feature so that the op is proxied to rbd-nbd +# (arranged here by blkdiscard before "rbd feature disable") doesn't hang +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} +rbd feature enable ${POOL}/${IMAGE} journaling +_sudo blkdiscard --offset 0 --length 4096 ${DEV} +rbd feature disable ${POOL}/${IMAGE} journaling +unmap_device ${DEV} ${PID} +DEV= + +# test that rbd_op_threads setting takes effect +EXPECTED=`ceph-conf --show-config-value librados_thread_count` +DEV=`_sudo rbd device --device-type nbd map ${POOL}/${IMAGE}` +get_pid ${POOL} +ACTUAL=`ps -p ${PID} -T | grep -c io_context_pool` +[ ${ACTUAL} -eq ${EXPECTED} ] +unmap_device ${DEV} ${PID} +EXPECTED=$((EXPECTED * 3 + 1)) +DEV=`_sudo rbd device --device-type nbd --rbd-op-threads ${EXPECTED} map ${POOL}/${IMAGE}` +get_pid ${POOL} +ACTUAL=`ps -p ${PID} -T | grep -c io_context_pool` +[ ${ACTUAL} -eq ${EXPECTED} ] +unmap_device ${DEV} ${PID} +DEV= + +echo OK diff --git a/qa/workunits/rbd/rbd_groups.sh b/qa/workunits/rbd/rbd_groups.sh new file mode 100755 index 000000000..a32618484 --- /dev/null +++ b/qa/workunits/rbd/rbd_groups.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash + +set -ex + +# +# rbd_consistency_groups.sh - test consistency groups cli commands +# + +# +# Functions +# + +create_group() +{ + local group_name=$1 + + rbd group create $group_name +} + +list_groups() +{ + rbd group list +} + +check_group_exists() +{ + local group_name=$1 + list_groups | grep $group_name +} + +remove_group() +{ + local group_name=$1 + + rbd group remove $group_name +} + +rename_group() +{ + local src_name=$1 + local dest_name=$2 + + rbd group rename $src_name $dest_name +} + +check_group_does_not_exist() +{ + local group_name=$1 + for v in $(list_groups); do + if [ "$v" == "$group_name" ]; then + return 1 + fi + done + return 0 +} + +create_image() +{ + local image_name=$1 + rbd create --size 10M $image_name +} + +remove_image() +{ + local image_name=$1 + rbd remove $image_name +} + +add_image_to_group() +{ + local image_name=$1 + local group_name=$2 + rbd group image add $group_name $image_name +} + +remove_image_from_group() +{ + local image_name=$1 + local group_name=$2 + rbd group image remove $group_name $image_name +} + +check_image_in_group() +{ + local image_name=$1 + local group_name=$2 + for v in $(rbd group image list $group_name); do + local vtrimmed=${v#*/} + if [ "$vtrimmed" = "$image_name" ]; then + return 0 + fi + done + return 1 +} + +check_image_not_in_group() +{ + local image_name=$1 + local group_name=$2 + for v in $(rbd group image list $group_name); do + local vtrimmed=${v#*/} + if [ "$vtrimmed" = "$image_name" ]; then + return 1 + fi + done + return 0 +} + +create_snapshot() +{ + local group_name=$1 + local snap_name=$2 + rbd group snap create $group_name@$snap_name +} + +create_snapshots() +{ + local group_name=$1 + local snap_name=$2 + local snap_count=$3 + for i in `seq 1 $snap_count`; do + rbd group snap create $group_name@$snap_name$i + done +} + +remove_snapshot() +{ + local group_name=$1 + local snap_name=$2 + rbd group snap remove $group_name@$snap_name +} + +remove_snapshots() +{ + local group_name=$1 + local snap_name=$2 + local snap_count=$3 + for i in `seq 1 $snap_count`; do + rbd group snap remove $group_name@$snap_name$i + done +} + +rename_snapshot() +{ + local group_name=$1 + local snap_name=$2 + local new_snap_name=$3 + rbd group snap rename $group_name@$snap_name $new_snap_name +} + +list_snapshots() +{ + local group_name=$1 + rbd group snap list $group_name +} + +rollback_snapshot() +{ + local group_name=$1 + local snap_name=$2 + rbd group snap rollback $group_name@$snap_name +} + +check_snapshot_in_group() +{ + local group_name=$1 + local snap_name=$2 + list_snapshots $group_name | grep $snap_name +} + +check_snapshots_count_in_group() +{ + local group_name=$1 + local snap_name=$2 + local expected_count=$3 + local actual_count + actual_count=$(list_snapshots $group_name | grep -c $snap_name) + (( actual_count == expected_count )) +} + +check_snapshot_not_in_group() +{ + local group_name=$1 + local snap_name=$2 + for v in $(list_snapshots $group_name | awk '{print $1}'); do + if [ "$v" = "$snap_name" ]; then + return 1 + fi + done + return 0 +} + +echo "TEST: create remove consistency group" +group="test_consistency_group" +new_group="test_new_consistency_group" +create_group $group +check_group_exists $group +rename_group $group $new_group +check_group_exists $new_group +remove_group $new_group +check_group_does_not_exist $new_group +echo "PASSED" + +echo "TEST: add remove images to consistency group" +image="test_image" +group="test_consistency_group" +create_image $image +create_group $group +add_image_to_group $image $group +check_image_in_group $image $group +remove_image_from_group $image $group +check_image_not_in_group $image $group +remove_group $group +remove_image $image +echo "PASSED" + +echo "TEST: create remove snapshots of consistency group" +image="test_image" +group="test_consistency_group" +snap="group_snap" +new_snap="new_group_snap" +sec_snap="group_snap2" +create_image $image +create_group $group +add_image_to_group $image $group +create_snapshot $group $snap +check_snapshot_in_group $group $snap +rename_snapshot $group $snap $new_snap +check_snapshot_not_in_group $group $snap +create_snapshot $group $sec_snap +check_snapshot_in_group $group $sec_snap +rollback_snapshot $group $new_snap +remove_snapshot $group $new_snap +check_snapshot_not_in_group $group $new_snap +remove_snapshot $group $sec_snap +check_snapshot_not_in_group $group $sec_snap +remove_group $group +remove_image $image +echo "PASSED" + +echo "TEST: list snapshots of consistency group" +image="test_image" +group="test_consistency_group" +snap="group_snap" +create_image $image +create_group $group +add_image_to_group $image $group +create_snapshots $group $snap 10 +check_snapshots_count_in_group $group $snap 10 +remove_snapshots $group $snap 10 +create_snapshots $group $snap 100 +check_snapshots_count_in_group $group $snap 100 +remove_snapshots $group $snap 100 +remove_group $group +remove_image $image +echo "PASSED" + +echo "OK" diff --git a/qa/workunits/rbd/rbd_mirror_bootstrap.sh b/qa/workunits/rbd/rbd_mirror_bootstrap.sh new file mode 100755 index 000000000..6ef06f2b8 --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_bootstrap.sh @@ -0,0 +1,58 @@ +#!/bin/sh -ex +# +# rbd_mirror_bootstrap.sh - test peer bootstrap create/import +# + +RBD_MIRROR_MANUAL_PEERS=1 +RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-1} +. $(dirname $0)/rbd_mirror_helpers.sh + +setup + +testlog "TEST: bootstrap cluster2 from cluster1" +# create token on cluster1 and import to cluster2 +TOKEN=${TEMPDIR}/peer-token +TOKEN_2=${TEMPDIR}/peer-token-2 +CEPH_ARGS='' rbd --cluster ${CLUSTER1} mirror pool peer bootstrap create ${POOL} > ${TOKEN} +CEPH_ARGS='' rbd --cluster ${CLUSTER1} mirror pool peer bootstrap create ${PARENT_POOL} > ${TOKEN_2} +cmp ${TOKEN} ${TOKEN_2} + +CEPH_ARGS='' rbd --cluster ${CLUSTER2} --pool ${POOL} mirror pool peer bootstrap import ${TOKEN} --direction rx-only +CEPH_ARGS='' rbd --cluster ${CLUSTER2} --pool ${PARENT_POOL} mirror pool peer bootstrap import ${TOKEN} --direction rx-tx + +start_mirrors ${CLUSTER1} +start_mirrors ${CLUSTER2} + +testlog "TEST: verify rx-only direction" +# rx-only peer is added immediately by "rbd mirror pool peer bootstrap import" +rbd --cluster ${CLUSTER2} --pool ${POOL} mirror pool info --format json | jq -e '.peers[0].direction == "rx-only"' +# tx-only peer is added asynchronously by mirror_peer_ping class method +while ! rbd --cluster ${CLUSTER1} --pool ${POOL} mirror pool info --format json | jq -e '.peers | length > 0'; do + sleep 1 +done +rbd --cluster ${CLUSTER1} --pool ${POOL} mirror pool info --format json | jq -e '.peers[0].direction == "tx-only"' + +create_image_and_enable_mirror ${CLUSTER1} ${POOL} image1 + +wait_for_image_replay_started ${CLUSTER2} ${POOL} image1 +write_image ${CLUSTER1} ${POOL} image1 100 +wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} image1 + +testlog "TEST: verify rx-tx direction" +# both rx-tx peers are added immediately by "rbd mirror pool peer bootstrap import" +rbd --cluster ${CLUSTER1} --pool ${PARENT_POOL} mirror pool info --format json | jq -e '.peers[0].direction == "rx-tx"' +rbd --cluster ${CLUSTER2} --pool ${PARENT_POOL} mirror pool info --format json | jq -e '.peers[0].direction == "rx-tx"' + +create_image ${CLUSTER1} ${PARENT_POOL} image1 +create_image ${CLUSTER2} ${PARENT_POOL} image2 + +enable_mirror ${CLUSTER1} ${PARENT_POOL} image1 +enable_mirror ${CLUSTER2} ${PARENT_POOL} image2 + +wait_for_image_replay_started ${CLUSTER2} ${PARENT_POOL} image1 +write_image ${CLUSTER1} ${PARENT_POOL} image1 100 +wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${PARENT_POOL} image1 + +wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} image2 +write_image ${CLUSTER2} ${PARENT_POOL} image2 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} image2 diff --git a/qa/workunits/rbd/rbd_mirror_fsx_compare.sh b/qa/workunits/rbd/rbd_mirror_fsx_compare.sh new file mode 100755 index 000000000..0ba3c97d7 --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_fsx_compare.sh @@ -0,0 +1,38 @@ +#!/bin/sh -ex +# +# rbd_mirror_fsx_compare.sh - test rbd-mirror daemon under FSX workload +# +# The script is used to compare FSX-generated images between two clusters. +# + +. $(dirname $0)/rbd_mirror_helpers.sh + +trap 'cleanup $?' INT TERM EXIT + +setup_tempdir + +testlog "TEST: wait for all images" +image_count=$(rbd --cluster ${CLUSTER1} --pool ${POOL} ls | wc -l) +retrying_seconds=0 +sleep_seconds=10 +while [ ${retrying_seconds} -le 7200 ]; do + [ $(rbd --cluster ${CLUSTER2} --pool ${POOL} ls | wc -l) -ge ${image_count} ] && break + sleep ${sleep_seconds} + retrying_seconds=$(($retrying_seconds+${sleep_seconds})) +done + +testlog "TEST: snapshot all pool images" +snap_id=`uuidgen` +for image in $(rbd --cluster ${CLUSTER1} --pool ${POOL} ls); do + create_snapshot ${CLUSTER1} ${POOL} ${image} ${snap_id} +done + +testlog "TEST: wait for snapshots" +for image in $(rbd --cluster ${CLUSTER1} --pool ${POOL} ls); do + wait_for_snap_present ${CLUSTER2} ${POOL} ${image} ${snap_id} +done + +testlog "TEST: compare image snapshots" +for image in $(rbd --cluster ${CLUSTER1} --pool ${POOL} ls); do + compare_image_snapshots ${POOL} ${image} +done diff --git a/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh b/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh new file mode 100755 index 000000000..d988987ba --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh @@ -0,0 +1,10 @@ +#!/bin/sh -ex +# +# rbd_mirror_fsx_prepare.sh - test rbd-mirror daemon under FSX workload +# +# The script is used to compare FSX-generated images between two clusters. +# + +. $(dirname $0)/rbd_mirror_helpers.sh + +setup diff --git a/qa/workunits/rbd/rbd_mirror_ha.sh b/qa/workunits/rbd/rbd_mirror_ha.sh new file mode 100755 index 000000000..37739a83d --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_ha.sh @@ -0,0 +1,210 @@ +#!/bin/sh -ex +# +# rbd_mirror_ha.sh - test rbd-mirror daemons in HA mode +# + +RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-7} + +. $(dirname $0)/rbd_mirror_helpers.sh + +setup + +is_leader() +{ + local instance=$1 + local pool=$2 + + test -n "${pool}" || pool=${POOL} + + admin_daemon "${CLUSTER1}:${instance}" \ + rbd mirror status ${pool} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} | + grep '"leader": true' +} + +wait_for_leader() +{ + local s instance + + for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64; do + sleep $s + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + is_leader ${instance} || continue + LEADER=${instance} + return 0 + done + done + + LEADER= + return 1 +} + +release_leader() +{ + local pool=$1 + local cmd="rbd mirror leader release" + + test -n "${pool}" && cmd="${cmd} ${pool} ${CLUSTER2}" + + admin_daemon "${CLUSTER1}:${LEADER}" ${cmd} +} + +wait_for_leader_released() +{ + local i + + test -n "${LEADER}" + for i in `seq 10`; do + is_leader ${LEADER} || return 0 + sleep 1 + done + + return 1 +} + +test_replay() +{ + local image + + for image; do + wait_for_image_replay_started ${CLUSTER1}:${LEADER} ${POOL} ${image} + write_image ${CLUSTER2} ${POOL} ${image} 100 + wait_for_replay_complete ${CLUSTER1}:${LEADER} ${CLUSTER2} ${POOL} \ + ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' \ + 'primary_position' \ + "${MIRROR_USER_ID_PREFIX}${LEADER} on $(hostname -s)" + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} \ + 'down+unknown' + fi + compare_images ${POOL} ${image} + done +} + +testlog "TEST: start first daemon instance and test replay" +start_mirror ${CLUSTER1}:0 +image1=test1 +create_image ${CLUSTER2} ${POOL} ${image1} +LEADER=0 +test_replay ${image1} + +testlog "TEST: release leader and wait it is reacquired" +is_leader 0 ${POOL} +is_leader 0 ${PARENT_POOL} +release_leader ${POOL} +wait_for_leader_released +is_leader 0 ${PARENT_POOL} +wait_for_leader +release_leader +wait_for_leader_released +expect_failure "" is_leader 0 ${PARENT_POOL} +wait_for_leader + +testlog "TEST: start second daemon instance and test replay" +start_mirror ${CLUSTER1}:1 +image2=test2 +create_image ${CLUSTER2} ${POOL} ${image2} +test_replay ${image1} ${image2} + +testlog "TEST: release leader and test it is acquired by secondary" +is_leader 0 ${POOL} +is_leader 0 ${PARENT_POOL} +release_leader ${POOL} +wait_for_leader_released +wait_for_leader +test_replay ${image1} ${image2} +release_leader +wait_for_leader_released +wait_for_leader +test "${LEADER}" = 0 + +testlog "TEST: stop first daemon instance and test replay" +stop_mirror ${CLUSTER1}:0 +image3=test3 +create_image ${CLUSTER2} ${POOL} ${image3} +LEADER=1 +test_replay ${image1} ${image2} ${image3} + +testlog "TEST: start first daemon instance and test replay" +start_mirror ${CLUSTER1}:0 +image4=test4 +create_image ${CLUSTER2} ${POOL} ${image4} +test_replay ${image3} ${image4} + +testlog "TEST: crash leader and test replay" +stop_mirror ${CLUSTER1}:1 -KILL +image5=test5 +create_image ${CLUSTER2} ${POOL} ${image5} +LEADER=0 +test_replay ${image1} ${image4} ${image5} + +testlog "TEST: start crashed leader and test replay" +start_mirror ${CLUSTER1}:1 +image6=test6 +create_image ${CLUSTER2} ${POOL} ${image6} +test_replay ${image1} ${image6} + +testlog "TEST: start yet another daemon instance and test replay" +start_mirror ${CLUSTER1}:2 +image7=test7 +create_image ${CLUSTER2} ${POOL} ${image7} +test_replay ${image1} ${image7} + +testlog "TEST: release leader and test it is acquired by secondary" +is_leader 0 +release_leader +wait_for_leader_released +wait_for_leader +test_replay ${image1} ${image2} + +testlog "TEST: stop leader and test replay" +stop_mirror ${CLUSTER1}:${LEADER} +image8=test8 +create_image ${CLUSTER2} ${POOL} ${image8} +prev_leader=${LEADER} +wait_for_leader +test_replay ${image1} ${image8} + +testlog "TEST: start previous leader and test replay" +start_mirror ${CLUSTER1}:${prev_leader} +image9=test9 +create_image ${CLUSTER2} ${POOL} ${image9} +test_replay ${image1} ${image9} + +testlog "TEST: crash leader and test replay" +stop_mirror ${CLUSTER1}:${LEADER} -KILL +image10=test10 +create_image ${CLUSTER2} ${POOL} ${image10} +prev_leader=${LEADER} +wait_for_leader +test_replay ${image1} ${image10} + +testlog "TEST: start previous leader and test replay" +start_mirror ${CLUSTER1}:${prev_leader} +image11=test11 +create_image ${CLUSTER2} ${POOL} ${image11} +test_replay ${image1} ${image11} + +testlog "TEST: start some more daemon instances and test replay" +start_mirror ${CLUSTER1}:3 +start_mirror ${CLUSTER1}:4 +start_mirror ${CLUSTER1}:5 +start_mirror ${CLUSTER1}:6 +image13=test13 +create_image ${CLUSTER2} ${POOL} ${image13} +test_replay ${leader} ${image1} ${image13} + +testlog "TEST: release leader and test it is acquired by secondary" +release_leader +wait_for_leader_released +wait_for_leader +test_replay ${image1} ${image2} + +testlog "TEST: in loop: stop leader and test replay" +for i in 0 1 2 3 4 5; do + stop_mirror ${CLUSTER1}:${LEADER} + wait_for_leader + test_replay ${image1} +done + +stop_mirror ${CLUSTER1}:${LEADER} diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh new file mode 100755 index 000000000..f4961b925 --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -0,0 +1,1488 @@ +#!/bin/sh +# +# rbd_mirror_helpers.sh - shared rbd-mirror daemon helper functions +# +# The scripts starts two ("local" and "remote") clusters using mstart.sh script, +# creates a temporary directory, used for cluster configs, daemon logs, admin +# socket, temporary files, and launches rbd-mirror daemon. +# +# There are several env variables useful when troubleshooting a test failure: +# +# RBD_MIRROR_NOCLEANUP - if not empty, don't run the cleanup (stop processes, +# destroy the clusters and remove the temp directory) +# on exit, so it is possible to check the test state +# after failure. +# RBD_MIRROR_TEMDIR - use this path when creating the temporary directory +# (should not exist) instead of running mktemp(1). +# RBD_MIRROR_ARGS - use this to pass additional arguments to started +# rbd-mirror daemons. +# RBD_MIRROR_VARGS - use this to pass additional arguments to vstart.sh +# when starting clusters. +# RBD_MIRROR_INSTANCES - number of daemons to start per cluster +# RBD_MIRROR_CONFIG_KEY - if not empty, use config-key for remote cluster +# secrets +# The cleanup can be done as a separate step, running the script with +# `cleanup ${RBD_MIRROR_TEMDIR}' arguments. +# +# Note, as other workunits tests, rbd_mirror_journal.sh expects to find ceph binaries +# in PATH. +# +# Thus a typical troubleshooting session: +# +# From Ceph src dir (CEPH_SRC_PATH), start the test in NOCLEANUP mode and with +# TEMPDIR pointing to a known location: +# +# cd $CEPH_SRC_PATH +# PATH=$CEPH_SRC_PATH:$PATH +# RBD_MIRROR_NOCLEANUP=1 RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ +# ../qa/workunits/rbd/rbd_mirror_journal.sh +# +# After the test failure cd to TEMPDIR and check the current state: +# +# cd /tmp/tmp.rbd_mirror +# ls +# less rbd-mirror.cluster1_daemon.$pid.log +# ceph --cluster cluster1 -s +# ceph --cluster cluster1 -s +# rbd --cluster cluster2 -p mirror ls +# rbd --cluster cluster2 -p mirror journal status --image test +# ceph --admin-daemon rbd-mirror.cluster1_daemon.cluster1.$pid.asok help +# ... +# +# Also you can execute commands (functions) from the script: +# +# cd $CEPH_SRC_PATH +# export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror +# ../qa/workunits/rbd/rbd_mirror_journal.sh status +# ../qa/workunits/rbd/rbd_mirror_journal.sh stop_mirror cluster1 +# ../qa/workunits/rbd/rbd_mirror_journal.sh start_mirror cluster2 +# ../qa/workunits/rbd/rbd_mirror_journal.sh flush cluster2 +# ... +# +# Eventually, run the cleanup: +# +# cd $CEPH_SRC_PATH +# RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ +# ../qa/workunits/rbd/rbd_mirror_journal.sh cleanup +# + +if type xmlstarlet > /dev/null 2>&1; then + XMLSTARLET=xmlstarlet +elif type xml > /dev/null 2>&1; then + XMLSTARLET=xml +else + echo "Missing xmlstarlet binary!" + exit 1 +fi + +RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-2} + +CLUSTER1=cluster1 +CLUSTER2=cluster2 +PEER_CLUSTER_SUFFIX= +POOL=mirror +PARENT_POOL=mirror_parent +NS1=ns1 +NS2=ns2 +TEMPDIR= +CEPH_ID=${CEPH_ID:-mirror} +RBD_IMAGE_FEATURES=${RBD_IMAGE_FEATURES:-layering,exclusive-lock,journaling} +MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.} +MIRROR_POOL_MODE=${MIRROR_POOL_MODE:-pool} +MIRROR_IMAGE_MODE=${MIRROR_IMAGE_MODE:-journal} + +export CEPH_ARGS="--id ${CEPH_ID}" + +LAST_MIRROR_INSTANCE=$((${RBD_MIRROR_INSTANCES} - 1)) + +CEPH_ROOT=$(readlink -f $(dirname $0)/../../../src) +CEPH_BIN=. +CEPH_SRC=. +if [ -e CMakeCache.txt ]; then + CEPH_SRC=${CEPH_ROOT} + CEPH_ROOT=${PWD} + CEPH_BIN=./bin + + # needed for ceph CLI under cmake + export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH} + export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind:${CEPH_ROOT}/lib/cython_modules/lib.3 +fi + +# These vars facilitate running this script in an environment with +# ceph installed from packages, like teuthology. These are not defined +# by default. +# +# RBD_MIRROR_USE_EXISTING_CLUSTER - if set, do not start and stop ceph clusters +# RBD_MIRROR_USE_RBD_MIRROR - if set, use an existing instance of rbd-mirror +# running as ceph client $CEPH_ID. If empty, +# this script will start and stop rbd-mirror + +# +# Functions +# + +# Parse a value in format cluster[:instance] and set cluster and instance vars. +set_cluster_instance() +{ + local val=$1 + local cluster_var_name=$2 + local instance_var_name=$3 + + cluster=${val%:*} + instance=${val##*:} + + if [ "${instance}" = "${val}" ]; then + # instance was not specified, use default + instance=0 + fi + + eval ${cluster_var_name}=${cluster} + eval ${instance_var_name}=${instance} +} + +daemon_asok_file() +{ + local local_cluster=$1 + local cluster=$2 + local instance + + set_cluster_instance "${local_cluster}" local_cluster instance + + echo $(ceph-conf --cluster $local_cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'admin socket') +} + +daemon_pid_file() +{ + local cluster=$1 + local instance + + set_cluster_instance "${cluster}" cluster instance + + echo $(ceph-conf --cluster $cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'pid file') +} + +testlog() +{ + echo $(date '+%F %T') $@ | tee -a "${TEMPDIR}/rbd-mirror.test.log" >&2 +} + +expect_failure() +{ + local expected="$1" ; shift + local out=${TEMPDIR}/expect_failure.out + + if "$@" > ${out} 2>&1 ; then + cat ${out} >&2 + return 1 + fi + + if [ -z "${expected}" ]; then + return 0 + fi + + if ! grep -q "${expected}" ${out} ; then + cat ${out} >&2 + return 1 + fi + + return 0 +} + +mkfname() +{ + echo "$@" | sed -e 's|[/ ]|_|g' +} + +create_users() +{ + local cluster=$1 + + CEPH_ARGS='' ceph --cluster "${cluster}" \ + auth get-or-create client.${CEPH_ID} \ + mon 'profile rbd' osd 'profile rbd' mgr 'profile rbd' >> \ + ${CEPH_ROOT}/run/${cluster}/keyring + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + CEPH_ARGS='' ceph --cluster "${cluster}" \ + auth get-or-create client.${MIRROR_USER_ID_PREFIX}${instance} \ + mon 'profile rbd-mirror' osd 'profile rbd' mgr 'profile rbd' >> \ + ${CEPH_ROOT}/run/${cluster}/keyring + done +} + +setup_cluster() +{ + local cluster=$1 + + CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${cluster} -n ${RBD_MIRROR_VARGS} + + cd ${CEPH_ROOT} + rm -f ${TEMPDIR}/${cluster}.conf + ln -s $(readlink -f run/${cluster}/ceph.conf) \ + ${TEMPDIR}/${cluster}.conf + + cd ${TEMPDIR} + create_users "${cluster}" + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + cat<<EOF >> ${TEMPDIR}/${cluster}.conf +[client.${MIRROR_USER_ID_PREFIX}${instance}] + admin socket = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.asok + pid file = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.pid + log file = ${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.log +EOF + done +} + +peer_add() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local client_cluster=$1 ; shift + local remote_cluster="${client_cluster##*@}" + + local uuid_var_name + if [ -n "$1" ]; then + uuid_var_name=$1 ; shift + fi + + local error_code + local peer_uuid + + for s in 1 2 4 8 16 32; do + set +e + peer_uuid=$(rbd --cluster ${cluster} mirror pool peer add \ + ${pool} ${client_cluster} $@) + error_code=$? + set -e + + if [ $error_code -eq 17 ]; then + # raced with a remote heartbeat ping -- remove and retry + sleep $s + peer_uuid=$(rbd mirror pool info --cluster ${cluster} --pool ${pool} --format xml | \ + xmlstarlet sel -t -v "//peers/peer[site_name='${remote_cluster}']/uuid") + + CEPH_ARGS='' rbd --cluster ${cluster} --pool ${pool} mirror pool peer remove ${peer_uuid} + else + test $error_code -eq 0 + if [ -n "$uuid_var_name" ]; then + eval ${uuid_var_name}=${peer_uuid} + fi + return 0 + fi + done + + return 1 +} + +setup_pools() +{ + local cluster=$1 + local remote_cluster=$2 + local mon_map_file + local mon_addr + local admin_key_file + local uuid + + CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${POOL} 64 64 + CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${PARENT_POOL} 64 64 + + CEPH_ARGS='' rbd --cluster ${cluster} pool init ${POOL} + CEPH_ARGS='' rbd --cluster ${cluster} pool init ${PARENT_POOL} + + if [ -n "${RBD_MIRROR_CONFIG_KEY}" ]; then + PEER_CLUSTER_SUFFIX=-DNE + fi + + CEPH_ARGS='' rbd --cluster ${cluster} mirror pool enable \ + --site-name ${cluster}${PEER_CLUSTER_SUFFIX} ${POOL} ${MIRROR_POOL_MODE} + rbd --cluster ${cluster} mirror pool enable ${PARENT_POOL} image + + rbd --cluster ${cluster} namespace create ${POOL}/${NS1} + rbd --cluster ${cluster} namespace create ${POOL}/${NS2} + + rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS1} ${MIRROR_POOL_MODE} + rbd --cluster ${cluster} mirror pool enable ${POOL}/${NS2} image + + if [ -z ${RBD_MIRROR_MANUAL_PEERS} ]; then + if [ -z ${RBD_MIRROR_CONFIG_KEY} ]; then + peer_add ${cluster} ${POOL} ${remote_cluster} + peer_add ${cluster} ${PARENT_POOL} ${remote_cluster} + else + mon_map_file=${TEMPDIR}/${remote_cluster}.monmap + CEPH_ARGS='' ceph --cluster ${remote_cluster} mon getmap > ${mon_map_file} + mon_addr=$(monmaptool --print ${mon_map_file} | grep -E 'mon\.' | + head -n 1 | sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' | sed -E 's/\/[0-9]+//g') + + admin_key_file=${TEMPDIR}/${remote_cluster}.client.${CEPH_ID}.key + CEPH_ARGS='' ceph --cluster ${remote_cluster} auth get-key client.${CEPH_ID} > ${admin_key_file} + + CEPH_ARGS='' peer_add ${cluster} ${POOL} \ + client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} '' \ + --remote-mon-host "${mon_addr}" --remote-key-file ${admin_key_file} + + peer_add ${cluster} ${PARENT_POOL} client.${CEPH_ID}@${remote_cluster}${PEER_CLUSTER_SUFFIX} uuid + CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr} + CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file} + fi + fi +} + +setup_tempdir() +{ + if [ -n "${RBD_MIRROR_TEMDIR}" ]; then + test -d "${RBD_MIRROR_TEMDIR}" || + mkdir "${RBD_MIRROR_TEMDIR}" + TEMPDIR="${RBD_MIRROR_TEMDIR}" + cd ${TEMPDIR} + else + TEMPDIR=`mktemp -d` + fi +} + +setup() +{ + local c + trap 'cleanup $?' INT TERM EXIT + + setup_tempdir + if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then + setup_cluster "${CLUSTER1}" + setup_cluster "${CLUSTER2}" + fi + + setup_pools "${CLUSTER1}" "${CLUSTER2}" + setup_pools "${CLUSTER2}" "${CLUSTER1}" + + if [ -n "${RBD_MIRROR_MIN_COMPAT_CLIENT}" ]; then + CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd \ + set-require-min-compat-client ${RBD_MIRROR_MIN_COMPAT_CLIENT} + CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd \ + set-require-min-compat-client ${RBD_MIRROR_MIN_COMPAT_CLIENT} + fi +} + +cleanup() +{ + local error_code=$1 + + set +e + + if [ "${error_code}" -ne 0 ]; then + status + fi + + if [ -z "${RBD_MIRROR_NOCLEANUP}" ]; then + local cluster instance + + CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it + CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it + CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it + CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it + + for cluster in "${CLUSTER1}" "${CLUSTER2}"; do + stop_mirrors "${cluster}" + done + + if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then + cd ${CEPH_ROOT} + CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER1} + CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER2} + fi + test "${RBD_MIRROR_TEMDIR}" = "${TEMPDIR}" || rm -Rf ${TEMPDIR} + fi + + if [ "${error_code}" -eq 0 ]; then + echo "OK" + else + echo "FAIL" + fi + + exit ${error_code} +} + +start_mirror() +{ + local cluster=$1 + local instance + + set_cluster_instance "${cluster}" cluster instance + + test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return + + rbd-mirror \ + --cluster ${cluster} \ + --id ${MIRROR_USER_ID_PREFIX}${instance} \ + --rbd-mirror-delete-retry-interval=5 \ + --rbd-mirror-image-state-check-interval=5 \ + --rbd-mirror-journal-poll-age=1 \ + --rbd-mirror-pool-replayers-refresh-interval=5 \ + --debug-rbd=30 --debug-journaler=30 \ + --debug-rbd_mirror=30 \ + --daemonize=true \ + ${RBD_MIRROR_ARGS} +} + +start_mirrors() +{ + local cluster=$1 + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + start_mirror "${cluster}:${instance}" + done +} + +stop_mirror() +{ + local cluster=$1 + local sig=$2 + + test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return + + local pid + pid=$(cat $(daemon_pid_file "${cluster}") 2>/dev/null) || : + if [ -n "${pid}" ] + then + kill ${sig} ${pid} + for s in 1 2 4 8 16 32; do + sleep $s + ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break + done + ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' + fi + rm -f $(daemon_asok_file "${cluster}" "${CLUSTER1}") + rm -f $(daemon_asok_file "${cluster}" "${CLUSTER2}") + rm -f $(daemon_pid_file "${cluster}") +} + +stop_mirrors() +{ + local cluster=$1 + local sig=$2 + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + stop_mirror "${cluster}:${instance}" "${sig}" + done +} + +admin_daemon() +{ + local cluster=$1 ; shift + local instance + + set_cluster_instance "${cluster}" cluster instance + + local asok_file=$(daemon_asok_file "${cluster}:${instance}" "${cluster}") + test -S "${asok_file}" + + ceph --admin-daemon ${asok_file} $@ +} + +admin_daemons() +{ + local cluster_instance=$1 ; shift + local cluster="${cluster_instance%:*}" + local instance="${cluster_instance##*:}" + local loop_instance + + for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16; do + sleep ${s} + if [ "${instance}" != "${cluster_instance}" ]; then + admin_daemon "${cluster}:${instance}" $@ && return 0 + else + for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + admin_daemon "${cluster}:${loop_instance}" $@ && return 0 + done + fi + done + return 1 +} + +all_admin_daemons() +{ + local cluster=$1 ; shift + + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + admin_daemon "${cluster}:${instance}" $@ + done +} + +status() +{ + local cluster daemon image_pool image_ns image + + for cluster in ${CLUSTER1} ${CLUSTER2} + do + echo "${cluster} status" + CEPH_ARGS='' ceph --cluster ${cluster} -s + CEPH_ARGS='' ceph --cluster ${cluster} service dump + CEPH_ARGS='' ceph --cluster ${cluster} service status + echo + + for image_pool in ${POOL} ${PARENT_POOL} + do + for image_ns in "" "${NS1}" "${NS2}" + do + echo "${cluster} ${image_pool} ${image_ns} images" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls -l + echo + + echo "${cluster} ${image_pool}${image_ns} mirror pool info" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool info + echo + + echo "${cluster} ${image_pool}${image_ns} mirror pool status" + CEPH_ARGS='' rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" mirror pool status --verbose + echo + + for image in `rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls 2>/dev/null` + do + echo "image ${image} info" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" info ${image} + echo + echo "image ${image} journal status" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" journal status --image ${image} + echo + echo "image ${image} snapshots" + rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" snap ls --all ${image} + echo + done + + echo "${cluster} ${image_pool} ${image_ns} rbd_mirroring omap vals" + rados --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" listomapvals rbd_mirroring + echo "${cluster} ${image_pool} ${image_ns} rbd_mirror_leader omap vals" + rados --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" listomapvals rbd_mirror_leader + echo + done + done + done + + local ret + + for cluster in "${CLUSTER1}" "${CLUSTER2}" + do + for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do + local pid_file=$(daemon_pid_file ${cluster}:${instance}) + if [ ! -e ${pid_file} ] + then + echo "${cluster} rbd-mirror not running or unknown" \ + "(${pid_file} not exist)" + continue + fi + + local pid + pid=$(cat ${pid_file} 2>/dev/null) || : + if [ -z "${pid}" ] + then + echo "${cluster} rbd-mirror not running or unknown" \ + "(can't find pid using ${pid_file})" + ret=1 + continue + fi + + echo "${daemon} rbd-mirror process in ps output:" + if ps auxww | + awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}' + then + echo + echo "${cluster} rbd-mirror not running" \ + "(can't find pid $pid in ps output)" + ret=1 + continue + fi + echo + + local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster}) + if [ ! -S "${asok_file}" ] + then + echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)" + ret=1 + continue + fi + + echo "${cluster} rbd-mirror status" + ceph --admin-daemon ${asok_file} rbd mirror status + echo + done + done + + return ${ret} +} + +flush() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local cmd="rbd mirror flush" + + if [ -n "${image}" ] + then + cmd="${cmd} ${pool}/${image}" + fi + + admin_daemons "${cluster}" ${cmd} +} + +test_image_replay_state() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local test_state=$4 + local status_result + local current_state=stopped + + status_result=$(admin_daemons "${cluster}" rbd mirror status ${pool}/${image} | grep -i 'state') || return 1 + echo "${status_result}" | grep -i 'Replaying' && current_state=started + test "${test_state}" = "${current_state}" +} + +wait_for_image_replay_state() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state=$4 + local s + + # TODO: add a way to force rbd-mirror to update replayers + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do + sleep ${s} + test_image_replay_state "${cluster}" "${pool}" "${image}" "${state}" && return 0 + done + return 1 +} + +wait_for_image_replay_started() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + wait_for_image_replay_state "${cluster}" "${pool}" "${image}" started +} + +wait_for_image_replay_stopped() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + wait_for_image_replay_state "${cluster}" "${pool}" "${image}" stopped +} + +get_journal_position() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local id_regexp=$4 + + # Parse line like below, looking for the first position + # [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=9], [object_number=0, tag_tid=3, entry_tid=8], [object_number=3, tag_tid=3, entry_tid=7], [object_number=2, tag_tid=3, entry_tid=6]]]] + + local status_log=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.status) + rbd --cluster ${cluster} journal status --image ${pool}/${image} | + tee ${status_log} >&2 + sed -nEe 's/^.*\[id='"${id_regexp}"',.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' \ + ${status_log} +} + +get_master_journal_position() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + get_journal_position "${cluster}" "${pool}" "${image}" '' +} + +get_mirror_journal_position() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + get_journal_position "${cluster}" "${pool}" "${image}" '..*' +} + +wait_for_journal_replay_complete() +{ + local local_cluster=$1 + local cluster=$2 + local pool=$3 + local image=$4 + local s master_pos mirror_pos last_mirror_pos + local master_tag master_entry mirror_tag mirror_entry + + while true; do + for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do + sleep ${s} + flush "${local_cluster}" "${pool}" "${image}" + master_pos=$(get_master_journal_position "${cluster}" "${pool}" "${image}") + mirror_pos=$(get_mirror_journal_position "${cluster}" "${pool}" "${image}") + test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0 + test "${mirror_pos}" != "${last_mirror_pos}" && break + done + + test "${mirror_pos}" = "${last_mirror_pos}" && return 1 + last_mirror_pos="${mirror_pos}" + + # handle the case where the mirror is ahead of the master + master_tag=$(echo "${master_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) + mirror_tag=$(echo "${mirror_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2) + master_entry=$(echo "${master_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) + mirror_entry=$(echo "${mirror_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2) + test "${master_tag}" = "${mirror_tag}" -a ${master_entry} -le ${mirror_entry} && return 0 + done + return 1 +} + +mirror_image_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster "${cluster}" mirror image snapshot "${pool}/${image}" +} + +get_newest_mirror_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local log=$4 + + rbd --cluster "${cluster}" snap list --all "${pool}/${image}" --format xml | \ + xmlstarlet sel -t -c "//snapshots/snapshot[namespace/complete='true' and position()=last()]" > \ + ${log} || true +} + +wait_for_snapshot_sync_complete() +{ + local local_cluster=$1 + local cluster=$2 + local pool=$3 + local image=$4 + + local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.status) + local local_status_log=${TEMPDIR}/$(mkfname ${local_cluster}-${pool}-${image}.status) + + mirror_image_snapshot "${cluster}" "${pool}" "${image}" + get_newest_mirror_snapshot "${cluster}" "${pool}" "${image}" "${status_log}" + local snapshot_id=$(xmlstarlet sel -t -v "//snapshot/id" < ${status_log}) + + while true; do + for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do + sleep ${s} + + get_newest_mirror_snapshot "${local_cluster}" "${pool}" "${image}" "${local_status_log}" + local primary_snapshot_id=$(xmlstarlet sel -t -v "//snapshot/namespace/primary_snap_id" < ${local_status_log}) + + test "${snapshot_id}" = "${primary_snapshot_id}" && return 0 + done + + return 1 + done + return 1 +} + +wait_for_replay_complete() +{ + local local_cluster=$1 + local cluster=$2 + local pool=$3 + local image=$4 + + if [ "${MIRROR_IMAGE_MODE}" = "journal" ]; then + wait_for_journal_replay_complete ${local_cluster} ${cluster} ${pool} ${image} + elif [ "${MIRROR_IMAGE_MODE}" = "snapshot" ]; then + wait_for_snapshot_sync_complete ${local_cluster} ${cluster} ${pool} ${image} + else + return 1 + fi +} + + +test_status_in_pool_dir() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state_pattern="$4" + local description_pattern="$5" + local service_pattern="$6" + + local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}-${image}.mirror_status) + CEPH_ARGS='' rbd --cluster ${cluster} mirror image status ${pool}/${image} | + tee ${status_log} >&2 + grep "^ state: .*${state_pattern}" ${status_log} || return 1 + grep "^ description: .*${description_pattern}" ${status_log} || return 1 + + if [ -n "${service_pattern}" ]; then + grep "service: *${service_pattern}" ${status_log} || return 1 + elif echo ${state_pattern} | grep '^up+'; then + grep "service: *${MIRROR_USER_ID_PREFIX}.* on " ${status_log} || return 1 + else + grep "service: " ${status_log} && return 1 + fi + + # recheck using `mirror pool status` command to stress test it. + + local last_update="$(sed -nEe 's/^ last_update: *(.*) *$/\1/p' ${status_log})" + test_mirror_pool_status_verbose \ + ${cluster} ${pool} ${image} "${state_pattern}" "${last_update}" && + return 0 + + echo "'mirror pool status' test failed" >&2 + exit 1 +} + +test_mirror_pool_status_verbose() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state_pattern="$4" + local prev_last_update="$5" + + local status_log=${TEMPDIR}/$(mkfname ${cluster}-${pool}.mirror_status) + + rbd --cluster ${cluster} mirror pool status ${pool} --verbose --format xml \ + > ${status_log} + + local last_update state + last_update=$($XMLSTARLET sel -t -v \ + "//images/image[name='${image}']/last_update" < ${status_log}) + state=$($XMLSTARLET sel -t -v \ + "//images/image[name='${image}']/state" < ${status_log}) + + echo "${state}" | grep "${state_pattern}" || + test "${last_update}" '>' "${prev_last_update}" +} + +wait_for_status_in_pool_dir() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state_pattern="$4" + local description_pattern="$5" + local service_pattern="$6" + + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do + sleep ${s} + test_status_in_pool_dir ${cluster} ${pool} ${image} "${state_pattern}" \ + "${description_pattern}" "${service_pattern}" && + return 0 + done + return 1 +} + +create_image() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local image=$1 ; shift + local size=128 + + if [ -n "$1" ]; then + size=$1 + shift + fi + + rbd --cluster ${cluster} create --size ${size} \ + --image-feature "${RBD_IMAGE_FEATURES}" $@ ${pool}/${image} +} + +create_image_and_enable_mirror() +{ + local cluster=$1 ; shift + local pool=$1 ; shift + local image=$1 ; shift + local mode=${1:-${MIRROR_IMAGE_MODE}} + if [ -n "$1" ]; then + shift + fi + + create_image ${cluster} ${pool} ${image} $@ + if [ "${MIRROR_POOL_MODE}" = "image" ] || [ "$pool" = "${PARENT_POOL}" ]; then + enable_mirror ${cluster} ${pool} ${image} ${mode} + fi +} + +enable_journaling() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} feature enable ${pool}/${image} journaling +} + +set_image_meta() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local key=$4 + local val=$5 + + rbd --cluster ${cluster} image-meta set ${pool}/${image} $key $val +} + +compare_image_meta() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local key=$4 + local value=$5 + + test `rbd --cluster ${cluster} image-meta get ${pool}/${image} ${key}` = "${value}" +} + +rename_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local new_name=$4 + + rbd --cluster=${cluster} rename ${pool}/${image} ${pool}/${new_name} +} + +remove_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} snap purge ${pool}/${image} + rbd --cluster=${cluster} rm ${pool}/${image} +} + +remove_image_retry() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + for s in 0 1 2 4 8 16 32; do + sleep ${s} + remove_image ${cluster} ${pool} ${image} && return 0 + done + return 1 +} + +trash_move() { + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} trash move ${pool}/${image} +} + +trash_restore() { + local cluster=$1 + local pool=$2 + local image_id=$3 + + rbd --cluster=${cluster} trash restore ${pool}/${image_id} +} + +clone_image() +{ + local cluster=$1 + local parent_pool=$2 + local parent_image=$3 + local parent_snap=$4 + local clone_pool=$5 + local clone_image=$6 + + shift 6 + + rbd --cluster ${cluster} clone \ + ${parent_pool}/${parent_image}@${parent_snap} \ + ${clone_pool}/${clone_image} --image-feature "${RBD_IMAGE_FEATURES}" $@ +} + +clone_image_and_enable_mirror() +{ + local cluster=$1 + local parent_pool=$2 + local parent_image=$3 + local parent_snap=$4 + local clone_pool=$5 + local clone_image=$6 + shift 6 + + local mode=${1:-${MIRROR_IMAGE_MODE}} + if [ -n "$1" ]; then + shift + fi + + clone_image ${cluster} ${parent_pool} ${parent_image} ${parent_snap} ${clone_pool} ${clone_image} $@ + enable_mirror ${cluster} ${clone_pool} ${clone_image} ${mode} +} + +disconnect_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} journal client disconnect \ + --image ${pool}/${image} +} + +create_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + rbd --cluster ${cluster} snap create ${pool}/${image}@${snap} +} + +remove_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + rbd --cluster ${cluster} snap rm ${pool}/${image}@${snap} +} + +rename_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + local new_snap=$5 + + rbd --cluster ${cluster} snap rename ${pool}/${image}@${snap} \ + ${pool}/${image}@${new_snap} +} + +purge_snapshots() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} snap purge ${pool}/${image} +} + +protect_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + rbd --cluster ${cluster} snap protect ${pool}/${image}@${snap} +} + +unprotect_snapshot() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + rbd --cluster ${cluster} snap unprotect ${pool}/${image}@${snap} +} + +unprotect_snapshot_retry() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + for s in 0 1 2 4 8 16 32; do + sleep ${s} + unprotect_snapshot ${cluster} ${pool} ${image} ${snap} && return 0 + done + return 1 +} + +wait_for_snap_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap_name=$4 + local s + + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do + sleep ${s} + rbd --cluster ${cluster} info ${pool}/${image}@${snap_name} || continue + return 0 + done + return 1 +} + +test_snap_moved_to_trash() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap_name=$4 + + rbd --cluster ${cluster} snap ls ${pool}/${image} --all | + grep -F " trash (${snap_name})" +} + +wait_for_snap_moved_to_trash() +{ + local s + + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do + sleep ${s} + test_snap_moved_to_trash $@ || continue + return 0 + done + return 1 +} + +test_snap_removed_from_trash() +{ + test_snap_moved_to_trash $@ && return 1 + return 0 +} + +wait_for_snap_removed_from_trash() +{ + local s + + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do + sleep ${s} + test_snap_removed_from_trash $@ || continue + return 0 + done + return 1 +} + +count_mirror_snaps() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} snap ls ${pool}/${image} --all | + grep -c -F " mirror (" +} + +write_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local count=$4 + local size=$5 + + test -n "${size}" || size=4096 + + rbd --cluster ${cluster} bench ${pool}/${image} --io-type write \ + --io-size ${size} --io-threads 1 --io-total $((size * count)) \ + --io-pattern rand +} + +stress_write_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local duration=$(awk 'BEGIN {srand(); print int(10 * rand()) + 5}') + + set +e + timeout ${duration}s ceph_test_rbd_mirror_random_write \ + --cluster ${cluster} ${pool} ${image} \ + --debug-rbd=20 --debug-journaler=20 \ + 2> ${TEMPDIR}/rbd-mirror-random-write.log + error_code=$? + set -e + + if [ $error_code -eq 124 ]; then + return 0 + fi + return 1 +} + +show_diff() +{ + local file1=$1 + local file2=$2 + + xxd ${file1} > ${file1}.xxd + xxd ${file2} > ${file2}.xxd + sdiff -s ${file1}.xxd ${file2}.xxd | head -n 64 + rm -f ${file1}.xxd ${file2}.xxd +} + +compare_images() +{ + local pool=$1 + local image=$2 + local ret=0 + + local rmt_export=${TEMPDIR}/$(mkfname ${CLUSTER2}-${pool}-${image}.export) + local loc_export=${TEMPDIR}/$(mkfname ${CLUSTER1}-${pool}-${image}.export) + + rm -f ${rmt_export} ${loc_export} + rbd --cluster ${CLUSTER2} export ${pool}/${image} ${rmt_export} + rbd --cluster ${CLUSTER1} export ${pool}/${image} ${loc_export} + if ! cmp ${rmt_export} ${loc_export} + then + show_diff ${rmt_export} ${loc_export} + ret=1 + fi + rm -f ${rmt_export} ${loc_export} + return ${ret} +} + +compare_image_snapshots() +{ + local pool=$1 + local image=$2 + local ret=0 + + local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export + local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export + + for snap_name in $(rbd --cluster ${CLUSTER1} --format xml \ + snap list ${pool}/${image} | \ + $XMLSTARLET sel -t -v "//snapshot/name" | \ + grep -E -v "^\.rbd-mirror\."); do + rm -f ${rmt_export} ${loc_export} + rbd --cluster ${CLUSTER2} export ${pool}/${image}@${snap_name} ${rmt_export} + rbd --cluster ${CLUSTER1} export ${pool}/${image}@${snap_name} ${loc_export} + if ! cmp ${rmt_export} ${loc_export} + then + show_diff ${rmt_export} ${loc_export} + ret=1 + fi + done + rm -f ${rmt_export} ${loc_export} + return ${ret} +} + +demote_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} mirror image demote ${pool}/${image} +} + +promote_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local force=$4 + + rbd --cluster=${cluster} mirror image promote ${pool}/${image} ${force} +} + +set_pool_mirror_mode() +{ + local cluster=$1 + local pool=$2 + local mode=${3:-${MIRROR_POOL_MODE}} + + rbd --cluster=${cluster} mirror pool enable ${pool} ${mode} +} + +disable_mirror() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} mirror image disable ${pool}/${image} +} + +enable_mirror() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local mode=${4:-${MIRROR_IMAGE_MODE}} + + rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode} + # Display image info including the global image id for debugging purpose + rbd --cluster=${cluster} info ${pool}/${image} +} + +test_image_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local test_state=$4 + local image_id=$5 + local current_state=deleted + local current_image_id + + current_image_id=$(get_image_id ${cluster} ${pool} ${image}) + test -n "${current_image_id}" && + test -z "${image_id}" -o "${image_id}" = "${current_image_id}" && + current_state=present + + test "${test_state}" = "${current_state}" +} + +wait_for_image_present() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local state=$4 + local image_id=$5 + local s + + test -n "${image_id}" || + image_id=$(get_image_id ${cluster} ${pool} ${image}) + + # TODO: add a way to force rbd-mirror to update replayers + for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do + sleep ${s} + test_image_present \ + "${cluster}" "${pool}" "${image}" "${state}" "${image_id}" && + return 0 + done + return 1 +} + +get_image_id() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster=${cluster} info ${pool}/${image} | + sed -ne 's/^.*block_name_prefix: rbd_data\.//p' +} + +request_resync_image() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local image_id_var_name=$4 + + eval "${image_id_var_name}='$(get_image_id ${cluster} ${pool} ${image})'" + eval 'test -n "$'${image_id_var_name}'"' + + rbd --cluster=${cluster} mirror image resync ${pool}/${image} +} + +get_image_data_pool() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} info ${pool}/${image} | + awk '$1 == "data_pool:" {print $2}' +} + +get_clone_format() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + rbd --cluster ${cluster} info ${pool}/${image} | + awk 'BEGIN { + format = 1 + } + $1 == "parent:" { + parent = $2 + } + /op_features: .*clone-child/ { + format = 2 + } + END { + if (!parent) exit 1 + print format + }' +} + +list_omap_keys() +{ + local cluster=$1 + local pool=$2 + local obj_name=$3 + + rados --cluster ${cluster} -p ${pool} listomapkeys ${obj_name} +} + +count_omap_keys_with_filter() +{ + local cluster=$1 + local pool=$2 + local obj_name=$3 + local filter=$4 + + list_omap_keys ${cluster} ${pool} ${obj_name} | grep -c ${filter} +} + +wait_for_omap_keys() +{ + local cluster=$1 + local pool=$2 + local obj_name=$3 + local filter=$4 + + for s in 0 1 2 2 4 4 8 8 8 16 16 32; do + sleep $s + + set +e + test "$(count_omap_keys_with_filter ${cluster} ${pool} ${obj_name} ${filter})" = 0 + error_code=$? + set -e + + if [ $error_code -eq 0 ]; then + return 0 + fi + done + + return 1 +} + +wait_for_image_in_omap() +{ + local cluster=$1 + local pool=$2 + + wait_for_omap_keys ${cluster} ${pool} rbd_mirroring status_global + wait_for_omap_keys ${cluster} ${pool} rbd_mirroring image_ + wait_for_omap_keys ${cluster} ${pool} rbd_mirror_leader image_map +} + +# +# Main +# + +if [ "$#" -gt 0 ] +then + if [ -z "${RBD_MIRROR_TEMDIR}" ] + then + echo "RBD_MIRROR_TEMDIR is not set" >&2 + exit 1 + fi + + TEMPDIR="${RBD_MIRROR_TEMDIR}" + cd ${TEMPDIR} + $@ + exit $? +fi diff --git a/qa/workunits/rbd/rbd_mirror_journal.sh b/qa/workunits/rbd/rbd_mirror_journal.sh new file mode 100755 index 000000000..54f6aeec8 --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_journal.sh @@ -0,0 +1,614 @@ +#!/bin/sh -ex +# +# rbd_mirror_journal.sh - test rbd-mirror daemon in journal-based mirroring mode +# +# The scripts starts two ("local" and "remote") clusters using mstart.sh script, +# creates a temporary directory, used for cluster configs, daemon logs, admin +# socket, temporary files, and launches rbd-mirror daemon. +# + +. $(dirname $0)/rbd_mirror_helpers.sh + +setup + +testlog "TEST: add image and test replay" +start_mirrors ${CLUSTER1} +image=test +create_image ${CLUSTER2} ${POOL} ${image} +set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1" +set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2" +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown' +fi +compare_images ${POOL} ${image} +compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1" +compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2" + +testlog "TEST: stop mirror, add image, start mirror and test replay" +stop_mirrors ${CLUSTER1} +image1=test1 +create_image ${CLUSTER2} ${POOL} ${image1} +write_image ${CLUSTER2} ${POOL} ${image1} 100 +start_mirrors ${CLUSTER1} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'primary_position' +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown' +fi +compare_images ${POOL} ${image1} + +testlog "TEST: test the first image is replaying after restart" +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +compare_images ${POOL} ${image} + +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + testlog "TEST: stop/start/restart mirror via admin socket" + all_admin_daemons ${CLUSTER1} rbd mirror stop + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror start + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror restart + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror stop + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror restart + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + + all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + flush ${CLUSTER1} + all_admin_daemons ${CLUSTER1} rbd mirror status +fi + +remove_image_retry ${CLUSTER2} ${POOL} ${image1} + +testlog "TEST: test image rename" +new_name="${image}_RENAMED" +rename_image ${CLUSTER2} ${POOL} ${image} ${new_name} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' +admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name} +admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' +rename_image ${CLUSTER2} ${POOL} ${new_name} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +testlog "TEST: test trash move restore" +image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image}) +trash_move ${CLUSTER2} ${POOL} ${image} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' +trash_restore ${CLUSTER2} ${POOL} ${image_id} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +testlog "TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)" +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} + +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + +testlog "TEST: failover and failback" +start_mirrors ${CLUSTER2} + +# demote and promote same cluster +demote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +compare_images ${POOL} ${image} + +# failover (unmodified) +demote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} + +# failback (unmodified) +demote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' +compare_images ${POOL} ${image} + +# failover +demote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} +write_image ${CLUSTER1} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'primary_position' +compare_images ${POOL} ${image} + +# failback +demote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' +compare_images ${POOL} ${image} + +testlog "TEST: failover / failback loop" +for i in `seq 1 20`; do + demote_image ${CLUSTER2} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' + promote_image ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' + demote_image ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' + promote_image ${CLUSTER2} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +done + +testlog "TEST: force promote" +force_promote_image=test_force_promote +create_image ${CLUSTER2} ${POOL} ${force_promote_image} +write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'primary_position' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' +promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force' +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image} +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' +write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100 +write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 +remove_image_retry ${CLUSTER1} ${POOL} ${force_promote_image} +remove_image_retry ${CLUSTER2} ${POOL} ${force_promote_image} + +testlog "TEST: cloned images" +testlog " - default" +parent_image=test_parent +parent_snap=snap +create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} +write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100 +create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} + +clone_image=test_clone +clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image} +write_image ${CLUSTER2} ${POOL} ${clone_image} 100 + +enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} journal +wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'primary_position' +compare_images ${PARENT_POOL} ${parent_image} + +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'primary_position' +compare_images ${POOL} ${clone_image} +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image} + +testlog " - clone v1" +clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}1 + +clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \ + ${clone_image}_v1 --rbd-default-clone-format 1 +test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1 +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1 +test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1 +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v1 +remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}1 +unprotect_snapshot_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} + +testlog " - clone v2" +parent_snap=snap_v2 +create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \ + ${clone_image}_v2 --rbd-default-clone-format 2 +test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2 +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2 +test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2 + +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2 +wait_for_image_present ${CLUSTER1} ${POOL} ${clone_image}_v2 'deleted' +test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} + +testlog " - clone v2 non-primary" +create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \ + ${clone_image}_v2 --rbd-default-clone-format 2 +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2 +wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} + +testlog "TEST: data pool" +dp_image=test_data_pool +create_image ${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL} +data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image} +data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1' +write_image ${CLUSTER2} ${POOL} ${dp_image} 100 +create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2' +write_image ${CLUSTER2} ${POOL} ${dp_image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'primary_position' +compare_images ${POOL} ${dp_image}@snap1 +compare_images ${POOL} ${dp_image}@snap2 +compare_images ${POOL} ${dp_image} +remove_image_retry ${CLUSTER2} ${POOL} ${dp_image} + +testlog "TEST: disable mirroring / delete non-primary image" +image2=test2 +image3=test3 +image4=test4 +image5=test5 +for i in ${image2} ${image3} ${image4} ${image5}; do + create_image ${CLUSTER2} ${POOL} ${i} + write_image ${CLUSTER2} ${POOL} ${i} 100 + create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' + create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' + if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then + protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' + protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' + fi + write_image ${CLUSTER2} ${POOL} ${i} 100 + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present' + wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2' +done + +set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' +for i in ${image2} ${image4}; do + disable_mirror ${CLUSTER2} ${POOL} ${i} +done + +unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1' +unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2' +for i in ${image3} ${image5}; do + remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' + remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' + remove_image_retry ${CLUSTER2} ${POOL} ${i} +done + +for i in ${image2} ${image3} ${image4} ${image5}; do + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' +done + +set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' +for i in ${image2} ${image4}; do + enable_journaling ${CLUSTER2} ${POOL} ${i} + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present' + wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2' + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i} + compare_images ${POOL} ${i} +done + +testlog "TEST: remove mirroring pool" +pool=pool_to_remove +for cluster in ${CLUSTER1} ${CLUSTER2}; do + CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${pool} 16 16 + CEPH_ARGS='' rbd --cluster ${cluster} pool init ${pool} + rbd --cluster ${cluster} mirror pool enable ${pool} pool +done +peer_add ${CLUSTER1} ${pool} ${CLUSTER2} +peer_add ${CLUSTER2} ${pool} ${CLUSTER1} +rdp_image=test_remove_data_pool +create_image ${CLUSTER2} ${pool} ${image} 128 +create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool} +write_image ${CLUSTER2} ${pool} ${image} 100 +write_image ${CLUSTER2} ${POOL} ${rdp_image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${pool} ${image} 'up+replaying' 'primary_position' +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'primary_position' +for cluster in ${CLUSTER1} ${CLUSTER2}; do + CEPH_ARGS='' ceph --cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it +done +remove_image_retry ${CLUSTER2} ${POOL} ${rdp_image} +wait_for_image_present ${CLUSTER1} ${POOL} ${rdp_image} 'deleted' +for i in 0 1 2 4 8 8 8 8 16 16; do + sleep $i + admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} || break +done +admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false + +testlog "TEST: snapshot rename" +snap_name='snap_rename' +create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0" +for i in `seq 1 20`; do + rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}" +done +wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}" + +unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1' +unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2' +for i in ${image2} ${image4}; do + remove_image_retry ${CLUSTER2} ${POOL} ${i} +done + +testlog "TEST: disable mirror while daemon is stopped" +stop_mirrors ${CLUSTER1} +stop_mirrors ${CLUSTER2} +set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' +disable_mirror ${CLUSTER2} ${POOL} ${image} +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + test_image_present ${CLUSTER1} ${POOL} ${image} 'present' +fi +start_mirrors ${CLUSTER1} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' +set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' +enable_journaling ${CLUSTER2} ${POOL} ${image} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +testlog "TEST: non-default namespace image mirroring" +testlog " - replay" +create_image ${CLUSTER2} ${POOL}/${NS1} ${image} +create_image ${CLUSTER2} ${POOL}/${NS2} ${image} +enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} journal +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image} +write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 +write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' 'primary_position' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' 'primary_position' +compare_images ${POOL}/${NS1} ${image} +compare_images ${POOL}/${NS2} ${image} + +testlog " - disable mirroring / delete image" +remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} +disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' +wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted' +remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image} + +testlog " - data pool" +dp_image=test_data_pool +create_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 128 --data-pool ${PARENT_POOL} +data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image} +data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'primary_position' +compare_images ${POOL}/${NS1} ${dp_image} +remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image} + +testlog "TEST: simple image resync" +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +compare_images ${POOL} ${image} + +testlog "TEST: image resync while replayer is stopped" +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + request_resync_image ${CLUSTER1} ${POOL} ${image} image_id + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' + compare_images ${POOL} ${image} +fi + +testlog "TEST: request image resync while daemon is offline" +stop_mirrors ${CLUSTER1} +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +start_mirrors ${CLUSTER1} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +compare_images ${POOL} ${image} +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +testlog "TEST: client disconnect" +image=laggy +create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K +write_image ${CLUSTER2} ${POOL} ${image} 10 + +testlog " - replay stopped after disconnect" +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" +disconnect_image ${CLUSTER2} ${POOL} ${image} +test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' + +testlog " - replay started after resync requested" +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" +compare_images ${POOL} ${image} + +testlog " - disconnected after max_concurrent_object_sets reached" +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" + set_image_meta ${CLUSTER2} ${POOL} ${image} \ + conf_rbd_journal_max_concurrent_object_sets 1 + write_image ${CLUSTER2} ${POOL} ${image} 20 16384 + write_image ${CLUSTER2} ${POOL} ${image} 20 16384 + test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" + set_image_meta ${CLUSTER2} ${POOL} ${image} \ + conf_rbd_journal_max_concurrent_object_sets 0 + + testlog " - replay is still stopped (disconnected) after restart" + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' +fi + +testlog " - replay started after resync requested" +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" +compare_images ${POOL} ${image} + +testlog " - rbd_mirroring_resync_after_disconnect config option" +set_image_meta ${CLUSTER2} ${POOL} ${image} \ + conf_rbd_mirroring_resync_after_disconnect true +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image}) +disconnect_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" +compare_images ${POOL} ${image} +set_image_meta ${CLUSTER2} ${POOL} ${image} \ + conf_rbd_mirroring_resync_after_disconnect false +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +disconnect_image ${CLUSTER2} ${POOL} ${image} +test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +testlog "TEST: split-brain" +image=split-brain +create_image ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +promote_image ${CLUSTER1} ${POOL} ${image} --force +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' +write_image ${CLUSTER1} ${POOL} ${image} 10 +demote_image ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain' +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +testlog "TEST: check if removed images' OMAP are removed" +start_mirrors ${CLUSTER2} +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} + +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + # teuthology will trash the daemon + testlog "TEST: no blocklists" + CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blocklist ls 2>&1 | grep -q "listed 0 entries" + CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blocklist ls 2>&1 | grep -q "listed 0 entries" +fi diff --git a/qa/workunits/rbd/rbd_mirror_snapshot.sh b/qa/workunits/rbd/rbd_mirror_snapshot.sh new file mode 100755 index 000000000..c70d48b09 --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_snapshot.sh @@ -0,0 +1,517 @@ +#!/bin/sh -ex +# +# rbd_mirror_snapshot.sh - test rbd-mirror daemon in snapshot-based mirroring mode +# +# The scripts starts two ("local" and "remote") clusters using mstart.sh script, +# creates a temporary directory, used for cluster configs, daemon logs, admin +# socket, temporary files, and launches rbd-mirror daemon. +# + +MIRROR_POOL_MODE=image +MIRROR_IMAGE_MODE=snapshot + +. $(dirname $0)/rbd_mirror_helpers.sh + +setup + +testlog "TEST: add image and test replay" +start_mirrors ${CLUSTER1} +image=test +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} +set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1" +set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2" +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown' +fi +compare_images ${POOL} ${image} +compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1" +compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2" + +testlog "TEST: stop mirror, add image, start mirror and test replay" +stop_mirrors ${CLUSTER1} +image1=test1 +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image1} +write_image ${CLUSTER2} ${POOL} ${image1} 100 +start_mirrors ${CLUSTER1} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown' +fi +compare_images ${POOL} ${image1} + +testlog "TEST: test the first image is replaying after restart" +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} + +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + testlog "TEST: stop/start/restart mirror via admin socket" + all_admin_daemons ${CLUSTER1} rbd mirror stop + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror start + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror restart + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror stop + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror restart + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + + all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + + all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' + + all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' + + flush ${CLUSTER1} + all_admin_daemons ${CLUSTER1} rbd mirror status +fi + +remove_image_retry ${CLUSTER2} ${POOL} ${image1} + +testlog "TEST: test image rename" +new_name="${image}_RENAMED" +rename_image ${CLUSTER2} ${POOL} ${image} ${new_name} +mirror_image_snapshot ${CLUSTER2} ${POOL} ${new_name} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' +admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name} +admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' +rename_image ${CLUSTER2} ${POOL} ${new_name} ${image} +mirror_image_snapshot ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +testlog "TEST: test trash move restore" +image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image}) +trash_move ${CLUSTER2} ${POOL} ${image} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' +trash_restore ${CLUSTER2} ${POOL} ${image_id} +enable_mirror ${CLUSTER2} ${POOL} ${image} snapshot +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +testlog "TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)" +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} + +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + +testlog "TEST: failover and failback" +start_mirrors ${CLUSTER2} + +# demote and promote same cluster +demote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} + +# failover (unmodified) +demote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} + +# failback (unmodified) +demote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' +compare_images ${POOL} ${image} + +# failover +demote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} +write_image ${CLUSTER1} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} + +# failback +demote_image ${CLUSTER1} ${POOL} ${image} +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' +promote_image ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' +compare_images ${POOL} ${image} + +testlog "TEST: failover / failback loop" +for i in `seq 1 20`; do + demote_image ${CLUSTER2} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' + promote_image ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' + demote_image ${CLUSTER1} ${POOL} ${image} + wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' + promote_image ${CLUSTER2} ${POOL} ${image} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +done +# check that demote (or other mirror snapshots) don't pile up +test "$(count_mirror_snaps ${CLUSTER1} ${POOL} ${image})" -le 3 +test "$(count_mirror_snaps ${CLUSTER2} ${POOL} ${image})" -le 3 + +testlog "TEST: force promote" +force_promote_image=test_force_promote +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image} +write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' +promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force' +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image} +wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped' +wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' +write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100 +write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 +remove_image_retry ${CLUSTER1} ${POOL} ${force_promote_image} +remove_image_retry ${CLUSTER2} ${POOL} ${force_promote_image} + +testlog "TEST: cloned images" +testlog " - default" +parent_image=test_parent +parent_snap=snap +create_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} +write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100 +create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} + +clone_image=test_clone +clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image} +write_image ${CLUSTER2} ${POOL} ${clone_image} 100 +enable_mirror ${CLUSTER2} ${POOL} ${clone_image} snapshot + +wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' +compare_images ${PARENT_POOL} ${parent_image} + +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' +compare_images ${POOL} ${clone_image} +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image} + +testlog " - clone v1" +clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} ${parent_image} \ + ${parent_snap} ${POOL} ${clone_image}1 + +clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} \ + ${parent_snap} ${POOL} ${clone_image}_v1 snapshot --rbd-default-clone-format 1 +test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1 +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1 +test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1 +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v1 +remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}1 +unprotect_snapshot_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} + +testlog " - clone v2" +parent_snap=snap_v2 +create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} \ + ${parent_snap} ${POOL} ${clone_image}_v2 snapshot --rbd-default-clone-format 2 +test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2 +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2 +test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2 + +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2 +wait_for_image_present ${CLUSTER1} ${POOL} ${clone_image}_v2 'deleted' +test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} + +testlog " - clone v2 non-primary" +create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} ${parent_image} \ + ${parent_snap} ${POOL} ${clone_image}_v2 snapshot --rbd-default-clone-format 2 +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2 +wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} + +testlog "TEST: data pool" +dp_image=test_data_pool +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${dp_image} snapshot 128 --data-pool ${PARENT_POOL} +data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image} +data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1' +write_image ${CLUSTER2} ${POOL} ${dp_image} 100 +create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2' +write_image ${CLUSTER2} ${POOL} ${dp_image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' +compare_images ${POOL} ${dp_image}@snap1 +compare_images ${POOL} ${dp_image}@snap2 +compare_images ${POOL} ${dp_image} +remove_image_retry ${CLUSTER2} ${POOL} ${dp_image} + +testlog "TEST: disable mirroring / delete non-primary image" +image2=test2 +image3=test3 +image4=test4 +image5=test5 +for i in ${image2} ${image3} ${image4} ${image5}; do + create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${i} + write_image ${CLUSTER2} ${POOL} ${i} 100 + create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' + create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' + if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then + protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' + protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' + fi + write_image ${CLUSTER2} ${POOL} ${i} 100 + mirror_image_snapshot ${CLUSTER2} ${POOL} ${i} + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present' + wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2' +done + +set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' +for i in ${image2} ${image4}; do + disable_mirror ${CLUSTER2} ${POOL} ${i} +done + +unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1' +unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2' +for i in ${image3} ${image5}; do + remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' + remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' + remove_image_retry ${CLUSTER2} ${POOL} ${i} +done + +for i in ${image2} ${image3} ${image4} ${image5}; do + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' +done + +testlog "TEST: snapshot rename" +snap_name='snap_rename' +enable_mirror ${CLUSTER2} ${POOL} ${image2} +create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0" +for i in `seq 1 20`; do + rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}" +done +mirror_image_snapshot ${CLUSTER2} ${POOL} ${image2} +wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}" + +unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1' +unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2' +for i in ${image2} ${image4}; do + remove_image_retry ${CLUSTER2} ${POOL} ${i} +done + +testlog "TEST: disable mirror while daemon is stopped" +stop_mirrors ${CLUSTER1} +stop_mirrors ${CLUSTER2} +disable_mirror ${CLUSTER2} ${POOL} ${image} +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + test_image_present ${CLUSTER1} ${POOL} ${image} 'present' +fi +start_mirrors ${CLUSTER1} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' +enable_mirror ${CLUSTER2} ${POOL} ${image} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +testlog "TEST: non-default namespace image mirroring" +testlog " - replay" +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${image} +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image} +write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 +write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' +compare_images ${POOL}/${NS1} ${image} +compare_images ${POOL}/${NS2} ${image} + +testlog " - disable mirroring / delete image" +remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} +disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} +wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' +wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted' +remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image} + +testlog " - data pool" +dp_image=test_data_pool +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${dp_image} snapshot 128 --data-pool ${PARENT_POOL} +data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image} +data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image}) +test "${data_pool}" = "${PARENT_POOL}" +write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' +compare_images ${POOL}/${NS1} ${dp_image} +remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image} + +testlog "TEST: simple image resync" +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} + +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + testlog "TEST: image resync while replayer is stopped" + admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + request_resync_image ${CLUSTER1} ${POOL} ${image} image_id + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + compare_images ${POOL} ${image} +fi + +testlog "TEST: request image resync while daemon is offline" +stop_mirrors ${CLUSTER1} +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +start_mirrors ${CLUSTER1} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +compare_images ${POOL} ${image} +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +testlog "TEST: split-brain" +image=split-brain +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +promote_image ${CLUSTER1} ${POOL} ${image} --force +wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' +write_image ${CLUSTER1} ${POOL} ${image} 10 +demote_image ${CLUSTER1} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain' +request_resync_image ${CLUSTER1} ${POOL} ${image} image_id +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +testlog "TEST: check if removed images' OMAP are removed" +start_mirrors ${CLUSTER2} +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} + +if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + # teuthology will trash the daemon + testlog "TEST: no blocklists" + CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blocklist ls 2>&1 | grep -q "listed 0 entries" + CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blocklist ls 2>&1 | grep -q "listed 0 entries" +fi diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh new file mode 100755 index 000000000..cb79aba7e --- /dev/null +++ b/qa/workunits/rbd/rbd_mirror_stress.sh @@ -0,0 +1,221 @@ +#!/bin/sh -ex +# +# rbd_mirror_stress.sh - stress test rbd-mirror daemon +# +# The following additional environment variables affect the test: +# +# RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write +# tool during the many image test +# + +IMAGE_COUNT=50 +export LOCKDEP=0 + +. $(dirname $0)/rbd_mirror_helpers.sh + +setup + +create_snap() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap_name=$4 + + rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap_name} \ + --debug-rbd=20 --debug-journaler=20 2> ${TEMPDIR}/rbd-snap-create.log +} + +compare_image_snaps() +{ + local pool=$1 + local image=$2 + local snap_name=$3 + local ret=0 + + local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export + local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export + + rm -f ${rmt_export} ${loc_export} + rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} ${rmt_export} + rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} ${loc_export} + if ! cmp ${rmt_export} ${loc_export} + then + show_diff ${rmt_export} ${loc_export} + ret=1 + fi + rm -f ${rmt_export} ${loc_export} + return ${ret} +} + +wait_for_pool_images() +{ + local cluster=$1 + local pool=$2 + local image_count=$3 + local s + local count + local last_count=0 + + while true; do + for s in `seq 1 40`; do + test $s -ne 1 && sleep 30 + count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2) + test "${count}" = "${image_count}" && return 0 + + # reset timeout if making forward progress + test $count -ne $last_count && break + done + + test $count -eq $last_count && break + last_count=$count + done + rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2 + return 1 +} + +wait_for_pool_healthy() +{ + local cluster=$1 + local pool=$2 + local s + local state + + for s in `seq 1 40`; do + test $s -ne 1 && sleep 30 + state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'image health:' | cut -d' ' -f 3) + test "${state}" = "ERROR" && break + test "${state}" = "OK" && return 0 + done + rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2 + return 1 +} + +start_mirrors ${CLUSTER1} +start_mirrors ${CLUSTER2} + +testlog "TEST: add image and test replay after client crashes" +image=test +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${MIRROR_IMAGE_MODE} '512M' +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + +clean_snap_name= +for i in `seq 1 10` +do + stress_write_image ${CLUSTER2} ${POOL} ${image} + + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + + snap_name="snap${i}" + create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name} + + if [ -n "${clean_snap_name}" ]; then + compare_image_snaps ${POOL} ${image} ${clean_snap_name} + fi + compare_image_snaps ${POOL} ${image} ${snap_name} + + clean_snap_name="snap${i}-clean" + create_snap ${CLUSTER2} ${POOL} ${image} ${clean_snap_name} +done + +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${clean_snap_name} + +for i in `seq 1 10` +do + snap_name="snap${i}" + compare_image_snaps ${POOL} ${image} ${snap_name} + + snap_name="snap${i}-clean" + compare_image_snaps ${POOL} ${image} ${snap_name} +done + +for i in `seq 1 10` +do + snap_name="snap${i}" + remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name} + + snap_name="snap${i}-clean" + remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name} +done + +remove_image_retry ${CLUSTER2} ${POOL} ${image} +wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' + +testlog "TEST: create many images" +snap_name="snap" +for i in `seq 1 ${IMAGE_COUNT}` +do + image="image_${i}" + create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${MIRROR_IMAGE_MODE} '128M' + if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then + write_image ${CLUSTER2} ${POOL} ${image} 100 + else + stress_write_image ${CLUSTER2} ${POOL} ${image} + fi +done + +wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT} +wait_for_pool_healthy ${CLUSTER2} ${POOL} + +wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT} +wait_for_pool_healthy ${CLUSTER1} ${POOL} + +testlog "TEST: compare many images" +for i in `seq 1 ${IMAGE_COUNT}` +do + image="image_${i}" + create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name} + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name} + compare_image_snaps ${POOL} ${image} ${snap_name} +done + +testlog "TEST: delete many images" +for i in `seq 1 ${IMAGE_COUNT}` +do + image="image_${i}" + remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name} + remove_image_retry ${CLUSTER2} ${POOL} ${image} +done + +testlog "TEST: image deletions should propagate" +wait_for_pool_images ${CLUSTER1} ${POOL} 0 +wait_for_pool_healthy ${CLUSTER1} ${POOL} 0 +for i in `seq 1 ${IMAGE_COUNT}` +do + image="image_${i}" + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' +done + +testlog "TEST: delete images during bootstrap" +set_pool_mirror_mode ${CLUSTER1} ${POOL} 'image' +set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' + +start_mirror ${CLUSTER1} +image=test + +for i in `seq 1 10` +do + image="image_${i}" + create_image ${CLUSTER2} ${POOL} ${image} '512M' + enable_mirror ${CLUSTER2} ${POOL} ${image} + + stress_write_image ${CLUSTER2} ${POOL} ${image} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' + + disable_mirror ${CLUSTER2} ${POOL} ${image} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' + purge_snapshots ${CLUSTER2} ${POOL} ${image} + remove_image_retry ${CLUSTER2} ${POOL} ${image} +done + +testlog "TEST: check if removed images' OMAP are removed" + +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} diff --git a/qa/workunits/rbd/rbd_support_module_recovery.sh b/qa/workunits/rbd/rbd_support_module_recovery.sh new file mode 100755 index 000000000..e9defced2 --- /dev/null +++ b/qa/workunits/rbd/rbd_support_module_recovery.sh @@ -0,0 +1,77 @@ +#!/bin/bash +set -ex + +POOL=rbd +IMAGE_PREFIX=image +NUM_IMAGES=20 +RUN_TIME=3600 + +rbd mirror pool enable ${POOL} image +rbd mirror pool peer add ${POOL} dummy + +# Create images and schedule their mirror snapshots +for ((i = 1; i <= ${NUM_IMAGES}; i++)); do + rbd create -s 1G --image-feature exclusive-lock ${POOL}/${IMAGE_PREFIX}$i + rbd mirror image enable ${POOL}/${IMAGE_PREFIX}$i snapshot + rbd mirror snapshot schedule add -p ${POOL} --image ${IMAGE_PREFIX}$i 1m +done + +# Run fio workloads on images via kclient +# Test the recovery of the rbd_support module and its scheduler from their +# librbd client being blocklisted while a exclusive lock gets passed around +# between their librbd client and a kclient trying to take mirror snapshots +# and perform I/O on the same image. +for ((i = 1; i <= ${NUM_IMAGES}; i++)); do + DEVS[$i]=$(sudo rbd device map ${POOL}/${IMAGE_PREFIX}$i) + fio --name=fiotest --filename=${DEVS[$i]} --rw=randrw --bs=4K --direct=1 \ + --ioengine=libaio --iodepth=2 --runtime=43200 --time_based \ + &> /dev/null & +done + +# Repeatedly blocklist rbd_support module's client ~10s after the module +# recovers from previous blocklisting +CURRENT_TIME=$(date +%s) +END_TIME=$((CURRENT_TIME + RUN_TIME)) +PREV_CLIENT_ADDR="" +CLIENT_ADDR="" +while ((CURRENT_TIME <= END_TIME)); do + if [[ -n "${CLIENT_ADDR}" ]] && + [[ "${CLIENT_ADDR}" != "${PREV_CLIENT_ADDR}" ]]; then + ceph osd blocklist add ${CLIENT_ADDR} + # Confirm rbd_support module's client is blocklisted + ceph osd blocklist ls | grep -q ${CLIENT_ADDR} + PREV_CLIENT_ADDR=${CLIENT_ADDR} + fi + sleep 10 + CLIENT_ADDR=$(ceph mgr dump | + jq .active_clients[] | + jq 'select(.name == "rbd_support")' | + jq -r '[.addrvec[0].addr, "/", .addrvec[0].nonce|tostring] | add') + CURRENT_TIME=$(date +%s) +done + +# Confirm that rbd_support module recovered from repeated blocklisting +# Check that you can add a mirror snapshot schedule after a few retries +for ((i = 1; i <= 24; i++)); do + rbd mirror snapshot schedule add -p ${POOL} \ + --image ${IMAGE_PREFIX}1 2m && break + sleep 10 +done +rbd mirror snapshot schedule ls -p ${POOL} --image ${IMAGE_PREFIX}1 | + grep 'every 2m' +# Verify that the schedule present before client blocklisting is preserved +rbd mirror snapshot schedule ls -p ${POOL} --image ${IMAGE_PREFIX}1 | + grep 'every 1m' +rbd mirror snapshot schedule rm -p ${POOL} --image ${IMAGE_PREFIX}1 2m +for ((i = 1; i <= ${NUM_IMAGES}; i++)); do + rbd mirror snapshot schedule rm -p ${POOL} --image ${IMAGE_PREFIX}$i 1m +done + +# cleanup +killall fio || true +wait +for ((i = 1; i <= ${NUM_IMAGES}; i++)); do + sudo rbd device unmap ${DEVS[$i]} +done + +echo OK diff --git a/qa/workunits/rbd/read-flags.sh b/qa/workunits/rbd/read-flags.sh new file mode 100755 index 000000000..7d787ce67 --- /dev/null +++ b/qa/workunits/rbd/read-flags.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -ex + +# create a snapshot, then export it and check that setting read flags works +# by looking at --debug-ms output + +function clean_up { + rm -f test.log || true + rbd snap remove test@snap || true + rbd rm test || true +} + +function test_read_flags { + local IMAGE=$1 + local SET_BALANCED=$2 + local SET_LOCALIZED=$3 + local EXPECT_BALANCED=$4 + local EXPECT_LOCALIZED=$5 + + local EXTRA_ARGS="--log-file test.log --debug-ms 1 --no-log-to-stderr" + if [ "$SET_BALANCED" = 'y' ]; then + EXTRA_ARGS="$EXTRA_ARGS --rbd-balance-snap-reads" + elif [ "$SET_LOCALIZED" = 'y' ]; then + EXTRA_ARGS="$EXTRA_ARGS --rbd-localize-snap-reads" + fi + + rbd export $IMAGE - $EXTRA_ARGS > /dev/null + if [ "$EXPECT_BALANCED" = 'y' ]; then + grep -q balance_reads test.log + else + grep -L balance_reads test.log | grep -q test.log + fi + if [ "$EXPECT_LOCALIZED" = 'y' ]; then + grep -q localize_reads test.log + else + grep -L localize_reads test.log | grep -q test.log + fi + rm -f test.log + +} + +clean_up + +trap clean_up INT TERM EXIT + +rbd create --image-feature layering -s 10 test +rbd snap create test@snap + +# export from non snapshot with or without settings should not have flags +test_read_flags test n n n n +test_read_flags test y y n n + +# export from snapshot should have read flags in log if they are set +test_read_flags test@snap n n n n +test_read_flags test@snap y n y n +test_read_flags test@snap n y n y + +# balanced_reads happens to take priority over localize_reads +test_read_flags test@snap y y y n + +echo OK diff --git a/qa/workunits/rbd/simple_big.sh b/qa/workunits/rbd/simple_big.sh new file mode 100755 index 000000000..70aafda4c --- /dev/null +++ b/qa/workunits/rbd/simple_big.sh @@ -0,0 +1,12 @@ +#!/bin/sh -ex + +mb=100000 + +rbd create foo --size $mb +DEV=$(sudo rbd map foo) +dd if=/dev/zero of=$DEV bs=1M count=$mb +dd if=$DEV of=/dev/null bs=1M count=$mb +sudo rbd unmap $DEV +rbd rm foo + +echo OK diff --git a/qa/workunits/rbd/test_admin_socket.sh b/qa/workunits/rbd/test_admin_socket.sh new file mode 100755 index 000000000..6b960787b --- /dev/null +++ b/qa/workunits/rbd/test_admin_socket.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -ex + +TMPDIR=/tmp/rbd_test_admin_socket$$ +mkdir $TMPDIR +trap "rm -fr $TMPDIR" 0 + +. $(dirname $0)/../../standalone/ceph-helpers.sh + +function expect_false() +{ + set -x + if "$@"; then return 1; else return 0; fi +} + +function rbd_watch_out_file() +{ + echo ${TMPDIR}/rbd_watch_$1.out +} + +function rbd_watch_pid_file() +{ + echo ${TMPDIR}/rbd_watch_$1.pid +} + +function rbd_watch_fifo() +{ + echo ${TMPDIR}/rbd_watch_$1.fifo +} + +function rbd_watch_asok() +{ + echo ${TMPDIR}/rbd_watch_$1.asok +} + +function rbd_get_perfcounter() +{ + local image=$1 + local counter=$2 + local name + + name=$(ceph --format xml --admin-daemon $(rbd_watch_asok ${image}) \ + perf schema | $XMLSTARLET el -d3 | + grep "/librbd-.*-${image}/${counter}\$") + test -n "${name}" || return 1 + + ceph --format xml --admin-daemon $(rbd_watch_asok ${image}) perf dump | + $XMLSTARLET sel -t -m "${name}" -v . +} + +function rbd_check_perfcounter() +{ + local image=$1 + local counter=$2 + local expected_val=$3 + local val= + + val=$(rbd_get_perfcounter ${image} ${counter}) + + test "${val}" -eq "${expected_val}" +} + +function rbd_watch_start() +{ + local image=$1 + local asok=$(rbd_watch_asok ${image}) + + mkfifo $(rbd_watch_fifo ${image}) + (cat $(rbd_watch_fifo ${image}) | + rbd --admin-socket ${asok} watch ${image} \ + > $(rbd_watch_out_file ${image}) 2>&1)& + + # find pid of the started rbd watch process + local pid + for i in `seq 10`; do + pid=$(ps auxww | awk "/[r]bd --admin.* watch ${image}/ {print \$2}") + test -n "${pid}" && break + sleep 0.1 + done + test -n "${pid}" + echo ${pid} > $(rbd_watch_pid_file ${image}) + + # find watcher admin socket + test -n "${asok}" + for i in `seq 10`; do + test -S "${asok}" && break + sleep 0.1 + done + test -S "${asok}" + + # configure debug level + ceph --admin-daemon "${asok}" config set debug_rbd 20 + + # check that watcher is registered + rbd status ${image} | expect_false grep "Watchers: none" +} + +function rbd_watch_end() +{ + local image=$1 + local regexp=$2 + + # send 'enter' to watch to exit + echo > $(rbd_watch_fifo ${image}) + # just in case it is not terminated + kill $(cat $(rbd_watch_pid_file ${image})) || : + + # output rbd watch out file for easier troubleshooting + cat $(rbd_watch_out_file ${image}) + + # cleanup + rm -f $(rbd_watch_fifo ${image}) $(rbd_watch_pid_file ${image}) \ + $(rbd_watch_out_file ${image}) $(rbd_watch_asok ${image}) +} + +pool="rbd" +image=testimg$$ +ceph_admin="ceph --admin-daemon $(rbd_watch_asok ${image})" + +rbd create --size 128 ${pool}/${image} + +# check rbd cache commands are present in help output +rbd_cache_flush="rbd cache flush ${pool}/${image}" +rbd_cache_invalidate="rbd cache invalidate ${pool}/${image}" + +rbd_watch_start ${image} +${ceph_admin} help | fgrep "${rbd_cache_flush}" +${ceph_admin} help | fgrep "${rbd_cache_invalidate}" +rbd_watch_end ${image} + +# test rbd cache commands with disabled and enabled cache +for conf_rbd_cache in false true; do + + rbd image-meta set ${image} conf_rbd_cache ${conf_rbd_cache} + + rbd_watch_start ${image} + + rbd_check_perfcounter ${image} flush 0 + ${ceph_admin} ${rbd_cache_flush} + # 'flush' counter should increase regardless if cache is enabled + rbd_check_perfcounter ${image} flush 1 + + rbd_check_perfcounter ${image} invalidate_cache 0 + ${ceph_admin} ${rbd_cache_invalidate} + # 'invalidate_cache' counter should increase regardless if cache is enabled + rbd_check_perfcounter ${image} invalidate_cache 1 + + rbd_watch_end ${image} +done + +rbd rm ${image} diff --git a/qa/workunits/rbd/test_librbd.sh b/qa/workunits/rbd/test_librbd.sh new file mode 100755 index 000000000..447306bb4 --- /dev/null +++ b/qa/workunits/rbd/test_librbd.sh @@ -0,0 +1,9 @@ +#!/bin/sh -e + +if [ -n "${VALGRIND}" ]; then + valgrind ${VALGRIND} --suppressions=${TESTDIR}/valgrind.supp \ + --error-exitcode=1 ceph_test_librbd +else + ceph_test_librbd +fi +exit 0 diff --git a/qa/workunits/rbd/test_librbd_python.sh b/qa/workunits/rbd/test_librbd_python.sh new file mode 100755 index 000000000..a33100829 --- /dev/null +++ b/qa/workunits/rbd/test_librbd_python.sh @@ -0,0 +1,12 @@ +#!/bin/sh -ex + +relpath=$(dirname $0)/../../../src/test/pybind + +if [ -n "${VALGRIND}" ]; then + valgrind ${VALGRIND} --suppressions=${TESTDIR}/valgrind.supp \ + --errors-for-leak-kinds=definite --error-exitcode=1 \ + python3 -m pytest -v $relpath/test_rbd.py "$@" +else + python3 -m pytest -v $relpath/test_rbd.py "$@" +fi +exit 0 diff --git a/qa/workunits/rbd/test_lock_fence.sh b/qa/workunits/rbd/test_lock_fence.sh new file mode 100755 index 000000000..7cf2d21c5 --- /dev/null +++ b/qa/workunits/rbd/test_lock_fence.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# can't use -e because of background process +set -x + +IMAGE=rbdrw-image +LOCKID=rbdrw +RELPATH=$(dirname $0)/../../../src/test/librbd +RBDRW=$RELPATH/rbdrw.py + +rbd create $IMAGE --size 10 --image-format 2 --image-shared || exit 1 + +# rbdrw loops doing I/O to $IMAGE after locking with lockid $LOCKID +python3 $RBDRW $IMAGE $LOCKID & +iochild=$! + +# give client time to lock and start reading/writing +LOCKS='[]' +while [ "$LOCKS" == '[]' ] +do + LOCKS=$(rbd lock list $IMAGE --format json) + sleep 1 +done + +clientaddr=$(rbd lock list $IMAGE | tail -1 | awk '{print $NF;}') +clientid=$(rbd lock list $IMAGE | tail -1 | awk '{print $1;}') +echo "clientaddr: $clientaddr" +echo "clientid: $clientid" + +ceph osd blocklist add $clientaddr || exit 1 + +wait $iochild +rbdrw_exitcode=$? +if [ $rbdrw_exitcode != 108 ] +then + echo "wrong exitcode from rbdrw: $rbdrw_exitcode" + exit 1 +else + echo "rbdrw stopped with ESHUTDOWN" +fi + +set -e +ceph osd blocklist rm $clientaddr +rbd lock remove $IMAGE $LOCKID "$clientid" +# rbdrw will have exited with an existing watch, so, until #3527 is fixed, +# hang out until the watch expires +sleep 30 +rbd rm $IMAGE +echo OK diff --git a/qa/workunits/rbd/test_rbd_mirror.sh b/qa/workunits/rbd/test_rbd_mirror.sh new file mode 100755 index 000000000..e139dd7e4 --- /dev/null +++ b/qa/workunits/rbd/test_rbd_mirror.sh @@ -0,0 +1,9 @@ +#!/bin/sh -e + +if [ -n "${VALGRIND}" ]; then + valgrind ${VALGRIND} --suppressions=${TESTDIR}/valgrind.supp \ + --error-exitcode=1 ceph_test_rbd_mirror +else + ceph_test_rbd_mirror +fi +exit 0 diff --git a/qa/workunits/rbd/test_rbd_tasks.sh b/qa/workunits/rbd/test_rbd_tasks.sh new file mode 100755 index 000000000..b9663e601 --- /dev/null +++ b/qa/workunits/rbd/test_rbd_tasks.sh @@ -0,0 +1,276 @@ +#!/usr/bin/env bash +set -ex + +POOL=rbd_tasks +POOL_NS=ns1 + +setup() { + trap 'cleanup' INT TERM EXIT + + ceph osd pool create ${POOL} 128 + rbd pool init ${POOL} + rbd namespace create ${POOL}/${POOL_NS} + + TEMPDIR=`mktemp -d` +} + +cleanup() { + ceph osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it + + rm -rf ${TEMPDIR} +} + +wait_for() { + local TEST_FN=$1 + shift 1 + local TEST_FN_ARGS=("$@") + + for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do + sleep ${s} + + ${TEST_FN} "${TEST_FN_ARGS[@]}" || continue + return 0 + done + return 1 +} + +task_exists() { + local TASK_ID=$1 + [[ -z "${TASK_ID}" ]] && exit 1 + + ceph rbd task list ${TASK_ID} || return 1 + return 0 +} + +task_dne() { + local TASK_ID=$1 + [[ -z "${TASK_ID}" ]] && exit 1 + + ceph rbd task list ${TASK_ID} || return 0 + return 1 +} + +task_in_progress() { + local TASK_ID=$1 + [[ -z "${TASK_ID}" ]] && exit 1 + + [[ $(ceph rbd task list ${TASK_ID} | jq '.in_progress') == 'true' ]] +} + +test_remove() { + echo "test_remove" + + local IMAGE=`uuidgen` + rbd create --size 1 --image-shared ${POOL}/${IMAGE} + + # MGR might require some time to discover the OSD map w/ new pool + wait_for ceph rbd task add remove ${POOL}/${IMAGE} +} + +test_flatten() { + echo "test_flatten" + + local PARENT_IMAGE=`uuidgen` + local CHILD_IMAGE=`uuidgen` + + rbd create --size 1 --image-shared ${POOL}/${PARENT_IMAGE} + rbd snap create ${POOL}/${PARENT_IMAGE}@snap + rbd clone ${POOL}/${PARENT_IMAGE}@snap ${POOL}/${POOL_NS}/${CHILD_IMAGE} --rbd-default-clone-format=2 + [[ "$(rbd info --format json ${POOL}/${POOL_NS}/${CHILD_IMAGE} | jq 'has("parent")')" == "true" ]] + + local TASK_ID=`ceph rbd task add flatten ${POOL}/${POOL_NS}/${CHILD_IMAGE} | jq --raw-output ".id"` + wait_for task_dne ${TASK_ID} + + [[ "$(rbd info --format json ${POOL}/${POOL_NS}/${CHILD_IMAGE} | jq 'has("parent")')" == "false" ]] +} + +test_trash_remove() { + echo "test_trash_remove" + + local IMAGE=`uuidgen` + rbd create --size 1 --image-shared ${POOL}/${IMAGE} + local IMAGE_ID=`rbd info --format json ${POOL}/${IMAGE} | jq --raw-output ".id"` + rbd trash mv ${POOL}/${IMAGE} + [[ -n "$(rbd trash list ${POOL})" ]] || exit 1 + + local TASK_ID=`ceph rbd task add trash remove ${POOL}/${IMAGE_ID} | jq --raw-output ".id"` + wait_for task_dne ${TASK_ID} + + [[ -z "$(rbd trash list ${POOL})" ]] || exit 1 +} + +test_migration_execute() { + echo "test_migration_execute" + + local SOURCE_IMAGE=`uuidgen` + local TARGET_IMAGE=`uuidgen` + rbd create --size 1 --image-shared ${POOL}/${SOURCE_IMAGE} + rbd migration prepare ${POOL}/${SOURCE_IMAGE} ${POOL}/${TARGET_IMAGE} + [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq --raw-output '.migration.state')" == "prepared" ]] + + local TASK_ID=`ceph rbd task add migration execute ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"` + wait_for task_dne ${TASK_ID} + + [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq --raw-output '.migration.state')" == "executed" ]] +} + +test_migration_commit() { + echo "test_migration_commit" + + local SOURCE_IMAGE=`uuidgen` + local TARGET_IMAGE=`uuidgen` + rbd create --size 1 --image-shared ${POOL}/${SOURCE_IMAGE} + rbd migration prepare ${POOL}/${SOURCE_IMAGE} ${POOL}/${TARGET_IMAGE} + [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq --raw-output '.migration.state')" == "prepared" ]] + + local TASK_ID=`ceph rbd task add migration execute ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"` + wait_for task_dne ${TASK_ID} + + TASK_ID=`ceph rbd task add migration commit ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"` + wait_for task_dne ${TASK_ID} + + [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq 'has("migration")')" == "false" ]] + (rbd info ${POOL}/${SOURCE_IMAGE} && return 1) || true + rbd info ${POOL}/${TARGET_IMAGE} +} + +test_migration_abort() { + echo "test_migration_abort" + + local SOURCE_IMAGE=`uuidgen` + local TARGET_IMAGE=`uuidgen` + rbd create --size 1 --image-shared ${POOL}/${SOURCE_IMAGE} + rbd migration prepare ${POOL}/${SOURCE_IMAGE} ${POOL}/${TARGET_IMAGE} + [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq --raw-output '.migration.state')" == "prepared" ]] + + local TASK_ID=`ceph rbd task add migration execute ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"` + wait_for task_dne ${TASK_ID} + + TASK_ID=`ceph rbd task add migration abort ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"` + wait_for task_dne ${TASK_ID} + + [[ "$(rbd status --format json ${POOL}/${SOURCE_IMAGE} | jq 'has("migration")')" == "false" ]] + rbd info ${POOL}/${SOURCE_IMAGE} + (rbd info ${POOL}/${TARGET_IMAGE} && return 1) || true +} + +test_list() { + echo "test_list" + + local IMAGE_1=`uuidgen` + local IMAGE_2=`uuidgen` + + rbd create --size 1T --image-shared ${POOL}/${IMAGE_1} + rbd create --size 1T --image-shared ${POOL}/${IMAGE_2} + + local TASK_ID_1=`ceph rbd task add remove ${POOL}/${IMAGE_1} | jq --raw-output ".id"` + local TASK_ID_2=`ceph rbd task add remove ${POOL}/${IMAGE_2} | jq --raw-output ".id"` + + local LIST_FILE="${TEMPDIR}/list_file" + ceph rbd task list > ${LIST_FILE} + cat ${LIST_FILE} + + [[ $(jq "[.[] | .id] | contains([\"${TASK_ID_1}\", \"${TASK_ID_2}\"])" ${LIST_FILE}) == "true" ]] + + ceph rbd task cancel ${TASK_ID_1} + ceph rbd task cancel ${TASK_ID_2} +} + +test_cancel() { + echo "test_cancel" + + local IMAGE=`uuidgen` + rbd create --size 1T --image-shared ${POOL}/${IMAGE} + local TASK_ID=`ceph rbd task add remove ${POOL}/${IMAGE} | jq --raw-output ".id"` + + wait_for task_exists ${TASK_ID} + + ceph rbd task cancel ${TASK_ID} + wait_for task_dne ${TASK_ID} +} + +test_duplicate_task() { + echo "test_duplicate_task" + + local IMAGE=`uuidgen` + rbd create --size 1T --image-shared ${POOL}/${IMAGE} + local IMAGE_ID=`rbd info --format json ${POOL}/${IMAGE} | jq --raw-output ".id"` + rbd trash mv ${POOL}/${IMAGE} + + local TASK_ID_1=`ceph rbd task add trash remove ${POOL}/${IMAGE_ID} | jq --raw-output ".id"` + local TASK_ID_2=`ceph rbd task add trash remove ${POOL}/${IMAGE_ID} | jq --raw-output ".id"` + + [[ "${TASK_ID_1}" == "${TASK_ID_2}" ]] + + ceph rbd task cancel ${TASK_ID_1} +} + +test_duplicate_name() { + echo "test_duplicate_name" + + local IMAGE=`uuidgen` + rbd create --size 1G --image-shared ${POOL}/${IMAGE} + local TASK_ID_1=`ceph rbd task add remove ${POOL}/${IMAGE} | jq --raw-output ".id"` + + wait_for task_dne ${TASK_ID_1} + + rbd create --size 1G --image-shared ${POOL}/${IMAGE} + local TASK_ID_2=`ceph rbd task add remove ${POOL}/${IMAGE} | jq --raw-output ".id"` + + [[ "${TASK_ID_1}" != "${TASK_ID_2}" ]] + wait_for task_dne ${TASK_ID_2} + + local TASK_ID_3=`ceph rbd task add remove ${POOL}/${IMAGE} | jq --raw-output ".id"` + + [[ "${TASK_ID_2}" == "${TASK_ID_3}" ]] +} + +test_progress() { + echo "test_progress" + + local IMAGE_1=`uuidgen` + local IMAGE_2=`uuidgen` + + rbd create --size 1 --image-shared ${POOL}/${IMAGE_1} + local TASK_ID_1=`ceph rbd task add remove ${POOL}/${IMAGE_1} | jq --raw-output ".id"` + + wait_for task_dne ${TASK_ID_1} + + local PROGRESS_FILE="${TEMPDIR}/progress_file" + ceph progress json > ${PROGRESS_FILE} + cat ${PROGRESS_FILE} + + [[ $(jq "[.completed | .[].id] | contains([\"${TASK_ID_1}\"])" ${PROGRESS_FILE}) == "true" ]] + + rbd create --size 1T --image-shared ${POOL}/${IMAGE_2} + local TASK_ID_2=`ceph rbd task add remove ${POOL}/${IMAGE_2} | jq --raw-output ".id"` + + wait_for task_in_progress ${TASK_ID_2} + ceph progress json > ${PROGRESS_FILE} + cat ${PROGRESS_FILE} + + [[ $(jq "[.events | .[].id] | contains([\"${TASK_ID_2}\"])" ${PROGRESS_FILE}) == "true" ]] + + ceph rbd task cancel ${TASK_ID_2} + wait_for task_dne ${TASK_ID_2} + + ceph progress json > ${PROGRESS_FILE} + cat ${PROGRESS_FILE} + + [[ $(jq "[.completed | map(select(.failed)) | .[].id] | contains([\"${TASK_ID_2}\"])" ${PROGRESS_FILE}) == "true" ]] +} + +setup +test_remove +test_flatten +test_trash_remove +test_migration_execute +test_migration_commit +test_migration_abort +test_list +test_cancel +test_duplicate_task +test_duplicate_name +test_progress + +echo OK diff --git a/qa/workunits/rbd/test_rbdmap_RBDMAPFILE.sh b/qa/workunits/rbd/test_rbdmap_RBDMAPFILE.sh new file mode 100755 index 000000000..501c69cd5 --- /dev/null +++ b/qa/workunits/rbd/test_rbdmap_RBDMAPFILE.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# +# Regression test for http://tracker.ceph.com/issues/14984 +# +# When the bug is present, starting the rbdmap service causes +# a bogus log message to be emitted to the log because the RBDMAPFILE +# environment variable is not set. +# +# When the bug is not present, starting the rbdmap service will emit +# no log messages, because /etc/ceph/rbdmap does not contain any lines +# that require processing. +# +set -ex + +echo "TEST: save timestamp for use later with journalctl --since" +TIMESTAMP=$(date +%Y-%m-%d\ %H:%M:%S) + +echo "TEST: assert that rbdmap has not logged anything since boot" +journalctl -b 0 -t rbdmap | grep 'rbdmap\[[[:digit:]]' && exit 1 +journalctl -b 0 -t init-rbdmap | grep 'rbdmap\[[[:digit:]]' && exit 1 + +echo "TEST: restart the rbdmap.service" +sudo systemctl restart rbdmap.service + +echo "TEST: ensure that /usr/bin/rbdmap runs to completion" +until sudo systemctl status rbdmap.service | grep 'active (exited)' ; do + sleep 0.5 +done + +echo "TEST: assert that rbdmap has not logged anything since TIMESTAMP" +journalctl --since "$TIMESTAMP" -t rbdmap | grep 'rbdmap\[[[:digit:]]' && exit 1 +journalctl --since "$TIMESTAMP" -t init-rbdmap | grep 'rbdmap\[[[:digit:]]' && exit 1 + +exit 0 diff --git a/qa/workunits/rbd/verify_pool.sh b/qa/workunits/rbd/verify_pool.sh new file mode 100755 index 000000000..08bcca506 --- /dev/null +++ b/qa/workunits/rbd/verify_pool.sh @@ -0,0 +1,27 @@ +#!/bin/sh -ex + +POOL_NAME=rbd_test_validate_pool +PG_NUM=32 + +tear_down () { + ceph osd pool delete $POOL_NAME $POOL_NAME --yes-i-really-really-mean-it || true +} + +set_up () { + tear_down + ceph osd pool create $POOL_NAME $PG_NUM + ceph osd pool mksnap $POOL_NAME snap + rbd pool init $POOL_NAME +} + +trap tear_down EXIT HUP INT +set_up + +# creating an image in a pool-managed snapshot pool should fail +rbd create --pool $POOL_NAME --size 1 foo && exit 1 || true + +# should succeed if the pool already marked as validated +printf "overwrite validated" | rados --pool $POOL_NAME put rbd_info - +rbd create --pool $POOL_NAME --size 1 foo + +echo OK diff --git a/qa/workunits/rename/all.sh b/qa/workunits/rename/all.sh new file mode 100755 index 000000000..f97ff773f --- /dev/null +++ b/qa/workunits/rename/all.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -ex + +dir=`dirname $0` + +CEPH_TOOL='./ceph' +$CEPH_TOOL || CEPH_TOOL='ceph' + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/prepare.sh + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/pri_nul.sh +rm ./?/* || true + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/rem_nul.sh +rm ./?/* || true + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/pri_pri.sh +rm ./?/* || true + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/rem_pri.sh +rm ./?/* || true + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/rem_rem.sh +rm ./?/* || true + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/pri_nul.sh +rm -r ./?/* || true + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/pri_pri.sh +rm -r ./?/* || true + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/dir_pri_pri.sh +rm -r ./?/* || true + +CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/dir_pri_nul.sh +rm -r ./?/* || true + diff --git a/qa/workunits/rename/dir_pri_nul.sh b/qa/workunits/rename/dir_pri_nul.sh new file mode 100755 index 000000000..dd8106b63 --- /dev/null +++ b/qa/workunits/rename/dir_pri_nul.sh @@ -0,0 +1,28 @@ +#!/bin/sh -ex + +# dir: srcdn=destdn +mkdir ./a/dir1 +mv ./a/dir1 ./a/dir1.renamed + +# dir: diff +mkdir ./a/dir2 +mv ./a/dir2 ./b/dir2 + +# dir: diff, child subtree on target +mkdir -p ./a/dir3/child/foo +$CEPH_TOOL mds tell 0 export_dir /a/dir3/child 1 +sleep 5 +mv ./a/dir3 ./b/dir3 + +# dir: diff, child subtree on other +mkdir -p ./a/dir4/child/foo +$CEPH_TOOL mds tell 0 export_dir /a/dir4/child 2 +sleep 5 +mv ./a/dir4 ./b/dir4 + +# dir: witness subtree adjustment +mkdir -p ./a/dir5/1/2/3/4 +$CEPH_TOOL mds tell 0 export_dir /a/dir5/1/2/3 2 +sleep 5 +mv ./a/dir5 ./b + diff --git a/qa/workunits/rename/dir_pri_pri.sh b/qa/workunits/rename/dir_pri_pri.sh new file mode 100755 index 000000000..de235fcd3 --- /dev/null +++ b/qa/workunits/rename/dir_pri_pri.sh @@ -0,0 +1,11 @@ +#!/bin/sh -ex + +# dir, srcdn=destdn +mkdir ./a/dir1 +mkdir ./a/dir2 +mv -T ./a/dir1 ./a/dir2 + +# dir, different +mkdir ./a/dir3 +mkdir ./b/dir4 +mv -T ./a/dir3 ./b/dir4 diff --git a/qa/workunits/rename/plan.txt b/qa/workunits/rename/plan.txt new file mode 100644 index 000000000..b423b4140 --- /dev/null +++ b/qa/workunits/rename/plan.txt @@ -0,0 +1,111 @@ +#!/bin/sh + +# srcdn destdn targeti + +## pri auth null auth - +## pri rep null auth - +## rem auth null auth - +## rem rep null auth - + +#/ pri auth null rep - dup of pr_na +#/ pri rep null rep - +#/ rem auth null rep - dup of rr_na +#/ rem rep null rep - + + +## pri auth pri auth - +# pri rep pri auth - +## rem auth pri auth - +# rem rep pri auth - + +# pri auth pri rep - +# pri rep pri rep - +# rem auth pri rep - +# rem rep pri rep - + +## pri auth rem auth auth +# pri rep rem auth auth +## rem auth rem auth auth +# rem rep rem auth auth + +# pri auth rem rep auth +# pri rep rem rep auth +# rem auth rem rep auth +# rem rep rem rep auth + +# pri auth rem auth rep +# pri rep rem auth rep +# rem auth rem auth rep +# rem rep rem auth rep + +# pri auth rem rep rep +# pri rep rem rep rep +# rem auth rem rep rep +# rem rep rem rep rep + + +types of operations + +pri nul + srcdn=destdn + diff + +rem nul + srci=srcdn=destdn + srci=srcdn + srcdn=destdn + srci=destdn + all different + +pri pri + srcdn=destdn + different + +rem pri + srci=srcdn=destdn + srci=srcdn + srcdn=destdn + srci=destdn + all different + +pri rem + srcdn=destdn=desti + srcdn=destdn + destdn=desti + srcdn=desti + all different + +rem rem + srci=srcdn=destdn=desti + srci=srcdn=destdn + srci=srcdn=desti + srci=destdn=desti + srcdni=destdn=desti + srci=srcdn destdn=desti + srci=destdn srcdn=desti + srci=desti srcdn=destdn + srci=srcdn + srci=destdn + srci=desti + srcdn=destdn + srcdn=desti + destdn=desti + all different + + + + + + + + + +p n same +r n same +p n diff +r n diff + +p p same +r p same + +p r diff --git a/qa/workunits/rename/prepare.sh b/qa/workunits/rename/prepare.sh new file mode 100755 index 000000000..b5ba4ae58 --- /dev/null +++ b/qa/workunits/rename/prepare.sh @@ -0,0 +1,21 @@ +#!/bin/sh -ex + +$CEPH_TOOL mds tell 0 injectargs '--mds-bal-interval 0' +$CEPH_TOOL mds tell 1 injectargs '--mds-bal-interval 0' +$CEPH_TOOL mds tell 2 injectargs '--mds-bal-interval 0' +$CEPH_TOOL mds tell 3 injectargs '--mds-bal-interval 0' +#$CEPH_TOOL mds tell 4 injectargs '--mds-bal-interval 0' + +mkdir -p ./a/a +mkdir -p ./b/b +mkdir -p ./c/c +mkdir -p ./d/d + +mount_dir=`df . | grep -o " /.*" | grep -o "/.*"` +cur_dir=`pwd` +ceph_dir=${cur_dir##$mount_dir} +$CEPH_TOOL mds tell 0 export_dir $ceph_dir/b 1 +$CEPH_TOOL mds tell 0 export_dir $ceph_dir/c 2 +$CEPH_TOOL mds tell 0 export_dir $ceph_dir/d 3 +sleep 5 + diff --git a/qa/workunits/rename/pri_nul.sh b/qa/workunits/rename/pri_nul.sh new file mode 100755 index 000000000..c40ec1d25 --- /dev/null +++ b/qa/workunits/rename/pri_nul.sh @@ -0,0 +1,11 @@ +#!/bin/sh -ex + +# srcdn=destdn +touch ./a/file1 +mv ./a/file1 ./a/file1.renamed + +# different +touch ./a/file2 +mv ./a/file2 ./b + + diff --git a/qa/workunits/rename/pri_pri.sh b/qa/workunits/rename/pri_pri.sh new file mode 100755 index 000000000..b74985fe3 --- /dev/null +++ b/qa/workunits/rename/pri_pri.sh @@ -0,0 +1,12 @@ +#!/bin/sh -ex + +# srcdn=destdn +touch ./a/file1 +touch ./a/file2 +mv ./a/file1 ./a/file2 + +# different (srcdn != destdn) +touch ./a/file3 +touch ./b/file4 +mv ./a/file3 ./b/file4 + diff --git a/qa/workunits/rename/pri_rem.sh b/qa/workunits/rename/pri_rem.sh new file mode 100755 index 000000000..a1cd03d10 --- /dev/null +++ b/qa/workunits/rename/pri_rem.sh @@ -0,0 +1,31 @@ +#!/bin/sh -ex + +dotest() { + src=$1 + desti=$2 + destdn=$3 + n=$4 + + touch ./$src/src$n + touch ./$desti/desti$n + ln ./$desti/desti$n ./$destdn/destdn$n + + mv ./$src/src$n ./$destdn/destdn$n +} + + +# srcdn=destdn=desti +dotest 'a' 'a' 'a' 1 + +# destdn=desti +dotest 'b' 'a' 'a' 2 + +# srcdn=destdn +dotest 'a' 'b' 'a' 3 + +# srcdn=desti +dotest 'a' 'a' 'b' 4 + +# all different +dotest 'a' 'b' 'c' 5 + diff --git a/qa/workunits/rename/rem_nul.sh b/qa/workunits/rename/rem_nul.sh new file mode 100755 index 000000000..a71033108 --- /dev/null +++ b/qa/workunits/rename/rem_nul.sh @@ -0,0 +1,29 @@ +#!/bin/sh -ex + +dotest() { + srci=$1 + srcdn=$2 + dest=$3 + n=$4 + + touch ./$srci/srci$n + ln ./$srci/srci$n ./$srcdn/srcdn$n + + mv ./$srcdn/srcdn$n ./$dest/dest$n +} + +# srci=srcdn=destdn +dotest 'a' 'a' 'a' 1 + +# srcdn=destdn +dotest 'b' 'a' 'a' 2 + +# srci=destdn +dotest 'a' 'b' 'a' 3 + +# srci=srcdn +dotest 'a' 'a' 'b' 4 + +# all different +dotest 'a' 'b' 'c' 5 + diff --git a/qa/workunits/rename/rem_pri.sh b/qa/workunits/rename/rem_pri.sh new file mode 100755 index 000000000..501ac5e1a --- /dev/null +++ b/qa/workunits/rename/rem_pri.sh @@ -0,0 +1,29 @@ +#!/bin/sh -ex + +dotest() { + srci=$1 + srcdn=$2 + dest=$3 + n=$4 + + touch ./$srci/srci$n + ln ./$srci/srci$n ./$srcdn/srcdn$n + touch ./$dest/dest$n + + mv ./$srcdn/srcdn$n ./$dest/dest$n +} + +# srci=srcdn=destdn +dotest 'a' 'a' 'a' 1 + +# srcdn=destdn +dotest 'b' 'a' 'a' 2 + +# srci=destdn +dotest 'a' 'b' 'a' 3 + +# srci=srcdn +dotest 'a' 'a' 'b' 4 + +# all different +dotest 'a' 'b' 'c' 5 diff --git a/qa/workunits/rename/rem_rem.sh b/qa/workunits/rename/rem_rem.sh new file mode 100755 index 000000000..80028c517 --- /dev/null +++ b/qa/workunits/rename/rem_rem.sh @@ -0,0 +1,61 @@ +#!/bin/sh -ex + +dotest() { + srci=$1 + srcdn=$2 + desti=$3 + destdn=$4 + n=$5 + + touch ./$srci/srci$n + ln ./$srci/srci$n ./$srcdn/srcdn$n + touch ./$desti/desti$n + ln ./$desti/desti$n ./$destdn/destdn$n + + mv ./$srcdn/srcdn$n ./$destdn/destdn$n +} + +# srci=srcdn=destdn=desti +dotest 'a' 'a' 'a' 'a' 1 + +# srcdn=destdn=desti +dotest 'b' 'a' 'a' 'a' 2 + +# srci=destdn=desti +dotest 'a' 'b' 'a' 'a' 3 + +# srci=srcdn=destdn +dotest 'a' 'a' 'b' 'a' 4 + +# srci=srcdn=desti +dotest 'a' 'a' 'a' 'b' 5 + +# srci=srcdn destdn=desti +dotest 'a' 'a' 'b' 'b' 6 + +# srci=destdn srcdn=desti +dotest 'a' 'b' 'b' 'a' 7 + +# srci=desti srcdn=destdn +dotest 'a' 'b' 'a' 'b' 8 + +# srci=srcdn +dotest 'a' 'a' 'b' 'c' 9 + +# srci=desti +dotest 'a' 'b' 'a' 'c' 10 + +# srci=destdn +dotest 'a' 'b' 'c' 'a' 11 + +# srcdn=desti +dotest 'a' 'b' 'b' 'c' 12 + +# srcdn=destdn +dotest 'a' 'b' 'c' 'b' 13 + +# destdn=desti +dotest 'a' 'b' 'c' 'c' 14 + +# all different +dotest 'a' 'b' 'c' 'd' 15 diff --git a/qa/workunits/rest/test-restful.sh b/qa/workunits/rest/test-restful.sh new file mode 100755 index 000000000..fde0d107a --- /dev/null +++ b/qa/workunits/rest/test-restful.sh @@ -0,0 +1,10 @@ +#!/bin/sh -ex + +mydir=`dirname $0` + +secret=`ceph config-key get mgr/restful/keys/admin` +url=$(ceph mgr dump|jq -r .services.restful|sed -e 's/\/$//') +echo "url $url secret $secret" +$mydir/test_mgr_rest_api.py $url $secret + +echo $0 OK diff --git a/qa/workunits/rest/test_mgr_rest_api.py b/qa/workunits/rest/test_mgr_rest_api.py new file mode 100755 index 000000000..74126ab78 --- /dev/null +++ b/qa/workunits/rest/test_mgr_rest_api.py @@ -0,0 +1,98 @@ +#! /usr/bin/env python3 + +import requests +import time +import sys +import json + +# Do not show the stupid message about verify=False. ignore exceptions bc +# this doesn't work on some distros. +try: + from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) +except: + pass + +if len(sys.argv) < 3: + print("Usage: %s <url> <admin_key>" % sys.argv[0]) + sys.exit(1) + +addr = sys.argv[1] +auth = ('admin', sys.argv[2]) +headers = {'Content-type': 'application/json'} + +request = None + +# Create a pool and get its id +request = requests.post( + addr + '/pool?wait=yes', + data=json.dumps({'name': 'supertestfriends', 'pg_num': 128}), + headers=headers, + verify=False, + auth=auth) +print(request.text) +request = requests.get(addr + '/pool', verify=False, auth=auth) +assert(request.json()[-1]['pool_name'] == 'supertestfriends') +pool_id = request.json()[-1]['pool'] + +# get a mon name +request = requests.get(addr + '/mon', verify=False, auth=auth) +firstmon = request.json()[0]['name'] +print('first mon is %s' % firstmon) + +# get a server name +request = requests.get(addr + '/osd', verify=False, auth=auth) +aserver = request.json()[0]['server'] +print('a server is %s' % aserver) + + +screenplay = [ + ('get', '/', {}), + ('get', '/config/cluster', {}), + ('get', '/crush/rule', {}), + ('get', '/doc', {}), + ('get', '/mon', {}), + ('get', '/mon/' + firstmon, {}), + ('get', '/osd', {}), + ('get', '/osd/0', {}), + ('get', '/osd/0/command', {}), + ('get', '/pool/1', {}), + ('get', '/server', {}), + ('get', '/server/' + aserver, {}), + ('post', '/osd/0/command', {'command': 'scrub'}), + ('post', '/pool?wait=1', {'name': 'supertestfriends', 'pg_num': 128}), + ('patch', '/osd/0', {'in': False}), + ('patch', '/config/osd', {'pause': True}), + ('get', '/config/osd', {}), + ('patch', '/pool/' + str(pool_id), {'size': 2}), + ('patch', '/config/osd', {'pause': False}), + ('patch', '/osd/0', {'in': True}), + ('get', '/pool', {}), + ('delete', '/pool/' + str(pool_id) + '?wait=1', {}), + ('get', '/request?page=0', {}), + ('delete', '/request', {}), + ('get', '/request', {}), + ('patch', '/pool/1', {'pg_num': 128}), + ('patch', '/pool/1', {'pgp_num': 128}), + ('get', '/perf?daemon=.*', {}), +] + +for method, endpoint, args in screenplay: + if method == 'sleep': + time.sleep(endpoint) + continue + url = addr + endpoint + print("URL = " + url) + request = getattr(requests, method)( + url, + data=json.dumps(args) if args else None, + headers=headers, + verify=False, + auth=auth) + assert request is not None + print(request.text) + if request.status_code != 200 or 'error' in request.json(): + print('ERROR: %s request for URL "%s" failed' % (method, url)) + sys.exit(1) + +print('OK') diff --git a/qa/workunits/restart/test-backtraces.py b/qa/workunits/restart/test-backtraces.py new file mode 100755 index 000000000..37ddef539 --- /dev/null +++ b/qa/workunits/restart/test-backtraces.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 + +from __future__ import print_function + +import subprocess +import json +import os +import time +import sys + +import rados as rados +import cephfs as cephfs + +prefix='testbt' + +def get_name(b, i, j): + c = '{pre}.{pid}.{i}.{j}'.format(pre=prefix, pid=os.getpid(), i=i, j=j) + return c, b + '/' + c + +def mkdir(ceph, d): + print("mkdir {d}".format(d=d), file=sys.stderr) + ceph.mkdir(d, 0o755) + return ceph.stat(d)['st_ino'] + +def create(ceph, f): + print("creating {f}".format(f=f), file=sys.stderr) + fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644) + ceph.close(fd) + return ceph.stat(f)['st_ino'] + +def set_mds_config_param(ceph, param): + with open('/dev/null', 'rb') as devnull: + confarg = '' + if conf != '': + confarg = '-c {c}'.format(c=conf) + r = subprocess.call("ceph {ca} mds tell a injectargs '{p}'".format(ca=confarg, p=param), shell=True, stdout=devnull) + if r != 0: + raise Exception + + +class _TrimIndentFile(object): + def __init__(self, fp): + self.fp = fp + + def readline(self): + line = self.fp.readline() + return line.lstrip(' \t') + +def _optionxform(s): + s = s.replace('_', ' ') + s = '_'.join(s.split()) + return s + +def conf_set_kill_mds(location, killnum): + print('setting mds kill config option for {l}.{k}'.format(l=location, k=killnum), file=sys.stderr) + print("restart mds a mds_kill_{l}_at {k}".format(l=location, k=killnum)) + sys.stdout.flush() + for l in sys.stdin.readline(): + if l == 'restarted': + break + +def flush(ceph, testnum): + print('flushing {t}'.format(t=testnum), file=sys.stderr) + set_mds_config_param(ceph, '--mds_log_max_segments 1') + + for i in range(1, 500): + f = '{p}.{pid}.{t}.{i}'.format(p=prefix, pid=os.getpid(), t=testnum, i=i) + print('flushing with create {f}'.format(f=f), file=sys.stderr) + fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644) + ceph.close(fd) + ceph.unlink(f) + + print('flush doing shutdown', file=sys.stderr) + ceph.shutdown() + print('flush reinitializing ceph', file=sys.stderr) + ceph = cephfs.LibCephFS(conffile=conf) + print('flush doing mount', file=sys.stderr) + ceph.mount() + return ceph + +def kill_mds(ceph, location, killnum): + print('killing mds: {l}.{k}'.format(l=location, k=killnum), file=sys.stderr) + set_mds_config_param(ceph, '--mds_kill_{l}_at {k}'.format(l=location, k=killnum)) + +def wait_for_mds(ceph): + # wait for restart + while True: + confarg = '' + if conf != '': + confarg = '-c {c}'.format(c=conf) + r = subprocess.check_output("ceph {ca} mds stat".format(ca=confarg), shell=True).decode() + if r.find('a=up:active'): + break + time.sleep(1) + +def decode(value): + + tmpfile = '/tmp/{p}.{pid}'.format(p=prefix, pid=os.getpid()) + with open(tmpfile, 'w+') as f: + f.write(value) + + p = subprocess.Popen( + [ + 'ceph-dencoder', + 'import', + tmpfile, + 'type', + 'inode_backtrace_t', + 'decode', + 'dump_json', + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + (stdout, _) = p.communicate(input=value) + p.stdin.close() + if p.returncode != 0: + raise Exception + os.remove(tmpfile) + return json.loads(stdout) + +class VerifyFailure(Exception): + pass + +def verify(rados_ioctx, ino, values, pool): + print('getting parent attr for ino: %lx.00000000' % ino, file=sys.stderr) + savede = None + for i in range(1, 20): + try: + savede = None + binbt = rados_ioctx.get_xattr('%lx.00000000' % ino, 'parent') + except rados.ObjectNotFound as e: + # wait for a bit to let segments get flushed out + savede = e + time.sleep(10) + if savede: + raise savede + + bt = decode(binbt) + + ind = 0 + if bt['ino'] != ino: + raise VerifyFailure('inode mismatch: {bi} != {ino}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( + bi=bt['ancestors'][ind]['dname'], ino=ino, bt=bt, i=ino, v=values)) + for (n, i) in values: + if bt['ancestors'][ind]['dirino'] != i: + raise VerifyFailure('ancestor dirino mismatch: {b} != {ind}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( + b=bt['ancestors'][ind]['dirino'], ind=i, bt=bt, i=ino, v=values)) + if bt['ancestors'][ind]['dname'] != n: + raise VerifyFailure('ancestor dname mismatch: {b} != {n}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( + b=bt['ancestors'][ind]['dname'], n=n, bt=bt, i=ino, v=values)) + ind += 1 + + if bt['pool'] != pool: + raise VerifyFailure('pool mismatch: {btp} != {p}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format( + btp=bt['pool'], p=pool, bt=bt, i=ino, v=values)) + +def make_abc(ceph, rooti, i): + expected_bt = [] + c, d = get_name("/", i, 0) + expected_bt = [(c, rooti)] + expected_bt + di = mkdir(ceph, d) + c, d = get_name(d, i, 1) + expected_bt = [(c, di)] + expected_bt + di = mkdir(ceph, d) + c, f = get_name(d, i, 2) + fi = create(ceph, f) + expected_bt = [(c, di)] + expected_bt + return fi, expected_bt + +test = -1 +if len(sys.argv) > 1: + test = int(sys.argv[1]) + +conf = '' +if len(sys.argv) > 2: + conf = sys.argv[2] + +radosobj = rados.Rados(conffile=conf) +radosobj.connect() +ioctx = radosobj.open_ioctx('data') + +ceph = cephfs.LibCephFS(conffile=conf) +ceph.mount() + +rooti = ceph.stat('/')['st_ino'] + +test = -1 +if len(sys.argv) > 1: + test = int(sys.argv[1]) + +conf = '/etc/ceph/ceph.conf' +if len(sys.argv) > 2: + conf = sys.argv[2] + +# create /a/b/c +# flush +# verify + +i = 0 +if test < 0 or test == i: + print('Running test %d: basic verify' % i, file=sys.stderr) + ino, expected_bt = make_abc(ceph, rooti, i) + ceph = flush(ceph, i) + verify(ioctx, ino, expected_bt, 0) + +i += 1 + +# kill-mds-at-openc-1 +# create /a/b/c +# restart-mds +# flush +# verify + +if test < 0 or test == i: + print('Running test %d: kill openc' % i, file=sys.stderr) + print("restart mds a") + sys.stdout.flush() + kill_mds(ceph, 'openc', 1) + ino, expected_bt = make_abc(ceph, rooti, i) + ceph = flush(ceph, i) + verify(ioctx, ino, expected_bt, 0) + +i += 1 + +# kill-mds-at-openc-1 +# create /a/b/c +# restart-mds with kill-mds-at-replay-1 +# restart-mds +# flush +# verify +if test < 0 or test == i: + print('Running test %d: kill openc/replay' % i, file=sys.stderr) + # these are reversed because we want to prepare the config + conf_set_kill_mds('journal_replay', 1) + kill_mds(ceph, 'openc', 1) + print("restart mds a") + sys.stdout.flush() + ino, expected_bt = make_abc(ceph, rooti, i) + ceph = flush(ceph, i) + verify(ioctx, ino, expected_bt, 0) + +i += 1 + +ioctx.close() +radosobj.shutdown() +ceph.shutdown() + +print("done") +sys.stdout.flush() diff --git a/qa/workunits/rgw/common.py b/qa/workunits/rgw/common.py new file mode 100755 index 000000000..2c9c5d035 --- /dev/null +++ b/qa/workunits/rgw/common.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +import errno +import subprocess +import logging as log +import boto3 +import botocore.exceptions +import random +import json +from time import sleep + +log.basicConfig(format = '%(message)s', level=log.DEBUG) +log.getLogger('botocore').setLevel(log.CRITICAL) +log.getLogger('boto3').setLevel(log.CRITICAL) +log.getLogger('urllib3').setLevel(log.CRITICAL) + +def exec_cmd(cmd, wait = True, **kwargs): + check_retcode = kwargs.pop('check_retcode', True) + kwargs['shell'] = True + kwargs['stdout'] = subprocess.PIPE + proc = subprocess.Popen(cmd, **kwargs) + log.info(proc.args) + if wait: + out, _ = proc.communicate() + if check_retcode: + assert(proc.returncode == 0) + return out + return (out, proc.returncode) + return '' + +def create_user(uid, display_name, access_key, secret_key): + _, ret = exec_cmd(f'radosgw-admin user create --uid {uid} --display-name "{display_name}" --access-key {access_key} --secret {secret_key}', check_retcode=False) + assert(ret == 0 or errno.EEXIST) + +def boto_connect(access_key, secret_key, config=None): + def try_connect(portnum, ssl, proto): + endpoint = proto + '://localhost:' + portnum + conn = boto3.resource('s3', + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + use_ssl=ssl, + endpoint_url=endpoint, + verify=False, + config=config, + ) + try: + list(conn.buckets.limit(1)) # just verify we can list buckets + except botocore.exceptions.ConnectionError as e: + print(e) + raise + print('connected to', endpoint) + return conn + try: + return try_connect('80', False, 'http') + except botocore.exceptions.ConnectionError: + try: # retry on non-privileged http port + return try_connect('8000', False, 'http') + except botocore.exceptions.ConnectionError: + # retry with ssl + return try_connect('443', True, 'https') + +def put_objects(bucket, key_list): + objs = [] + for key in key_list: + o = bucket.put_object(Key=key, Body=b"some_data") + objs.append((o.key, o.version_id)) + return objs + +def create_unlinked_objects(conn, bucket, key_list): + # creates an unlinked/unlistable object for each key in key_list + + object_versions = [] + try: + exec_cmd('ceph config set client rgw_debug_inject_set_olh_err 2') + exec_cmd('ceph config set client rgw_debug_inject_olh_cancel_modification_err true') + sleep(1) + for key in key_list: + tag = str(random.randint(0, 1_000_000)) + try: + bucket.put_object(Key=key, Body=b"some_data", Metadata = { + 'tag': tag, + }) + except Exception as e: + log.debug(e) + out = exec_cmd(f'radosgw-admin bi list --bucket {bucket.name} --object {key}') + instance_entries = filter( + lambda x: x['type'] == 'instance', + json.loads(out.replace(b'\x80', b'0x80'))) + found = False + for ie in instance_entries: + instance_id = ie['entry']['instance'] + ov = conn.ObjectVersion(bucket.name, key, instance_id).head() + if ov['Metadata'] and ov['Metadata']['tag'] == tag: + object_versions.append((key, instance_id)) + found = True + break + if not found: + raise Exception(f'failed to create unlinked object for key={key}') + finally: + exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err') + exec_cmd('ceph config rm client rgw_debug_inject_olh_cancel_modification_err') + return object_versions + diff --git a/qa/workunits/rgw/keystone-fake-server.py b/qa/workunits/rgw/keystone-fake-server.py new file mode 100755 index 000000000..c05ad7bfd --- /dev/null +++ b/qa/workunits/rgw/keystone-fake-server.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 Binero +# +# Author: Tobias Urdin <tobias.urdin@binero.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. + +from datetime import datetime, timedelta +import logging +import json +from http.server import BaseHTTPRequestHandler, HTTPServer + + +DEFAULT_DOMAIN = { + 'id': 'default', + 'name': 'Default', +} + + +PROJECTS = { + 'admin': { + 'domain': DEFAULT_DOMAIN, + 'id': 'a6944d763bf64ee6a275f1263fae0352', + 'name': 'admin', + }, + 'deadbeef': { + 'domain': DEFAULT_DOMAIN, + 'id': 'b4221c214dd64ee6a464g2153fae3813', + 'name': 'deadbeef', + }, +} + + +USERS = { + 'admin': { + 'domain': DEFAULT_DOMAIN, + 'id': '51cc68287d524c759f47c811e6463340', + 'name': 'admin', + }, + 'deadbeef': { + 'domain': DEFAULT_DOMAIN, + 'id': '99gg485738df758349jf8d848g774392', + 'name': 'deadbeef', + }, +} + + +USERROLES = { + 'admin': [ + { + 'id': '51cc68287d524c759f47c811e6463340', + 'name': 'admin', + } + ], + 'deadbeef': [ + { + 'id': '98bd32184f854f393a72b932g5334124', + 'name': 'Member', + } + ], +} + + +TOKENS = { + 'admin-token-1': { + 'username': 'admin', + 'project': 'admin', + 'expired': False, + }, + 'user-token-1': { + 'username': 'deadbeef', + 'project': 'deadbeef', + 'expired': False, + }, + 'user-token-2': { + 'username': 'deadbeef', + 'project': 'deadbeef', + 'expired': True, + }, +} + + +def _generate_token_result(username, project, expired=False): + userdata = USERS[username] + projectdata = PROJECTS[project] + userroles = USERROLES[username] + + if expired: + then = datetime.now() - timedelta(hours=2) + issued_at = then.strftime('%Y-%m-%dT%H:%M:%SZ') + expires_at = (then + timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M:%SZ') + else: + now = datetime.now() + issued_at = now.strftime('%Y-%m-%dT%H:%M:%SZ') + expires_at = (now + timedelta(seconds=10)).strftime('%Y-%m-%dT%H:%M:%SZ') + + result = { + 'token': { + 'audit_ids': ['3T2dc1CGQxyJsHdDu1xkcw'], + 'catalog': [], + 'expires_at': expires_at, + 'is_domain': False, + 'issued_at': issued_at, + 'methods': ['password'], + 'project': projectdata, + 'roles': userroles, + 'user': userdata, + } + } + + return result + + +COUNTERS = { + 'get_total': 0, + 'post_total': 0, +} + + +class HTTPRequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + # This is not part of the Keystone API + if self.path == '/stats': + self._handle_stats() + return + + if str(self.path).startswith('/v3/auth/tokens'): + self._handle_get_auth() + else: + self.send_response(403) + self.end_headers() + + def do_POST(self): + if self.path == '/v3/auth/tokens': + self._handle_post_auth() + else: + self.send_response(400) + self.end_headers() + + def _get_data(self): + length = int(self.headers.get('content-length')) + data = self.rfile.read(length).decode('utf8') + return json.loads(data) + + def _set_data(self, data): + jdata = json.dumps(data) + self.wfile.write(jdata.encode('utf8')) + + def _handle_stats(self): + self.send_response(200) + self.end_headers() + self._set_data(COUNTERS) + + def _handle_get_auth(self): + logging.info('Increasing get_total counter from %d -> %d' % (COUNTERS['get_total'], COUNTERS['get_total']+1)) + COUNTERS['get_total'] += 1 + auth_token = self.headers.get('X-Subject-Token', None) + if auth_token and auth_token in TOKENS: + tokendata = TOKENS[auth_token] + if tokendata['expired'] and 'allow_expired=1' not in self.path: + self.send_response(404) + self.end_headers() + else: + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + result = _generate_token_result(tokendata['username'], tokendata['project'], tokendata['expired']) + self._set_data(result) + else: + self.send_response(404) + self.end_headers() + + def _handle_post_auth(self): + logging.info('Increasing post_total counter from %d -> %d' % (COUNTERS['post_total'], COUNTERS['post_total']+1)) + COUNTERS['post_total'] += 1 + data = self._get_data() + user = data['auth']['identity']['password']['user'] + if user['name'] == 'admin' and user['password'] == 'ADMIN': + self.send_response(201) + self.send_header('Content-Type', 'application/json') + self.send_header('X-Subject-Token', 'admin-token-1') + self.end_headers() + tokendata = TOKENS['admin-token-1'] + result = _generate_token_result(tokendata['username'], tokendata['project'], tokendata['expired']) + self._set_data(result) + else: + self.send_response(401) + self.end_headers() + + +def main(): + logging.basicConfig(level=logging.DEBUG) + logging.info('Starting keystone-fake-server') + server = HTTPServer(('localhost', 5000), HTTPRequestHandler) + server.serve_forever() + + +if __name__ == '__main__': + main() diff --git a/qa/workunits/rgw/keystone-service-token.sh b/qa/workunits/rgw/keystone-service-token.sh new file mode 100755 index 000000000..fc39731ca --- /dev/null +++ b/qa/workunits/rgw/keystone-service-token.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Copyright (C) 2022 Binero +# +# Author: Tobias Urdin <tobias.urdin@binero.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +trap cleanup EXIT + +function cleanup() { + kill $KEYSTONE_FAKE_SERVER_PID + wait +} + +function run() { + $CEPH_ROOT/qa/workunits/rgw//keystone-fake-server.py & + KEYSTONE_FAKE_SERVER_PID=$! + # Give fake Keystone server some seconds to startup + sleep 5 + $CEPH_ROOT/qa/workunits/rgw/test-keystone-service-token.py +} + +main keystone-service-token "$@" diff --git a/qa/workunits/rgw/olh_noname_key b/qa/workunits/rgw/olh_noname_key new file mode 100644 index 000000000..6138c57cd --- /dev/null +++ b/qa/workunits/rgw/olh_noname_key @@ -0,0 +1 @@ +€1001_04/57/0457f727ec113e418d5b16d206b200ed068c0533554883ce811df7c932a3df68/2018_12_11/2889999/3386469/metadata.gz
\ No newline at end of file diff --git a/qa/workunits/rgw/olh_noname_val b/qa/workunits/rgw/olh_noname_val Binary files differnew file mode 100644 index 000000000..ff442e137 --- /dev/null +++ b/qa/workunits/rgw/olh_noname_val diff --git a/qa/workunits/rgw/run-bucket-check.sh b/qa/workunits/rgw/run-bucket-check.sh new file mode 100755 index 000000000..85e02db5e --- /dev/null +++ b/qa/workunits/rgw/run-bucket-check.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +# assume working ceph environment (radosgw-admin in path) and rgw on localhost:80 +# localhost::443 for ssl + +mydir=`dirname $0` + +python3 -m venv $mydir +source $mydir/bin/activate +pip install pip --upgrade +pip install boto3 + +## run test +$mydir/bin/python3 $mydir/test_rgw_bucket_check.py + +deactivate +echo OK. + diff --git a/qa/workunits/rgw/run-datacache.sh b/qa/workunits/rgw/run-datacache.sh new file mode 100755 index 000000000..5c00da1da --- /dev/null +++ b/qa/workunits/rgw/run-datacache.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +#assume working ceph environment (radosgw-admin in path) and rgw on localhost:80 +# localhost::443 for ssl + +mydir=`dirname $0` + +python3 -m venv $mydir +source $mydir/bin/activate +pip install pip --upgrade +pip install configobj + +## run test +$mydir/bin/python3 $mydir/test_rgw_datacache.py + +deactivate +echo OK. + diff --git a/qa/workunits/rgw/run-reshard.sh b/qa/workunits/rgw/run-reshard.sh new file mode 100755 index 000000000..bdab0aabb --- /dev/null +++ b/qa/workunits/rgw/run-reshard.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -ex + +# this test uses fault injection to abort during 'radosgw-admin bucket reshard' +# disable coredumps so teuthology won't mark a failure +ulimit -c 0 + +#assume working ceph environment (radosgw-admin in path) and rgw on localhost:80 +# localhost::443 for ssl + +mydir=`dirname $0` + +python3 -m venv $mydir +source $mydir/bin/activate +pip install pip --upgrade +pip install boto3 + +## run test +$mydir/bin/python3 $mydir/test_rgw_reshard.py + +deactivate +echo OK. + diff --git a/qa/workunits/rgw/run-s3tests.sh b/qa/workunits/rgw/run-s3tests.sh new file mode 100755 index 000000000..727bef9eb --- /dev/null +++ b/qa/workunits/rgw/run-s3tests.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -ex + +# run s3-tests from current directory. assume working +# ceph environment (radosgw-admin in path) and rgw on localhost:8000 +# (the vstart default). + +branch=$1 +[ -z "$1" ] && branch=master +port=$2 +[ -z "$2" ] && port=8000 # this is vstart's default + +## + +[ -z "$BUILD_DIR" ] && BUILD_DIR=build + +if [ -e CMakeCache.txt ]; then + BIN_PATH=$PWD/bin +elif [ -e $root_path/../${BUILD_DIR}/CMakeCache.txt ]; then + cd $root_path/../${BUILD_DIR} + BIN_PATH=$PWD/bin +fi +PATH=$PATH:$BIN_PATH + +dir=tmp.s3-tests.$$ + +# clone and bootstrap +mkdir $dir +cd $dir +git clone https://github.com/ceph/s3-tests +cd s3-tests +git checkout ceph-$branch +S3TEST_CONF=s3tests.conf.SAMPLE tox -- -m "not fails_on_rgw and not sse_s3 and not lifecycle_expiration and not test_of_sts and not webidentity_test" -v + +cd ../.. +rm -rf $dir + +echo OK. + diff --git a/qa/workunits/rgw/run-versioning.sh b/qa/workunits/rgw/run-versioning.sh new file mode 100755 index 000000000..df60b7b03 --- /dev/null +++ b/qa/workunits/rgw/run-versioning.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +# assume working ceph environment (radosgw-admin in path) and rgw on localhost:80 +# localhost::443 for ssl + +mydir=`dirname $0` + +python3 -m venv $mydir +source $mydir/bin/activate +pip install pip --upgrade +pip install boto3 + +## run test +$mydir/bin/python3 $mydir/test_rgw_versioning.py + +deactivate +echo OK. + diff --git a/qa/workunits/rgw/s3_bucket_quota.pl b/qa/workunits/rgw/s3_bucket_quota.pl new file mode 100755 index 000000000..7f5476ef6 --- /dev/null +++ b/qa/workunits/rgw/s3_bucket_quota.pl @@ -0,0 +1,393 @@ +#! /usr/bin/perl + +=head1 NAME + +s3_bucket_quota.pl - Script to test the rgw bucket quota functionality using s3 interface. + +=head1 SYNOPSIS + +Use: + perl s3_bucket_quota.pl [--help] + +Examples: + perl s3_bucket_quota.pl + or + perl s3_bucket_quota.pl --help + +=head1 DESCRIPTION + +This script intends to test the rgw bucket quota funcionality using s3 interface +and reports the test results + +=head1 ARGUMENTS + +s3_bucket_quota.pl takes the following arguments: + --help + (optional) Displays the usage message. + +=cut + +use Amazon::S3; +use Data::Dumper; +#use strict; +use IO::File; +use Getopt::Long; +use Digest::MD5; +use Pod::Usage(); +use FindBin; +use lib $FindBin::Bin; +use s3_utilities; +use Net::Domain qw(hostfqdn); + +my $help; + +Getopt::Long::GetOptions( + 'help' => \$help +); +Pod::Usage::pod2usage(-verbose => 1) && exit if ($help); + +#== local variables === +our $mytestfilename; +my $mytestfilename1; +my $logmsg; +my $kruft; +my $s3; +my $hostdom = $ENV{RGW_FQDN}||hostfqdn(); +my $port = $ENV{RGW_PORT}||80; +our $hostname = "$hostdom:$port"; +our $testfileloc; +my $rgw_user = "qa_user"; + +# Function that deletes the user $rgw_user and write to logfile. +sub delete_user +{ + my $cmd = "$radosgw_admin user rm --uid=$rgw_user"; + my $cmd_op = get_command_output($cmd); + if ($cmd_op !~ /aborting/){ + print "user $rgw_user deleted\n"; + } else { + print "user $rgw_user NOT deleted\n"; + return 1; + } + return 0; +} + +sub quota_set_max_size { + my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-size=1048576000`; + if ($set_quota !~ /./){ + print "quota set for the bucket: $bucketname \n"; + } else { + print "quota set failed for the bucket: $bucketname \n"; + exit 1; + } + return 0; +} + +sub quota_set_max_size_zero { + run_s3($rgw_user); + my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-size=0`; + if ($set_quota !~ /./){ + pass ("quota set for the bucket: $bucketname with max size as zero\n"); + } else { + fail ("quota set with max size 0 failed for the bucket: $bucketname \n"); + } + delete_bucket(); +} + +sub quota_set_max_objs_zero { + run_s3($rgw_user); + my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-objects=0`; + if ($set_quota !~ /./){ + pass ("quota set for the bucket: $bucketname with max objects as zero\n"); + } else { + fail ("quota set with max objects 0 failed for the bucket: $bucketname \n"); + } + delete_bucket(); +} + +sub quota_set_neg_size { + run_s3($rgw_user); + my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-size=-1`; + if ($set_quota !~ /./){ + pass ("quota set for the bucket: $bucketname with max size -1\n"); + } else { + fail ("quota set failed for the bucket: $bucketname with max size -1 \n"); + } + delete_bucket(); +} + +sub quota_set_neg_objs { + run_s3($rgw_user); + my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-objects=-1`; + if ($set_quota !~ /./){ + pass ("quota set for the bucket: $bucketname max objects -1 \n"); + } else { + fail ("quota set failed for the bucket: $bucketname \n with max objects -1"); + } + delete_bucket(); +} + +sub quota_set_user_objs { + my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=bucket`; + my $set_quota1 = `$radosgw_admin quota set --bucket=$bucketname --max-objects=1`; + if ($set_quota1 !~ /./){ + print "bucket quota max_objs set for the given user: $bucketname \n"; + } else { + print "bucket quota max_objs set failed for the given user: $bucketname \n"; + exit 1; + } + return 0; +} + +sub quota_set_user_size { + my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=bucket`; + my $set_quota1 = `$radosgw_admin quota set --bucket=$bucketname --max-size=1048576000`; + if ($set_quota1 !~ /./){ + print "bucket quota max size set for the given user: $bucketname \n"; + } else { + print "bucket quota max size set failed for the user: $bucketname \n"; + exit 1; + } + return 0; +} + +sub quota_set_max_obj { + # set max objects + my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-objects=1`; + if ($set_quota !~ /./){ + print "quota set for the bucket: $bucketname \n"; + } else { + print "quota set failed for the bucket: $bucketname \n"; + exit 1; + } + return 0; +} + +sub quota_enable { + my $en_quota = `$radosgw_admin quota enable --bucket=$bucketname`; + if ($en_quota !~ /./){ + print "quota enabled for the bucket: $bucketname \n"; + } else { + print "quota enable failed for the bucket: $bucketname \n"; + exit 1; + } + return 0; +} + +sub quota_disable { + my $dis_quota = `$radosgw_admin quota disable --bucket=$bucketname`; + if ($dis_quota !~ /./){ + print "quota disabled for the bucket: $bucketname \n"; + } else { + print "quota disable failed for the bucket: $bucketname \n"; + exit 1; + } + return 0; +} + +# upload a file to the bucket +sub upload_file { + print "adding file to bucket: $mytestfilename\n"; + ($bucket->add_key_filename( $mytestfilename, $testfileloc, + { content_type => 'text/plain', }, + ) and (print "upload file successful\n" ) and return 0 ) or (return 1); +} + +# delete the bucket +sub delete_bucket { + #($bucket->delete_key($mytestfilename1) and print "delete keys on bucket succeeded second time\n" ) or die $s3->err . "delete keys on bucket failed second time\n" . $s3->errstr; + ($bucket->delete_bucket) and (print "bucket delete succeeded \n") or die $s3->err . "delete bucket failed\n" . $s3->errstr; +} + +# set bucket quota with max_objects and verify +sub test_max_objects { + my $size = '10Mb'; + create_file($size); + run_s3($rgw_user); + quota_set_max_obj(); + quota_enable(); + my $ret_value = upload_file(); + if ($ret_value == 0){ + pass ( "Test max objects passed" ); + } else { + fail ( "Test max objects failed" ); + } + delete_user(); + delete_keys($mytestfilename); + delete_bucket(); +} + +# Set bucket quota for specific user and ensure max objects set for the user is validated +sub test_max_objects_per_user{ + my $size = '10Mb'; + create_file($size); + run_s3($rgw_user); + quota_set_user_objs(); + quota_enable(); + my $ret_value = upload_file(); + if ($ret_value == 0){ + pass ( "Test max objects for the given user passed" ); + } else { + fail ( "Test max objects for the given user failed" ); + } + delete_user(); + delete_keys($mytestfilename); + delete_bucket(); +} + +# set bucket quota with max_objects and try to exceed the max_objects and verify +sub test_beyond_max_objs { + my $size = "10Mb"; + create_file($size); + run_s3($rgw_user); + quota_set_max_obj(); + quota_enable(); + upload_file(); + my $ret_value = readd_file(); + if ($ret_value == 1){ + pass ( "set max objects and test beyond max objects passed" ); + } else { + fail ( "set max objects and test beyond max objects failed" ); + } + delete_user(); + delete_keys($mytestfilename); + delete_bucket(); +} + +# set bucket quota for a user with max_objects and try to exceed the max_objects and verify +sub test_beyond_max_objs_user { + my $size = "10Mb"; + create_file($size); + run_s3($rgw_user); + quota_set_user_objs(); + quota_enable(); + upload_file(); + my $ret_value = readd_file(); + if ($ret_value == 1){ + pass ( "set max objects for a given user and test beyond max objects passed" ); + } else { + fail ( "set max objects for a given user and test beyond max objects failed" ); + } + delete_user(); + delete_keys($mytestfilename); + delete_bucket(); +} + +# set bucket quota for max size and ensure it is validated +sub test_quota_size { + my $ret_value; + my $size = "2Gb"; + create_file($size); + run_s3($rgw_user); + quota_set_max_size(); + quota_enable(); + my $ret_value = upload_file(); + if ($ret_value == 1) { + pass ( "set max size and ensure that objects upload beyond max size is not entertained" ); + my $retdel = delete_keys($mytestfilename); + if ($retdel == 0) { + print "delete objects successful \n"; + my $size1 = "1Gb"; + create_file($size1); + my $ret_val1 = upload_file(); + if ($ret_val1 == 0) { + pass ( "set max size and ensure that the max size is in effect" ); + } else { + fail ( "set max size and ensure the max size takes effect" ); + } + } + } else { + fail ( "set max size and ensure that objects beyond max size is not allowed" ); + } + delete_user(); + delete_keys($mytestfilename); + delete_bucket(); +} + +# set bucket quota for max size for a given user and ensure it is validated +sub test_quota_size_user { + my $ret_value; + my $size = "2Gb"; + create_file($size); + run_s3($rgw_user); + quota_set_user_size(); + quota_enable(); + my $ret_value = upload_file(); + if ($ret_value == 1) { + pass ( "set max size for a given user and ensure that objects upload beyond max size is not entertained" ); + my $retdel = delete_keys($mytestfilename); + if ($retdel == 0) { + print "delete objects successful \n"; + my $size1 = "1Gb"; + create_file($size1); + my $ret_val1 = upload_file(); + if ($ret_val1 == 0) { + pass ( "set max size for a given user and ensure that the max size is in effect" ); + } else { + fail ( "set max size for a given user and ensure the max size takes effect" ); + } + } + } else { + fail ( "set max size for a given user and ensure that objects beyond max size is not allowed" ); + } + delete_user(); + delete_keys($mytestfilename); + delete_bucket(); +} + +# set bucket quota size but disable quota and verify +sub test_quota_size_disabled { + my $ret_value; + my $size = "2Gb"; + create_file($size); + run_s3($rgw_user); + quota_set_max_size(); + quota_disable(); + my $ret_value = upload_file(); + if ($ret_value == 0) { + pass ( "bucket quota size doesnt take effect when quota is disabled" ); + } else { + fail ( "bucket quota size doesnt take effect when quota is disabled" ); + } + delete_user(); + delete_keys($mytestfilename); + delete_bucket(); +} + +# set bucket quota size for a given user but disable quota and verify +sub test_quota_size_disabled_user { + my $ret_value; + my $size = "2Gb"; + create_file($size); + run_s3($rgw_user); + quota_set_user_size(); + quota_disable(); + my $ret_value = upload_file(); + if ($ret_value == 0) { + pass ( "bucket quota size for a given user doesnt take effect when quota is disabled" ); + } else { + fail ( "bucket quota size for a given user doesnt take effect when quota is disabled" ); + } + delete_user(); + delete_keys($mytestfilename); + delete_bucket(); +} + +# set bucket quota for specified user and verify + +#== Main starts here=== +ceph_os_info(); +test_max_objects(); +test_max_objects_per_user(); +test_beyond_max_objs(); +test_beyond_max_objs_user(); +quota_set_max_size_zero(); +quota_set_max_objs_zero(); +quota_set_neg_objs(); +quota_set_neg_size(); +test_quota_size(); +test_quota_size_user(); +test_quota_size_disabled(); +test_quota_size_disabled_user(); + +print "OK"; diff --git a/qa/workunits/rgw/s3_multipart_upload.pl b/qa/workunits/rgw/s3_multipart_upload.pl new file mode 100755 index 000000000..ab29e6b03 --- /dev/null +++ b/qa/workunits/rgw/s3_multipart_upload.pl @@ -0,0 +1,151 @@ +#! /usr/bin/perl + +=head1 NAME + +s3_multipart_upload.pl - Script to test rgw multipart upload using s3 interface. + +=head1 SYNOPSIS + +Use: + perl s3_multipart_upload.pl [--help] + +Examples: + perl s3_multipart_upload.pl + or + perl s3_multipart_upload.pl --help + +=head1 DESCRIPTION + +This script intends to test the rgw multipart upload followed by a download +and verify checksum using s3 interface and reports test results + +=head1 ARGUMENTS + +s3_multipart_upload.pl takes the following arguments: + --help + (optional) Displays the usage message. + +=cut + +use Amazon::S3; +use Data::Dumper; +use IO::File; +use Getopt::Long; +use Digest::MD5; +use Pod::Usage(); +use FindBin; +use lib $FindBin::Bin; +use s3_utilities; +use Net::Domain qw(hostfqdn); + +my $help; + +Getopt::Long::GetOptions( + 'help' => \$help +); +Pod::Usage::pod2usage(-verbose => 1) && exit if ($help); + +#== local variables === +my $s3; +my $hostdom = $ENV{RGW_FQDN}||hostfqdn(); +my $port = $ENV{RGW_PORT}||80; +our $hostname = "$hostdom:$port"; +our $testfileloc; +our $mytestfilename; + +# upload a file to the bucket +sub upload_file { + my ($fsize, $i) = @_; + create_file($fsize, $i); + print "adding file to bucket $bucketname: $mytestfilename\n"; + ($bucket->add_key_filename( $mytestfilename, $testfileloc, + { content_type => 'text/plain', }, + ) and (print "upload file successful\n" ) and return 0 ) or (print "upload failed\n" and return 1); +} + +# delete the bucket +sub delete_bucket { + ($bucket->delete_bucket) and (print "bucket delete succeeded \n") or die $s3->err . "delete bucket failed\n" . $s3->errstr; +} + +# Function to perform multipart upload of given file size to the user bucket via s3 interface +sub multipart_upload +{ + my ($size, $parts) = @_; + # generate random user every time + my $user = rand(); + # Divide the file size in to equal parts and upload to bucket in multiple parts + my $fsize = ($size/$parts); + my $fsize1; + run_s3($user); + if ($parts == 10){ + $fsize1 = '100Mb'; + } elsif ($parts == 100){ + $fsize1 = '10Mb'; + } + foreach my $i(1..$parts){ + print "uploading file - part $i \n"; + upload_file($fsize1, $i); + } + fetch_file_from_bucket($fsize1, $parts); + compare_cksum($fsize1, $parts); + purge_data($user); +} + +# Function to download the files from bucket to verify there is no data corruption +sub fetch_file_from_bucket +{ + # fetch file from the bucket + my ($fsize, $parts) = @_; + foreach my $i(1..$parts){ + my $src_file = "$fsize.$i"; + my $dest_file = "/tmp/downloadfile.$i"; + print + "Downloading $src_file from bucket to $dest_file \n"; + $response = + $bucket->get_key_filename( $src_file, GET, + $dest_file ) + or die $s3->err . ": " . $s3->errstr; + } +} + +# Compare the source file with destination file and verify checksum to ensure +# the files are not corrupted +sub compare_cksum +{ + my ($fsize, $parts)=@_; + my $md5 = Digest::MD5->new; + my $flag = 0; + foreach my $i (1..$parts){ + my $src_file = "/tmp/"."$fsize".".$i"; + my $dest_file = "/tmp/downloadfile".".$i"; + open( FILE, $src_file ) + or die "Error: Could not open $src_file for MD5 checksum..."; + open( DLFILE, $dest_file ) + or die "Error: Could not open $dest_file for MD5 checksum."; + binmode(FILE); + binmode(DLFILE); + my $md5sum = $md5->addfile(*FILE)->hexdigest; + my $md5sumdl = $md5->addfile(*DLFILE)->hexdigest; + close FILE; + close DLFILE; + # compare the checksums + if ( $md5sum eq $md5sumdl ) { + $flag++; + } + } + if ($flag == $parts){ + pass("checksum verification for multipart upload passed" ); + }else{ + fail("checksum verification for multipart upload failed" ); + } +} + +#== Main starts here=== +ceph_os_info(); +check(); +# The following test runs multi part upload of file size 1Gb in 10 parts +multipart_upload('1048576000', 10); +# The following test runs multipart upload of 1 Gb file in 100 parts +multipart_upload('1048576000', 100); +print "OK"; diff --git a/qa/workunits/rgw/s3_user_quota.pl b/qa/workunits/rgw/s3_user_quota.pl new file mode 100755 index 000000000..6d5c02a9a --- /dev/null +++ b/qa/workunits/rgw/s3_user_quota.pl @@ -0,0 +1,191 @@ +#! /usr/bin/perl + +=head1 NAME + +s3_user_quota.pl - Script to test the rgw user quota functionality using s3 interface. + +=head1 SYNOPSIS + +Use: + perl s3_user_quota.pl [--help] + +Examples: + perl s3_user_quota.pl + or + perl s3_user_quota.pl --help + +=head1 DESCRIPTION + +This script intends to test the rgw user quota funcionality using s3 interface +and reports the test results + +=head1 ARGUMENTS + +s3_user_quota.pl takes the following arguments: + --help + (optional) Displays the usage message. + +=cut + +use Amazon::S3; +use Data::Dumper; +use IO::File; +use Getopt::Long; +use Digest::MD5; +use Pod::Usage(); +use FindBin; +use lib $FindBin::Bin; +use s3_utilities; +use Net::Domain qw(hostfqdn); + +my $help; + +Getopt::Long::GetOptions( + 'help' => \$help +); +Pod::Usage::pod2usage(-verbose => 1) && exit if ($help); + +#== local variables === +our $mytestfilename; +my $mytestfilename1; +my $logmsg; +my $kruft; +my $s3; +my $hostdom = $ENV{RGW_FQDN}||hostfqdn(); +my $port = $ENV{RGW_PORT}||80; +our $hostname = "$hostdom:$port"; +our $testfileloc; +our $cnt; + +sub quota_set_max_size_per_user { + my ($maxsize, $size1,$rgw_user) = @_; + run_s3($rgw_user); + my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=user --max-size=$maxsize`; + if (($set_quota !~ /./)&&($maxsize == 0)){ + my $ret = test_max_objs($size1, $rgw_user); + if ($ret == 1){ + pass("quota set for user: $rgw_user with max_size=$maxsize passed" ); + }else { + fail("quota set for user: $rgw_user with max_size=$maxsize failed" ); + } + } elsif (($set_quota !~ /./) && ($maxsize != 0)) { + my $ret = test_max_objs($size1, $rgw_user); + if ($ret == 0){ + pass("quota set for user: $rgw_user with max_size=$maxsize passed" ); + }else { + fail("quota set for user: $rgw_user with max_size=$maxsize failed" ); + } + } + delete_keys($mytestfilename); + purge_data($rgw_user); + return 0; +} + +sub max_size_per_user { + my ($maxsize, $size1,$rgw_user) = @_; + run_s3($rgw_user); + my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=user --max-size=$maxsize`; + if (($set_quota !~ /./) && ($maxsize != 0)) { + my $ret = test_max_objs($size1, $rgw_user); + if ($ret == 0){ + $cnt++; + } + } + return $cnt; +} + +sub quota_set_max_obj_per_user { + # set max objects + my ($maxobjs, $size1, $rgw_user) = @_; + run_s3($rgw_user); + my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=user --max-objects=$maxobjs`; + if (($set_quota !~ /./) && ($maxobjs == 0)){ + my $ret = test_max_objs($size1, $rgw_user); + if ($ret == 1){ + pass("quota set for user: $rgw_user with max_objects=$maxobjs passed" ); + }else { + fail("quota set for user: $rgw_user with max_objects=$maxobjs failed" ); + } + } elsif (($set_quota !~ /./) && ($maxobjs == 1)) { + my $ret = test_max_objs($size1, $rgw_user); + if ($ret == 0){ + pass("quota set for user: $rgw_user with max_objects=$maxobjs passed" ); + }else { + fail("quota set for user: $rgw_user with max_objects=$maxobjs failed" ); + } + } + delete_keys($mytestfilename); + purge_data($rgw_user); +} + +sub quota_enable_user { + my ($rgw_user) = @_; + my $en_quota = `$radosgw_admin quota enable --uid=$rgw_user --quota-scope=user`; + if ($en_quota !~ /./){ + print "quota enabled for the user $rgw_user \n"; + } else { + print "quota enable failed for the user $rgw_user \n"; + exit 1; + } + return 0; +} + +sub quota_disable_user { + my $dis_quota = `$radosgw_admin quota disable --uid=$rgw_user --quota-scope=user`; + if ($dis_quota !~ /./){ + print "quota disabled for the user $rgw_user \n"; + } else { + print "quota disable failed for the user $rgw_user \n"; + exit 1; + } + return 0; +} + +# upload a file to the bucket +sub upload_file { + print "adding file to bucket $bucketname: $mytestfilename\n"; + ($bucket->add_key_filename( $mytestfilename, $testfileloc, + { content_type => 'text/plain', }, + ) and (print "upload file successful\n" ) and return 0 ) or (return 1); +} + +# delete the bucket +sub delete_bucket { + ($bucket->delete_bucket) and (print "bucket delete succeeded \n") or die $s3->err . "delete bucket failed\n" . $s3->errstr; +} + +#Function to upload the given file size to bucket and verify +sub test_max_objs { + my ($size, $rgw_user) = @_; + create_file($size); + quota_enable_user($rgw_user); + my $ret_value = upload_file(); + return $ret_value; +} + +# set user quota and ensure it is validated +sub test_user_quota_max_size{ + my ($max_buckets,$size, $fsize) = @_; + my $usr = rand(); + foreach my $i (1..$max_buckets){ + my $ret_value = max_size_per_user($size, $fsize, $usr ); + } + if ($ret_value == $max_buckets){ + fail( "user quota max size for $usr failed on $max_buckets buckets" ); + } else { + pass( "user quota max size for $usr passed on $max_buckets buckets" ); + } + delete_keys($mytestfilename); + purge_data($usr); +} + +#== Main starts here=== +ceph_os_info(); +check(); +quota_set_max_obj_per_user('0', '10Mb', 'usr1'); +quota_set_max_obj_per_user('1', '10Mb', 'usr2'); +quota_set_max_size_per_user(0, '10Mb', 'usr1'); +quota_set_max_size_per_user(1048576000, '1Gb', 'usr2'); +test_user_quota_max_size(3,1048576000,'100Mb'); +test_user_quota_max_size(2,1048576000, '1Gb'); +print "OK"; diff --git a/qa/workunits/rgw/s3_utilities.pm b/qa/workunits/rgw/s3_utilities.pm new file mode 100644 index 000000000..3c3fae900 --- /dev/null +++ b/qa/workunits/rgw/s3_utilities.pm @@ -0,0 +1,233 @@ +# Common subroutines shared by the s3 testing code +my $sec; +my $min; +my $hour; +my $mon; +my $year; +my $mday; +my $wday; +my $yday; +my $isdst; +my $PASS_CNT = 0; +my $FAIL_CNT = 0; + +our $radosgw_admin = $ENV{RGW_ADMIN}||"sudo radosgw-admin"; + +# function to get the current time stamp from the test set up +sub get_timestamp { + ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); + if ($mon < 10) { $mon = "0$mon"; } + if ($hour < 10) { $hour = "0$hour"; } + if ($min < 10) { $min = "0$min"; } + if ($sec < 10) { $sec = "0$sec"; } + $year=$year+1900; + return $year . '_' . $mon . '_' . $mday . '_' . $hour . '_' . $min . '_' . $sec; +} + +# Function to check if radosgw is already running +sub get_status { + my $service = "radosgw"; + my $cmd = "pgrep $service"; + my $status = get_cmd_op($cmd); + if ($status =~ /\d+/ ){ + return 0; + } + return 1; +} + +# function to execute the command and return output +sub get_cmd_op +{ + my $cmd = shift; + my $excmd = `$cmd`; + return $excmd; +} + +#Function that executes the CLI commands and returns the output of the command +sub get_command_output { + my $cmd_output = shift; + open( FH, ">>$test_log" ); + print FH "\"$cmd_output\"\n"; + my $exec_cmd = `$cmd_output 2>&1`; + print FH "$exec_cmd\n"; + close(FH); + return $exec_cmd; +} + +# Function to get the hostname +sub get_hostname +{ + my $cmd = "hostname"; + my $get_host = get_command_output($cmd); + chomp($get_host); + return($get_host); +} + +sub pass { + my ($comment) = @_; + print "Comment required." unless length $comment; + chomp $comment; + print_border2(); + print "Test case: $TC_CNT PASSED - $comment \n"; + print_border2(); + $PASS_CNT++; +} + +sub fail { + my ($comment) = @_; + print "Comment required." unless length $comment; + chomp $comment; + print_border2(); + print "Test case: $TC_CNT FAILED - $comment \n"; + print_border2(); + $FAIL_CNT++; +} + +sub print_border2 { + print "~" x 90 . "\n"; +} + +# Function to create the user "qa_user" and extract the user access_key and secret_key of the user +sub get_user_info +{ + my ($rgw_user) = @_; + my $cmd = "$radosgw_admin user create --uid=$rgw_user --display-name=$rgw_user"; + my $cmd_op = get_command_output($cmd); + if ($cmd_op !~ /keys/){ + return (0,0); + } + my @get_user = (split/\n/,$cmd_op); + foreach (@get_user) { + if ($_ =~ /access_key/ ){ + $get_acc_key = $_; + } elsif ($_ =~ /secret_key/ ){ + $get_sec_key = $_; + } + } + my $access_key = $get_acc_key; + my $acc_key = (split /:/, $access_key)[1]; + $acc_key =~ s/\\//g; + $acc_key =~ s/ //g; + $acc_key =~ s/"//g; + $acc_key =~ s/,//g; + my $secret_key = $get_sec_key; + my $sec_key = (split /:/, $secret_key)[1]; + $sec_key =~ s/\\//g; + $sec_key =~ s/ //g; + $sec_key =~ s/"//g; + $sec_key =~ s/,//g; + return ($acc_key, $sec_key); +} + +# Function that deletes the given user and all associated user data +sub purge_data +{ + my ($rgw_user) = @_; + my $cmd = "$radosgw_admin user rm --uid=$rgw_user --purge-data"; + my $cmd_op = get_command_output($cmd); + if ($cmd_op !~ /./){ + print "user $rgw_user deleted\n"; + } else { + print "user $rgw_user NOT deleted\n"; + return 1; + } + return 0; +} + +# Read PRETTY_NAME from /etc/os-release +sub os_pretty_name +{ + open(FH, '<', '/etc/os-release') or die $!; + while (my $line = <FH>) { + chomp $line; + if ($line =~ /^\s*PRETTY_NAME=\"?([^"]*)\"?/) { + return $1; + } + } + close(FH); +} + + +# Function to get the Ceph and distro info +sub ceph_os_info +{ + my $ceph_v = get_command_output ( "ceph -v" ); + my @ceph_arr = split(" ",$ceph_v); + $ceph_v = "Ceph Version: $ceph_arr[2]"; + my $os_distro = os_pretty_name(); + $os_distro = "Linux Flavor:$os_distro"; + return ($ceph_v, $os_distro); +} + +# Execute the test case based on the input to the script +sub create_file { + my ($file_size, $part) = @_; + my $cnt; + $mytestfilename = "$file_size.$part"; + $testfileloc = "/tmp/".$mytestfilename; + if ($file_size == '10Mb'){ + $cnt = 1; + } elsif ($file_size == '100Mb'){ + $cnt = 10; + } elsif ($file_size == '500Mb'){ + $cnt = 50; + } elsif ($file_size == '1Gb'){ + $cnt = 100; + } elsif ($file_size == '2Gb'){ + $cnt = 200; + } + my $ret = system("dd if=/dev/zero of=$testfileloc bs=10485760 count=$cnt"); + if ($ret) { exit 1 }; + return 0; +} + +sub run_s3 +{ +# Run tests for the S3 functionality + # Modify access key and secret key to suit the user account + my ($user) = @_; + our ( $access_key, $secret_key ) = get_user_info($user); + if ( ($access_key) && ($secret_key) ) { + $s3 = Amazon::S3->new( + { + aws_access_key_id => $access_key, + aws_secret_access_key => $secret_key, + host => $hostname, + secure => 0, + retry => 1, + } + ); + } + +our $bucketname = 'buck_'.get_timestamp(); +# create a new bucket (the test bucket) +our $bucket = $s3->add_bucket( { bucket => $bucketname } ) + or die $s3->err. "bucket $bucketname create failed\n". $s3->errstr; + print "Bucket Created: $bucketname \n"; + return 0; +} + +# delete keys +sub delete_keys { + (($bucket->delete_key($_[0])) and return 0) or return 1; +} + +# Read the file back to bucket +sub readd_file { + system("dd if=/dev/zero of=/tmp/10MBfile1 bs=10485760 count=1"); + $mytestfilename1 = '10MBfile1'; + print "readding file to bucket: $mytestfilename1\n"; + ((($bucket->add_key_filename( $mytestfilename1, $testfileloc, + { content_type => 'text/plain', }, + )) and (print "readding file success\n") and return 0) or (return 1)); +} + +# check if rgw service is already running +sub check +{ + my $state = get_status(); + if ($state) { + exit 1; + } +} +1 diff --git a/qa/workunits/rgw/test-keystone-service-token.py b/qa/workunits/rgw/test-keystone-service-token.py new file mode 100755 index 000000000..2c7f21e93 --- /dev/null +++ b/qa/workunits/rgw/test-keystone-service-token.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 Binero +# +# Author: Tobias Urdin <tobias.urdin@binero.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. + +import sys +import requests +import time + + +# b4221c214dd64ee6a464g2153fae3813 is ID of deadbeef project +SWIFT_URL = 'http://localhost:8000/swift/v1/AUTH_b4221c214dd64ee6a464g2153fae3813' +KEYSTONE_URL = 'http://localhost:5000' + + +def get_stats(): + stats_url = '%s/stats' % KEYSTONE_URL + return requests.get(stats_url) + + +def test_list_containers(): + # Loop five list container requests with same token + for i in range(0, 5): + r = requests.get(SWIFT_URL, headers={'X-Auth-Token': 'user-token-1'}) + if r.status_code != 204: + print('FAILED, status code is %d not 204' % r.status_code) + sys.exit(1) + + # Get stats from fake Keystone server + r = get_stats() + if r.status_code != 200: + print('FAILED, status code is %d not 200' % r.status_code) + sys.exit(1) + stats = r.json() + + # Verify admin token was cached + if stats['post_total'] != 1: + print('FAILED, post_total stat is %d not 1' % stats['post_total']) + sys.exit(1) + + # Verify user token was cached + if stats['get_total'] != 1: + print('FAILED, get_total stat is %d not 1' % stats['get_total']) + sys.exit(1) + + print('Wait for cache to be invalid') + time.sleep(11) + + r = requests.get(SWIFT_URL, headers={'X-Auth-Token': 'user-token-1'}) + if r.status_code != 204: + print('FAILED, status code is %d not 204' % r.status_code) + sys.exit(1) + + # Get stats from fake Keystone server + r = get_stats() + if r.status_code != 200: + print('FAILED, status code is %d not 200' % r.status_code) + sys.exit(1) + stats = r.json() + + if stats['post_total'] != 2: + print('FAILED, post_total stat is %d not 2' % stats['post_total']) + sys.exit(1) + + if stats['get_total'] != 2: + print('FAILED, get_total stat is %d not 2' % stats['get_total']) + sys.exit(1) + + +def test_expired_token(): + # Try listing containers with an expired token + for i in range(0, 3): + r = requests.get(SWIFT_URL, headers={'X-Auth-Token': 'user-token-2'}) + if r.status_code != 401: + print('FAILED, status code is %d not 401' % r.status_code) + sys.exit(1) + + # Get stats from fake Keystone server + r = get_stats() + if r.status_code != 200: + print('FAILED, status code is %d not 200' % r.status_code) + sys.exit(1) + stats = r.json() + + # Verify admin token was cached + if stats['post_total'] != 2: + print('FAILED, post_total stat is %d not 2' % stats['post_total']) + sys.exit(1) + + # Verify we got to fake Keystone server since expired tokens is not cached + if stats['get_total'] != 5: + print('FAILED, get_total stat is %d not 5' % stats['get_total']) + sys.exit(1) + + +def test_expired_token_with_service_token(): + # Try listing containers with an expired token but with a service token + for i in range(0, 3): + r = requests.get(SWIFT_URL, headers={'X-Auth-Token': 'user-token-2', 'X-Service-Token': 'admin-token-1'}) + if r.status_code != 204: + print('FAILED, status code is %d not 204' % r.status_code) + sys.exit(1) + + # Get stats from fake Keystone server + r = get_stats() + if r.status_code != 200: + print('FAILED, status code is %d not 200' % r.status_code) + sys.exit(1) + stats = r.json() + + # Verify admin token was cached + if stats['post_total'] != 2: + print('FAILED, post_total stat is %d not 2' % stats['post_total']) + sys.exit(1) + + # Verify we got to fake Keystone server since expired tokens is not cached + if stats['get_total'] != 7: + print('FAILED, get_total stat is %d not 7' % stats['get_total']) + sys.exit(1) + + print('Wait for cache to be invalid') + time.sleep(11) + + r = requests.get(SWIFT_URL, headers={'X-Auth-Token': 'user-token-2', 'X-Service-Token': 'admin-token-1'}) + if r.status_code != 204: + print('FAILED, status code is %d not 204' % r.status_code) + sys.exit(1) + + # Get stats from fake Keystone server + r = get_stats() + if r.status_code != 200: + print('FAILED, status code is %d not 200' % r.status_code) + sys.exit(1) + stats = r.json() + + if stats['post_total'] != 3: + print('FAILED, post_total stat is %d not 3' % stats['post_total']) + sys.exit(1) + + if stats['get_total'] != 9: + print('FAILED, get_total stat is %d not 9' % stats['get_total']) + sys.exit(1) + + +def test_expired_token_with_invalid_service_token(): + print('Wait for cache to be invalid') + time.sleep(11) + + # Test with a token that doesn't have allowed role as service token + for i in range(0, 3): + r = requests.get(SWIFT_URL, headers={'X-Auth-Token': 'user-token-2', 'X-Service-Token': 'user-token-1'}) + if r.status_code != 401: + print('FAILED, status code is %d not 401' % r.status_code) + sys.exit(1) + + # Make sure we get user-token-1 cached + r = requests.get(SWIFT_URL, headers={'X-Auth-Token': 'user-token-1'}) + if r.status_code != 204: + print('FAILED, status code is %d not 204' % r.status_code) + sys.exit(1) + + # Test that a cached token (that is invalid as service token) cannot be used as service token + for i in range(0, 3): + r = requests.get(SWIFT_URL, headers={'X-Auth-Token': 'user-token-2', 'X-Service-Token': 'user-token-1'}) + if r.status_code != 401: + print('FAILED, status code is %d not 401' % r.status_code) + sys.exit(1) + + +def main(): + test_list_containers() + test_expired_token() + test_expired_token_with_service_token() + test_expired_token_with_invalid_service_token() + + +if __name__ == '__main__': + main() diff --git a/qa/workunits/rgw/test_librgw_file.sh b/qa/workunits/rgw/test_librgw_file.sh new file mode 100755 index 000000000..1371ff711 --- /dev/null +++ b/qa/workunits/rgw/test_librgw_file.sh @@ -0,0 +1,59 @@ +#!/bin/sh -e + + +if [ -z ${AWS_ACCESS_KEY_ID} ] +then + export AWS_ACCESS_KEY_ID=`openssl rand -base64 20` + export AWS_SECRET_ACCESS_KEY=`openssl rand -base64 40` + + radosgw-admin user create --uid ceph-test-librgw-file \ + --access-key $AWS_ACCESS_KEY_ID \ + --secret $AWS_SECRET_ACCESS_KEY \ + --display-name "librgw test user" \ + --email librgw@example.com || echo "librgw user exists" + + # keyring override for teuthology env + KEYRING="/etc/ceph/ceph.keyring" + K="-k ${KEYRING}" +fi + +# nfsns is the main suite + +# create herarchy, and then list it +echo "phase 1.1" +ceph_test_librgw_file_nfsns ${K} --hier1 --dirs1 --create --rename --verbose + +# the older librgw_file can consume the namespace +echo "phase 1.2" +ceph_test_librgw_file_nfsns ${K} --getattr --verbose + +# and delete the hierarchy +echo "phase 1.3" +ceph_test_librgw_file_nfsns ${K} --hier1 --dirs1 --delete --verbose + +# bulk create/delete buckets +echo "phase 2.1" +ceph_test_librgw_file_cd ${K} --create --multi --verbose +echo "phase 2.2" +ceph_test_librgw_file_cd ${K} --delete --multi --verbose + +# write continuation test +echo "phase 3.1" +ceph_test_librgw_file_aw ${K} --create --large --verify +echo "phase 3.2" +ceph_test_librgw_file_aw ${K} --delete --large + +# continued readdir +echo "phase 4.1" +ceph_test_librgw_file_marker ${K} --create --marker1 --marker2 --nobjs=100 --verbose +echo "phase 4.2" +ceph_test_librgw_file_marker ${K} --delete --verbose + +# advanced i/o--but skip readv/writev for now--split delete from +# create and stat ops to avoid fault in sysobject cache +echo "phase 5.1" +ceph_test_librgw_file_gp ${K} --get --stat --put --create +echo "phase 5.2" +ceph_test_librgw_file_gp ${K} --delete + +exit 0 diff --git a/qa/workunits/rgw/test_rgw_bucket_check.py b/qa/workunits/rgw/test_rgw_bucket_check.py new file mode 100755 index 000000000..bfa6d65d6 --- /dev/null +++ b/qa/workunits/rgw/test_rgw_bucket_check.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 + +import logging as log +import json +import botocore +from common import exec_cmd, create_user, boto_connect, put_objects, create_unlinked_objects +from botocore.config import Config + +""" +Tests behavior of radosgw-admin bucket check commands. +""" +# The test cases in this file have been annotated for inventory. +# To extract the inventory (in csv format) use the command: +# +# grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //' +# +# + +""" Constants """ +USER = 'check-tester' +DISPLAY_NAME = 'Check Testing' +ACCESS_KEY = 'OJODXSLNX4LUNHQG99PA' +SECRET_KEY = '3l6ffld34qaymfomuh832j94738aie2x4p2o8h6n' +BUCKET_NAME = 'check-bucket' + +def main(): + """ + execute bucket check commands + """ + create_user(USER, DISPLAY_NAME, ACCESS_KEY, SECRET_KEY) + + connection = boto_connect(ACCESS_KEY, SECRET_KEY, Config(retries = { + 'total_max_attempts': 1, + })) + + # pre-test cleanup + try: + bucket = connection.Bucket(BUCKET_NAME) + bucket.objects.all().delete() + bucket.object_versions.all().delete() + bucket.delete() + except botocore.exceptions.ClientError as e: + if not e.response['Error']['Code'] == 'NoSuchBucket': + raise + + bucket = connection.create_bucket(Bucket=BUCKET_NAME) + + null_version_keys = ['a', 'z'] + null_version_objs = put_objects(bucket, null_version_keys) + + connection.BucketVersioning(BUCKET_NAME).enable() + + ok_keys = ['a', 'b', 'c', 'd'] + unlinked_keys = ['c', 'd', 'e', 'f'] + ok_objs = put_objects(bucket, ok_keys) + + # TESTCASE 'recalculated bucket check stats are correct' + log.debug('TEST: recalculated bucket check stats are correct\n') + exec_cmd(f'radosgw-admin bucket check --fix --bucket {BUCKET_NAME}') + out = exec_cmd(f'radosgw-admin bucket stats --bucket {BUCKET_NAME}') + json_out = json.loads(out) + log.debug(json_out['usage']) + assert json_out['usage']['rgw.main']['num_objects'] == 6 + + # TESTCASE 'bucket check unlinked does not report normal entries' + log.debug('TEST: bucket check unlinked does not report normal entries\n') + out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {BUCKET_NAME} --min-age-hours 0 --dump-keys') + json_out = json.loads(out) + assert len(json_out) == 0 + + unlinked_objs = create_unlinked_objects(connection, bucket, unlinked_keys) + + # TESTCASE 'bucket check unlinked finds unlistable entries' + log.debug('TEST: bucket check unlinked finds unlistable entries\n') + out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {BUCKET_NAME} --min-age-hours 0 --dump-keys') + json_out = json.loads(out) + assert len(json_out) == len(unlinked_keys) + + # TESTCASE 'unlinked entries are not listable' + log.debug('TEST: unlinked entries are not listable\n') + for ov in bucket.object_versions.all(): + assert (ov.key, ov.version_id) not in unlinked_objs, f'object "{ov.key}:{ov.version_id}" was found in bucket listing' + + # TESTCASE 'GET returns 404 for unlinked entry keys that have no other versions' + log.debug('TEST: GET returns 404 for unlinked entry keys that have no other versions\n') + noent_keys = set(unlinked_keys) - set(ok_keys) + for key in noent_keys: + try: + bucket.Object(key).get() + assert False, 'GET did not return 404 for key={key} with no prior successful PUT' + except botocore.exceptions.ClientError as e: + assert e.response['ResponseMetadata']['HTTPStatusCode'] == 404 + + # TESTCASE 'bucket check unlinked fixes unlistable entries' + log.debug('TEST: bucket check unlinked fixes unlistable entries\n') + out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {BUCKET_NAME} --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys') + json_out = json.loads(out) + assert len(json_out) == len(unlinked_keys) + for o in unlinked_objs: + try: + connection.ObjectVersion(bucket.name, o[0], o[1]).head() + assert False, f'head for unlistable object {o[0]}:{o[1]} succeeded after fix' + except botocore.exceptions.ClientError as e: + assert e.response['ResponseMetadata']['HTTPStatusCode'] == 404 + + # TESTCASE 'bucket check unlinked fix does not affect normal entries' + log.debug('TEST: bucket check unlinked does not affect normal entries\n') + all_listable = list(bucket.object_versions.all()) + assert len(all_listable) == len(ok_keys) + len(null_version_keys), 'some normal objects were not accounted for in object listing after unlinked fix' + for o in ok_objs: + assert o in map(lambda x: (x.key, x.version_id), all_listable), "normal object not listable after fix" + connection.ObjectVersion(bucket.name, o[0], o[1]).head() + + # TESTCASE 'bucket check unlinked does not find new unlistable entries after fix' + log.debug('TEST: bucket check unlinked does not find new unlistable entries after fix\n') + out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {BUCKET_NAME} --min-age-hours 0 --dump-keys') + json_out = json.loads(out) + assert len(json_out) == 0 + + # for this set of keys we can produce leftover OLH object/entries by + # deleting the normal object instance since we should already have a leftover + # pending xattr on the OLH object due to the errors associated with the + # prior unlinked entries that were created for the same keys + leftover_pending_xattr_keys = set(ok_keys).intersection(unlinked_keys) + objs_to_delete = filter(lambda x: x[0] in leftover_pending_xattr_keys, ok_objs) + + for o in objs_to_delete: + connection.ObjectVersion(bucket.name, o[0], o[1]).delete() + + for key in leftover_pending_xattr_keys: + out = exec_cmd(f'radosgw-admin bi list --bucket {BUCKET_NAME} --object {key}') + idx_entries = json.loads(out.replace(b'\x80', b'0x80')) + assert len(idx_entries) > 0, 'failed to create leftover OLH entries for key {key}' + + # TESTCASE 'bucket check olh finds leftover OLH entries' + log.debug('TEST: bucket check olh finds leftover OLH entries\n') + out = exec_cmd(f'radosgw-admin bucket check olh --bucket {BUCKET_NAME} --dump-keys') + json_out = json.loads(out) + assert len(json_out) == len(leftover_pending_xattr_keys) + + # TESTCASE 'bucket check olh fixes leftover OLH entries' + log.debug('TEST: bucket check olh fixes leftover OLH entries\n') + out = exec_cmd(f'radosgw-admin bucket check olh --bucket {BUCKET_NAME} --fix --rgw-olh-pending-timeout-sec 0 --dump-keys') + json_out = json.loads(out) + assert len(json_out) == len(leftover_pending_xattr_keys) + + for key in leftover_pending_xattr_keys: + out = exec_cmd(f'radosgw-admin bi list --bucket {BUCKET_NAME} --object {key}') + idx_entries = json.loads(out.replace(b'\x80', b'0x80')) + assert len(idx_entries) == 0, 'index entries still exist for key={key} after olh fix' + + # TESTCASE 'bucket check olh does not find new leftover OLH entries after fix' + log.debug('TEST: bucket check olh does not find new leftover OLH entries after fix\n') + out = exec_cmd(f'radosgw-admin bucket check olh --bucket {BUCKET_NAME} --dump-keys') + json_out = json.loads(out) + assert len(json_out) == 0 + + # TESTCASE 'bucket check fixes do not affect null version objects' + log.debug('TEST: verify that bucket check fixes do not affect null version objects\n') + for o in null_version_objs: + connection.ObjectVersion(bucket.name, o[0], 'null').head() + + all_versions = list(map(lambda x: (x.key, x.version_id), bucket.object_versions.all())) + for key in null_version_keys: + assert (key, 'null') in all_versions + + # TESTCASE 'bucket check stats are correct in the presence of unlinked entries' + log.debug('TEST: bucket check stats are correct in the presence of unlinked entries\n') + bucket.object_versions.all().delete() + null_version_objs = put_objects(bucket, null_version_keys) + ok_objs = put_objects(bucket, ok_keys) + unlinked_objs = create_unlinked_objects(connection, bucket, unlinked_keys) + exec_cmd(f'radosgw-admin bucket check --fix --bucket {BUCKET_NAME}') + out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {BUCKET_NAME} --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys') + json_out = json.loads(out) + assert len(json_out) == len(unlinked_keys) + bucket.object_versions.all().delete() + out = exec_cmd(f'radosgw-admin bucket stats --bucket {BUCKET_NAME}') + json_out = json.loads(out) + log.debug(json_out['usage']) + assert json_out['usage']['rgw.main']['size'] == 0 + assert json_out['usage']['rgw.main']['num_objects'] == 0 + assert json_out['usage']['rgw.main']['size_actual'] == 0 + assert json_out['usage']['rgw.main']['size_kb'] == 0 + assert json_out['usage']['rgw.main']['size_kb_actual'] == 0 + assert json_out['usage']['rgw.main']['size_kb_utilized'] == 0 + + # Clean up + log.debug("Deleting bucket {}".format(BUCKET_NAME)) + bucket.object_versions.all().delete() + bucket.delete() + +main() +log.info("Completed bucket check tests") diff --git a/qa/workunits/rgw/test_rgw_datacache.py b/qa/workunits/rgw/test_rgw_datacache.py new file mode 100755 index 000000000..f070ec0f1 --- /dev/null +++ b/qa/workunits/rgw/test_rgw_datacache.py @@ -0,0 +1,209 @@ +#!/usr/bin/python3 + +import logging as log +from configobj import ConfigObj +import subprocess +import json +import os + +""" +Runs a test against a rgw with the data cache enabled. A client must be +set in the config for this task. This client must be the same client +that is in the config for the `rgw` task. + +In the `overrides` section `datacache` and `datacache` must be configured for +the `rgw` task and the ceph conf overrides must contain the below config +variables in the client section. + +`s3cmd` must be added as an extra_package to the install task. + +In the `workunit` task, `- rgw/run-datacache.sh` must be set for the client that +is in the config for the `rgw` task. The `RGW_DATACACHE_PATH` variable must be +set in the workunit's `env` and it must match the `datacache_path` given to the +`rgw` task in `overrides`. +Ex: +- install: + extra_packages: + deb: ['s3cmd'] + rpm: ['s3cmd'] +- overrides: + rgw: + datacache: true + datacache_path: /tmp/rgw_datacache + install: + extra_packages: + deb: ['s3cmd'] + rpm: ['s3cmd'] + ceph: + conf: + client: + rgw d3n l1 datacache persistent path: /tmp/rgw_datacache/ + rgw d3n l1 datacache size: 10737417240 + rgw d3n l1 local datacache enabled: true + rgw enable ops log: true +- rgw: + client.0: +- workunit: + clients: + client.0: + - rgw/run-datacache.sh + env: + RGW_DATACACHE_PATH: /tmp/rgw_datacache + cleanup: true +""" + +log.basicConfig(level=log.DEBUG) + +""" Constants """ +USER = 'rgw_datacache_user' +DISPLAY_NAME = 'DatacacheUser' +ACCESS_KEY = 'NX5QOQKC6BH2IDN8HC7A' +SECRET_KEY = 'LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn' +BUCKET_NAME = 'datacachebucket' +FILE_NAME = '7M.dat' +GET_FILE_NAME = '7M-get.dat' + +def exec_cmd(cmd): + log.debug("exec_cmd(%s)", cmd) + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + out, err = proc.communicate() + if proc.returncode == 0: + log.info('command succeeded') + if out is not None: log.info(out) + return out + else: + raise Exception("error: %s \nreturncode: %s" % (err, proc.returncode)) + except Exception as e: + log.error('command failed') + log.error(e) + return False + +def get_radosgw_endpoint(): + out = exec_cmd('sudo netstat -nltp | egrep "rados|valgr"') # short for radosgw/valgrind + x = out.decode('utf8').split(" ") + port = [i for i in x if ':' in i][0].split(':')[1] + log.info('radosgw port: %s' % port) + proto = "http" + hostname = '127.0.0.1' + + if port == '443': + proto = "https" + + endpoint = hostname + + log.info("radosgw endpoint is: %s", endpoint) + return endpoint, proto + +def create_s3cmd_config(path, proto): + """ + Creates a minimal config file for s3cmd + """ + log.info("Creating s3cmd config...") + + use_https_config = "False" + log.info("proto for s3cmd config is %s", proto) + if proto == "https": + use_https_config = "True" + + s3cmd_config = ConfigObj( + indent_type='', + infile={ + 'default': + { + 'host_bucket': 'no.way.in.hell', + 'use_https': use_https_config, + }, + } + ) + + f = open(path, 'wb') + s3cmd_config.write(f) + f.close() + log.info("s3cmd config written") + +def get_cmd_output(cmd_out): + out = cmd_out.decode('utf8') + out = out.strip('\n') + return out + +def main(): + """ + execute the datacache test + """ + # setup for test + cache_dir = os.environ['RGW_DATACACHE_PATH'] + log.debug("datacache dir from config is: %s", cache_dir) + + out = exec_cmd('pwd') + pwd = get_cmd_output(out) + log.debug("pwd is: %s", pwd) + + endpoint, proto = get_radosgw_endpoint() + + # create 7M file to put + outfile = pwd + '/' + FILE_NAME + exec_cmd('dd if=/dev/urandom of=%s bs=1M count=7' % (outfile)) + + # create user + exec_cmd('radosgw-admin user create --uid %s --display-name %s --access-key %s --secret %s' + % (USER, DISPLAY_NAME, ACCESS_KEY, SECRET_KEY)) + + # create s3cmd config + s3cmd_config_path = pwd + '/s3cfg' + create_s3cmd_config(s3cmd_config_path, proto) + + # create a bucket + exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --no-check-hostname --host=%s mb s3://%s' + % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME)) + + # put an object in the bucket + exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --no-check-hostname --host=%s put %s s3://%s' + % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, outfile, BUCKET_NAME)) + + # get object from bucket + get_file_path = pwd + '/' + GET_FILE_NAME + exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --no-check-hostname --host=%s get s3://%s/%s %s --force' + % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME, FILE_NAME, get_file_path)) + + # get info of object + out = exec_cmd('radosgw-admin object stat --bucket=%s --object=%s' % (BUCKET_NAME, FILE_NAME)) + + json_op = json.loads(out) + cached_object_name = json_op['manifest']['prefix'] + log.debug("Cached object name is: %s", cached_object_name) + + # check that the cache is enabled (does the cache directory empty) + out = exec_cmd('find %s -type f | wc -l' % (cache_dir)) + chk_cache_dir = int(get_cmd_output(out)) + log.debug("Check cache dir content: %s", chk_cache_dir) + if chk_cache_dir == 0: + log.info("NOTICE: datacache test object not found, inspect if datacache was bypassed or disabled during this check.") + return + + # list the files in the cache dir for troubleshooting + out = exec_cmd('ls -l %s' % (cache_dir)) + # get name of cached object and check if it exists in the cache + out = exec_cmd('find %s -name "*%s1"' % (cache_dir, cached_object_name)) + cached_object_path = get_cmd_output(out) + log.debug("Path of file in datacache is: %s", cached_object_path) + out = exec_cmd('basename %s' % (cached_object_path)) + basename_cmd_out = get_cmd_output(out) + log.debug("Name of file in datacache is: %s", basename_cmd_out) + + # check to see if the cached object is in Ceph + out = exec_cmd('rados ls -p default.rgw.buckets.data') + rados_ls_out = get_cmd_output(out) + log.debug("rados ls output is: %s", rados_ls_out) + + assert(basename_cmd_out in rados_ls_out) + log.debug("RGW Datacache test SUCCESS") + + # remove datacache dir + #cmd = exec_cmd('rm -rf %s' % (cache_dir)) + #log.debug("RGW Datacache dir deleted") + #^ commenting for future refrence - the work unit will continue running tests and if the cache_dir is removed + # all the writes to cache will fail with errno 2 ENOENT No such file or directory. + +main() +log.info("Completed Datacache tests") diff --git a/qa/workunits/rgw/test_rgw_gc_log.sh b/qa/workunits/rgw/test_rgw_gc_log.sh new file mode 100755 index 000000000..ab4015aef --- /dev/null +++ b/qa/workunits/rgw/test_rgw_gc_log.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_rgw_gc_log + +exit 0 diff --git a/qa/workunits/rgw/test_rgw_obj.sh b/qa/workunits/rgw/test_rgw_obj.sh new file mode 100755 index 000000000..01dd2b5ee --- /dev/null +++ b/qa/workunits/rgw/test_rgw_obj.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_rgw_obj + +exit 0 diff --git a/qa/workunits/rgw/test_rgw_orphan_list.sh b/qa/workunits/rgw/test_rgw_orphan_list.sh new file mode 100755 index 000000000..34d550cea --- /dev/null +++ b/qa/workunits/rgw/test_rgw_orphan_list.sh @@ -0,0 +1,519 @@ +#!/usr/bin/env bash + +# set -x +set -e + +# if defined, debug messages will be displayed and prepended with the string +# debug="DEBUG" + +huge_size=5100 # in megabytes +big_size=7 # in megabytes + +huge_obj=/tmp/huge_obj.temp.$$ +big_obj=/tmp/big_obj.temp.$$ +empty_obj=/tmp/empty_obj.temp.$$ + +fifo=/tmp/orphan-fifo.$$ +awscli_dir=${HOME}/awscli_temp +export PATH=${PATH}:${awscli_dir} + +rgw_host=$(hostname --fqdn) +if echo "$rgw_host" | grep -q '\.' ; then + : +else + host_domain=".front.sepia.ceph.com" + echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain" + rgw_host="${rgw_host}${host_domain}" +fi +rgw_port=80 + +echo "Fully Qualified Domain Name: $rgw_host" + +success() { + echo OK. + exit 0 +} + +######################################################################## +# INSTALL AND CONFIGURE TOOLING + +install_awscli() { + # NB: this does verify authenticity and integrity of downloaded + # file; see + # https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html + here="$(pwd)" + cd "$HOME" + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + mkdir -p $awscli_dir + ./aws/install -i $awscli_dir + cd "$here" +} + +uninstall_awscli() { + here="$(pwd)" + cd "$HOME" + rm -rf $awscli_dir ./aws awscliv2.zip + cd "$here" +} + +sudo yum -y install s3cmd +sudo yum -y install python3-setuptools +sudo yum -y install python3-pip +sudo pip3 install --upgrade setuptools +sudo pip3 install python-swiftclient + +# get ready for transition from s3cmd to awscli +if false ;then + install_awscli + aws --version + uninstall_awscli +fi + +s3config=/tmp/s3config.$$ + +# do not include the port when it is 80; the host base is used in the +# v4 signature and it needs to follow this convention for signatures +# to match +if [ "$rgw_port" -ne 80 ] ;then + s3_host_base="${rgw_host}:${rgw_port}" +else + s3_host_base="$rgw_host" +fi + +cat >${s3config} <<EOF +[default] +host_base = $s3_host_base +access_key = 0555b35654ad1656d804 +secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== +bucket_location = us-east-1 +check_ssl_certificate = True +check_ssl_hostname = True +default_mime_type = binary/octet-stream +delete_removed = False +dry_run = False +enable_multipart = True +encoding = UTF-8 +encrypt = False +follow_symlinks = False +force = False +guess_mime_type = True +host_bucket = anything.with.three.dots +multipart_chunk_size_mb = 15 +multipart_max_chunks = 10000 +recursive = False +recv_chunk = 65536 +send_chunk = 65536 +signature_v2 = False +socket_timeout = 300 +use_https = False +use_mime_magic = True +verbosity = WARNING +EOF + + +# set up swift authentication +export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0 +export ST_USER=test:tester +export ST_KEY=testing + +create_users() { + # Create S3 user + local akey='0555b35654ad1656d804' + local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==' + radosgw-admin user create --uid testid \ + --access-key $akey --secret $skey \ + --display-name 'M. Tester' --email tester@ceph.com + + # Create Swift user + radosgw-admin user create --subuser=test:tester \ + --display-name=Tester-Subuser --key-type=swift \ + --secret=testing --access=full +} + +myswift() { + if [ -n "$debug" ] ;then + echo "${debug}: swift --verbose --debug $@" + fi + swift --verbose --debug "$@" + local code=$? + if [ $code -ne 0 ] ;then + echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@"" + exit $code + fi +} + +mys3cmd() { + if [ -n "$debug" ] ;then + echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@" + fi + s3cmd --config=${s3config} --verbose --debug "$@" + local code=$? + if [ $code -ne 0 ] ;then + echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@"" + exit $code + fi +} + +mys3uploadkill() { + if [ $# -ne 5 ] ;then + echo "$0: error expecting 5 arguments" + exit 1 + fi + + local_file="$1" + remote_bkt="$2" + remote_obj="$3" + fifo="$4" + stop_part="$5" + + mkfifo $fifo + + s3cmd --config=${s3config} put $local_file \ + s3://${remote_bkt}/${remote_obj} \ + --progress \ + --multipart-chunk-size-mb=5 >$fifo & + set +e # don't allow errors to stop script + while read line ;do + echo "$line" | grep --quiet "part $stop_part " + if [ ${PIPESTATUS[1]} -eq 0 ] ;then + kill -9 $(jobs -p) + break + fi + done <$fifo + set -e + + rm -f $fifo +} + +mys3upload() { + obj=$1 + bucket=$2 + dest_obj=$3 + + mys3cmd put -q $obj s3://${bucket}/$dest_obj +} + +######################################################################## +# PREP + +create_users +dd if=/dev/urandom of=$big_obj bs=1M count=${big_size} +dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size} +touch $empty_obj + +quick_tests() { + echo TRY A SWIFT COMMAND + myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test + myswift list + myswift list swift-plain-ctr + + echo TRY A RADOSGW-ADMIN COMMAND + radosgw-admin bucket list # make sure rgw is up and running +} + +######################################################################## +# S3 TESTS + +#################################### +# regular multipart test + +mys3cmd mb s3://multipart-bkt +mys3upload $huge_obj multipart-bkt multipart-obj +mys3cmd ls +mys3cmd ls s3://multipart-bkt + +#################################### +# multipart test with incomplete uploads + +bkt="incomplete-mp-bkt-1" + +mys3cmd mb s3://$bkt + +mys3uploadkill $huge_obj $bkt incomplete-mp-obj-c $fifo 20 + +# generate an incomplete multipart with more than 1,000 parts +mys3uploadkill $huge_obj $bkt incomplete-mp-obj-b $fifo 1005 + +# generate more than 1000 incomplet multiparts +for c in $(seq 1005) ;do + mys3uploadkill $huge_obj $bkt incomplete-mp-obj-c-$c $fifo 3 +done + +#################################### +# resharded bucket + +bkt=resharded-bkt-1 + +mys3cmd mb s3://$bkt + +for f in $(seq 8) ; do + dest_obj="reshard-obj-${f}" + mys3cmd put -q $big_obj s3://${bkt}/$dest_obj +done + +radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it +radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it + +#################################### +# versioned bucket + +if true ;then + echo "WARNING: versioned bucket test currently turned off" +else + bkt=versioned-bkt-1 + + mys3cmd mb s3://$bkt + + # bucket-enable-versioning $bkt + + for f in $(seq 3) ;do + for g in $(seq 10) ;do + dest_obj="versioned-obj-${g}" + mys3cmd put -q $big_obj s3://${bkt}/$dest_obj + done + done + + for g in $(seq 1 2 10) ;do + dest_obj="versioned-obj-${g}" + mys3cmd rm s3://${bkt}/$dest_obj + done +fi + +############################################################ +# copy small objects + +o_bkt="orig-bkt-1" +d_bkt="copy-bkt-1" +mys3cmd mb s3://$o_bkt + +for f in $(seq 4) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1 +mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3 + +for f in $(seq 5 6) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj +done + +############################################################ +# copy small objects and delete original + +o_bkt="orig-bkt-2" +d_bkt="copy-bkt-2" + +mys3cmd mb s3://$o_bkt + +for f in $(seq 4) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1 +mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3 + +for f in $(seq 5 6) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj +done + +mys3cmd rb --recursive s3://${o_bkt} + +############################################################ +# copy multipart objects + +o_bkt="orig-mp-bkt-3" +d_bkt="copy-mp-bkt-3" + +mys3cmd mb s3://$o_bkt + +for f in $(seq 2) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \ + s3://${d_bkt}/copied-multipart-obj-1 + +for f in $(seq 5 5) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj +done + + +############################################################ +# copy multipart objects and delete original + +o_bkt="orig-mp-bkt-4" +d_bkt="copy-mp-bkt-4" + +mys3cmd mb s3://$o_bkt + +for f in $(seq 2) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \ + s3://${d_bkt}/copied-multipart-obj-1 + +for f in $(seq 5 5) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj +done + +mys3cmd rb --recursive s3://$o_bkt + +######################################################################## +# SWIFT TESTS + +# 600MB +segment_size=629145600 + +############################################################ +# plain test + +for f in $(seq 4) ;do + myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f +done + +############################################################ +# zero-len test + +myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/ +myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1 +myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1 +myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1 + +############################################################ +# dlo test + +# upload in 300MB segments +myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \ + -S $segment_size + +############################################################ +# slo test + +# upload in 300MB segments +myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \ + -S $segment_size --use-slo + +############################################################ +# large object copy test + +# upload in 300MB segments +o_ctr=swift-orig-ctr +o_obj=slo-orig-obj-1 +d_ctr=swift-copy-ctr +d_obj=slo-copy-obj-1 +myswift upload $o_ctr $big_obj --object-name $o_obj + +myswift copy --destination /${d_ctr}/${d_obj} \ + $o_ctr $o_obj + +myswift delete $o_ctr $o_obj + +############################################################ +# huge dlo object copy test + +o_ctr=swift-orig-dlo-ctr-1 +o_obj=dlo-orig-dlo-obj-1 +d_ctr=swift-copy-dlo-ctr-1 +d_obj=dlo-copy-dlo-obj-1 + +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size + +myswift copy --destination /${d_ctr}/${d_obj} \ + $o_ctr $o_obj + +############################################################ +# huge dlo object copy and orig delete + +o_ctr=swift-orig-dlo-ctr-2 +o_obj=dlo-orig-dlo-obj-2 +d_ctr=swift-copy-dlo-ctr-2 +d_obj=dlo-copy-dlo-obj-2 + +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size + +myswift copy --destination /${d_ctr}/${d_obj} \ + $o_ctr $o_obj + +myswift delete $o_ctr $o_obj + +############################################################ +# huge slo object copy test + +o_ctr=swift-orig-slo-ctr-1 +o_obj=slo-orig-slo-obj-1 +d_ctr=swift-copy-slo-ctr-1 +d_obj=slo-copy-slo-obj-1 +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size --use-slo + +myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj + +############################################################ +# huge slo object copy test and orig delete + +o_ctr=swift-orig-slo-ctr-2 +o_obj=slo-orig-slo-obj-2 +d_ctr=swift-copy-slo-ctr-2 +d_obj=slo-copy-slo-obj-2 +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size --use-slo + +myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj + +myswift delete $o_ctr $o_obj + +######################################################################## +# FORCE GARBAGE COLLECTION + +sleep 6 # since for testing age at which gc can happen is 5 secs +radosgw-admin gc process --include-all + + +######################################## +# DO ORPHAN LIST + +pool="default.rgw.buckets.data" + +rgw-orphan-list $pool + +# we only expect there to be one output file, but loop just in case +ol_error="" +for f in orphan-list-*.out ; do + if [ -s "$f" ] ;then # if file non-empty + ol_error="${ol_error}:$f" + echo "One ore more orphans found in $f:" + cat "$f" + fi +done + +if [ -n "$ol_error" ] ;then + echo "ERROR: orphans found when none expected" + exit 1 +fi + +######################################################################## +# CLEAN UP + +rm -f $empty_obj $big_obj $huge_obj $s3config + +success diff --git a/qa/workunits/rgw/test_rgw_reshard.py b/qa/workunits/rgw/test_rgw_reshard.py new file mode 100755 index 000000000..6326e7b17 --- /dev/null +++ b/qa/workunits/rgw/test_rgw_reshard.py @@ -0,0 +1,311 @@ +#!/usr/bin/python3 + +import errno +import time +import logging as log +import json +import os +from common import exec_cmd, boto_connect, create_user, put_objects, create_unlinked_objects + +""" +Rgw manual and dynamic resharding testing against a running instance +""" +# The test cases in this file have been annotated for inventory. +# To extract the inventory (in csv format) use the command: +# +# grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //' +# +# + +""" Constants """ +USER = 'tester' +DISPLAY_NAME = 'Testing' +ACCESS_KEY = 'NX5QOQKC6BH2IDN8HC7A' +SECRET_KEY = 'LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn' +BUCKET_NAME = 'a-bucket' +VER_BUCKET_NAME = 'myver' +INDEX_POOL = 'default.rgw.buckets.index' + +class BucketStats: + def __init__(self, bucket_name, bucket_id, num_objs=0, size_kb=0, num_shards=0): + self.bucket_name = bucket_name + self.bucket_id = bucket_id + self.num_objs = num_objs + self.size_kb = size_kb + self.num_shards = num_shards if num_shards > 0 else 1 + + def get_num_shards(self): + self.num_shards = get_bucket_num_shards(self.bucket_name, self.bucket_id) + + +def get_bucket_stats(bucket_name): + """ + function to get bucket stats + """ + cmd = exec_cmd("radosgw-admin bucket stats --bucket {}".format(bucket_name)) + json_op = json.loads(cmd) + #print(json.dumps(json_op, indent = 4, sort_keys=True)) + bucket_id = json_op['id'] + num_shards = json_op['num_shards'] + if len(json_op['usage']) > 0: + num_objects = json_op['usage']['rgw.main']['num_objects'] + size_kb = json_op['usage']['rgw.main']['size_kb'] + else: + num_objects = 0 + size_kb = 0 + log.debug(" \nBUCKET_STATS: \nbucket: {} id: {} num_objects: {} size_kb: {} num_shards: {}\n".format(bucket_name, bucket_id, + num_objects, size_kb, num_shards)) + return BucketStats(bucket_name, bucket_id, num_objects, size_kb, num_shards) + +def get_bucket_layout(bucket_name): + res = exec_cmd("radosgw-admin bucket layout --bucket {}".format(bucket_name)) + return json.loads(res) + +def get_bucket_shard0(bucket_name): + bucket_id = get_bucket_stats(bucket_name).bucket_id + index_gen = get_bucket_layout(bucket_name)['layout']['current_index']['gen'] + return '.dir.%s.%d.0' % (bucket_id, index_gen) + +def get_bucket_num_shards(bucket_name, bucket_id): + """ + function to get bucket num shards + """ + metadata = 'bucket.instance:' + bucket_name + ':' + bucket_id + cmd = exec_cmd('radosgw-admin metadata get {}'.format(metadata)) + json_op = json.loads(cmd) + num_shards = json_op['data']['bucket_info']['num_shards'] + return num_shards + +def run_bucket_reshard_cmd(bucket_name, num_shards, **kwargs): + cmd = 'radosgw-admin bucket reshard --bucket {} --num-shards {}'.format(bucket_name, num_shards) + cmd += ' --rgw-reshard-bucket-lock-duration 30' # reduce to minimum + if 'error_at' in kwargs: + cmd += ' --inject-error-at {}'.format(kwargs.pop('error_at')) + elif 'abort_at' in kwargs: + cmd += ' --inject-abort-at {}'.format(kwargs.pop('abort_at')) + if 'error_code' in kwargs: + cmd += ' --inject-error-code {}'.format(kwargs.pop('error_code')) + return exec_cmd(cmd, **kwargs) + +def test_bucket_reshard(conn, name, **fault): + # create a bucket with non-default ACLs to verify that reshard preserves them + bucket = conn.create_bucket(Bucket=name, ACL='authenticated-read') + grants = bucket.Acl().grants + + objs = [] + try: + # create objs + for i in range(0, 20): + objs += [bucket.put_object(Key='key' + str(i), Body=b"some_data")] + + old_shard_count = get_bucket_stats(name).num_shards + num_shards_expected = old_shard_count + 1 + + # try reshard with fault injection + _, ret = run_bucket_reshard_cmd(name, num_shards_expected, check_retcode=False, **fault) + + if fault.get('error_code') == errno.ECANCELED: + assert(ret == 0) # expect ECANCELED to retry and succeed + else: + assert(ret != 0 and ret != errno.EBUSY) + + # check shard count + cur_shard_count = get_bucket_stats(name).num_shards + assert(cur_shard_count == old_shard_count) + + # verify that the bucket is writeable by deleting an object + objs.pop().delete() + + assert grants == bucket.Acl().grants # recheck grants after cancel + + # retry reshard without fault injection. if radosgw-admin aborted, + # we'll have to retry until the reshard lock expires + while True: + _, ret = run_bucket_reshard_cmd(name, num_shards_expected, check_retcode=False) + if ret == errno.EBUSY: + log.info('waiting 30 seconds for reshard lock to expire...') + time.sleep(30) + continue + assert(ret == 0) + break + + # recheck shard count + final_shard_count = get_bucket_stats(name).num_shards + assert(final_shard_count == num_shards_expected) + + assert grants == bucket.Acl().grants # recheck grants after commit + finally: + # cleanup on resharded bucket must succeed + bucket.delete_objects(Delete={'Objects':[{'Key':o.key} for o in objs]}) + bucket.delete() + + +def main(): + """ + execute manual and dynamic resharding commands + """ + create_user(USER, DISPLAY_NAME, ACCESS_KEY, SECRET_KEY) + + connection = boto_connect(ACCESS_KEY, SECRET_KEY) + + # create a bucket + bucket = connection.create_bucket(Bucket=BUCKET_NAME) + ver_bucket = connection.create_bucket(Bucket=VER_BUCKET_NAME) + connection.BucketVersioning(VER_BUCKET_NAME).enable() + + bucket_acl = connection.BucketAcl(BUCKET_NAME).load() + ver_bucket_acl = connection.BucketAcl(VER_BUCKET_NAME).load() + + # TESTCASE 'reshard-add','reshard','add','add bucket to resharding queue','succeeds' + log.debug('TEST: reshard add\n') + + num_shards_expected = get_bucket_stats(BUCKET_NAME).num_shards + 1 + cmd = exec_cmd('radosgw-admin reshard add --bucket {} --num-shards {}'.format(BUCKET_NAME, num_shards_expected)) + cmd = exec_cmd('radosgw-admin reshard list') + json_op = json.loads(cmd) + log.debug('bucket name {}'.format(json_op[0]['bucket_name'])) + assert json_op[0]['bucket_name'] == BUCKET_NAME + assert json_op[0]['tentative_new_num_shards'] == num_shards_expected + + # TESTCASE 'reshard-process','reshard','','process bucket resharding','succeeds' + log.debug('TEST: reshard process\n') + cmd = exec_cmd('radosgw-admin reshard process') + time.sleep(5) + # check bucket shards num + bucket_stats1 = get_bucket_stats(BUCKET_NAME) + if bucket_stats1.num_shards != num_shards_expected: + log.error("Resharding failed on bucket {}. Expected number of shards are not created\n".format(BUCKET_NAME)) + + # TESTCASE 'reshard-add','reshard','add','add non empty bucket to resharding queue','succeeds' + log.debug('TEST: reshard add non empty bucket\n') + # create objs + num_objs = 8 + for i in range(0, num_objs): + connection.Object(BUCKET_NAME, ('key'+str(i))).put(Body=b"some_data") + + num_shards_expected = get_bucket_stats(BUCKET_NAME).num_shards + 1 + cmd = exec_cmd('radosgw-admin reshard add --bucket {} --num-shards {}'.format(BUCKET_NAME, num_shards_expected)) + cmd = exec_cmd('radosgw-admin reshard list') + json_op = json.loads(cmd) + assert json_op[0]['bucket_name'] == BUCKET_NAME + assert json_op[0]['tentative_new_num_shards'] == num_shards_expected + + # TESTCASE 'reshard process ,'reshard','process','reshard non empty bucket','succeeds' + log.debug('TEST: reshard process non empty bucket\n') + cmd = exec_cmd('radosgw-admin reshard process') + # check bucket shards num + bucket_stats1 = get_bucket_stats(BUCKET_NAME) + if bucket_stats1.num_shards != num_shards_expected: + log.error("Resharding failed on bucket {}. Expected number of shards are not created\n".format(BUCKET_NAME)) + + # TESTCASE 'manual bucket resharding','inject error','fail','check bucket accessibility', 'retry reshard' + log.debug('TEST: reshard bucket with EIO injected at set_target_layout\n') + test_bucket_reshard(connection, 'error-at-set-target-layout', error_at='set_target_layout') + log.debug('TEST: reshard bucket with ECANCELED injected at set_target_layout\n') + test_bucket_reshard(connection, 'error-at-set-target-layout', error_at='set_target_layout', error_code=errno.ECANCELED) + log.debug('TEST: reshard bucket with abort at set_target_layout\n') + test_bucket_reshard(connection, 'abort-at-set-target-layout', abort_at='set_target_layout') + + log.debug('TEST: reshard bucket with EIO injected at block_writes\n') + test_bucket_reshard(connection, 'error-at-block-writes', error_at='block_writes') + log.debug('TEST: reshard bucket with abort at block_writes\n') + test_bucket_reshard(connection, 'abort-at-block-writes', abort_at='block_writes') + + log.debug('TEST: reshard bucket with EIO injected at commit_target_layout\n') + test_bucket_reshard(connection, 'error-at-commit-target-layout', error_at='commit_target_layout') + log.debug('TEST: reshard bucket with ECANCELED injected at commit_target_layout\n') + test_bucket_reshard(connection, 'error-at-commit-target-layout', error_at='commit_target_layout', error_code=errno.ECANCELED) + log.debug('TEST: reshard bucket with abort at commit_target_layout\n') + test_bucket_reshard(connection, 'abort-at-commit-target-layout', abort_at='commit_target_layout') + + log.debug('TEST: reshard bucket with EIO injected at do_reshard\n') + test_bucket_reshard(connection, 'error-at-do-reshard', error_at='do_reshard') + log.debug('TEST: reshard bucket with abort at do_reshard\n') + test_bucket_reshard(connection, 'abort-at-do-reshard', abort_at='do_reshard') + + # TESTCASE 'versioning reshard-','bucket', reshard','versioning reshard','succeeds' + log.debug(' test: reshard versioned bucket') + num_shards_expected = get_bucket_stats(VER_BUCKET_NAME).num_shards + 1 + cmd = exec_cmd('radosgw-admin bucket reshard --bucket {} --num-shards {}'.format(VER_BUCKET_NAME, + num_shards_expected)) + # check bucket shards num + ver_bucket_stats = get_bucket_stats(VER_BUCKET_NAME) + assert ver_bucket_stats.num_shards == num_shards_expected + + # TESTCASE 'check acl' + new_bucket_acl = connection.BucketAcl(BUCKET_NAME).load() + assert new_bucket_acl == bucket_acl + new_ver_bucket_acl = connection.BucketAcl(VER_BUCKET_NAME).load() + assert new_ver_bucket_acl == ver_bucket_acl + + # TESTCASE 'check reshard removes olh entries with empty name' + log.debug(' test: reshard removes olh entries with empty name') + bucket.objects.all().delete() + + + # get name of shard 0 object, add a bogus olh entry with empty name + bucket_shard0 = get_bucket_shard0(BUCKET_NAME) + if 'CEPH_ROOT' in os.environ: + k = '%s/qa/workunits/rgw/olh_noname_key' % os.environ['CEPH_ROOT'] + v = '%s/qa/workunits/rgw/olh_noname_val' % os.environ['CEPH_ROOT'] + else: + k = 'olh_noname_key' + v = 'olh_noname_val' + exec_cmd('rados -p %s setomapval %s --omap-key-file %s < %s' % (INDEX_POOL, bucket_shard0, k, v)) + + # check that bi list has one entry with empty name + cmd = exec_cmd('radosgw-admin bi list --bucket %s' % BUCKET_NAME) + json_op = json.loads(cmd.decode('utf-8', 'ignore')) # ignore utf-8 can't decode 0x80 + assert len(json_op) == 1 + assert json_op[0]['entry']['key']['name'] == '' + + # reshard to prune the bogus olh + cmd = exec_cmd('radosgw-admin bucket reshard --bucket %s --num-shards %s --yes-i-really-mean-it' % (BUCKET_NAME, 1)) + + # get that bi list has zero entries + cmd = exec_cmd('radosgw-admin bi list --bucket %s' % BUCKET_NAME) + json_op = json.loads(cmd.decode('utf-8', 'ignore')) # ignore utf-8 can't decode 0x80 + assert len(json_op) == 0 + + # TESTCASE 'check that PUT succeeds during reshard' + log.debug(' test: PUT succeeds during reshard') + num_shards = get_bucket_stats(VER_BUCKET_NAME).num_shards + exec_cmd('''radosgw-admin --inject-delay-at=do_reshard --inject-delay-ms=5000 \ + bucket reshard --bucket {} --num-shards {}''' + .format(VER_BUCKET_NAME, num_shards + 1), wait = False) + time.sleep(1) + ver_bucket.put_object(Key='put_during_reshard', Body=b"some_data") + log.debug('put object successful') + + # TESTCASE 'check that bucket stats are correct after reshard with unlinked entries' + log.debug('TEST: check that bucket stats are correct after reshard with unlinked entries\n') + ver_bucket.object_versions.all().delete() + ok_keys = ['a', 'b', 'c'] + unlinked_keys = ['x', 'y', 'z'] + put_objects(ver_bucket, ok_keys) + create_unlinked_objects(connection, ver_bucket, unlinked_keys) + cmd = exec_cmd(f'radosgw-admin bucket reshard --bucket {VER_BUCKET_NAME} --num-shards 17 --yes-i-really-mean-it') + out = exec_cmd(f'radosgw-admin bucket check unlinked --bucket {VER_BUCKET_NAME} --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys') + json_out = json.loads(out) + assert len(json_out) == len(unlinked_keys) + ver_bucket.object_versions.all().delete() + out = exec_cmd(f'radosgw-admin bucket stats --bucket {VER_BUCKET_NAME}') + json_out = json.loads(out) + log.debug(json_out['usage']) + assert json_out['usage']['rgw.main']['size'] == 0 + assert json_out['usage']['rgw.main']['num_objects'] == 0 + assert json_out['usage']['rgw.main']['size_actual'] == 0 + assert json_out['usage']['rgw.main']['size_kb'] == 0 + assert json_out['usage']['rgw.main']['size_kb_actual'] == 0 + assert json_out['usage']['rgw.main']['size_kb_utilized'] == 0 + + # Clean up + log.debug("Deleting bucket {}".format(BUCKET_NAME)) + bucket.objects.all().delete() + bucket.delete() + log.debug("Deleting bucket {}".format(VER_BUCKET_NAME)) + ver_bucket.object_versions.all().delete() + ver_bucket.delete() + +main() +log.info("Completed resharding tests") diff --git a/qa/workunits/rgw/test_rgw_s3_mp_reupload.py b/qa/workunits/rgw/test_rgw_s3_mp_reupload.py new file mode 100755 index 000000000..b3cb2d5ab --- /dev/null +++ b/qa/workunits/rgw/test_rgw_s3_mp_reupload.py @@ -0,0 +1,121 @@ +import boto3 +import botocore.exceptions +import sys +import os +import subprocess + +#boto3.set_stream_logger(name='botocore') + +# handles two optional system arguments: +# <bucket-name> : default is "bkt134" +# <0 or 1> : 0 -> upload aborted, 1 -> completed; default is completed + +if len(sys.argv) >= 2: + bucket_name = sys.argv[1] +else: + bucket_name = "bkt314738362229" +print("bucket nams is %s" % bucket_name) + +complete_mpu = True +if len(sys.argv) >= 3: + complete_mpu = int(sys.argv[2]) > 0 + +versioned_bucket = False +if len(sys.argv) >= 4: + versioned_bucket = int(sys.argv[3]) > 0 + +rgw_host = os.environ['RGW_HOST'] +access_key = os.environ['RGW_ACCESS_KEY'] +secret_key = os.environ['RGW_SECRET_KEY'] + +try: + endpoint='http://%s:%d' % (rgw_host, 80) + client = boto3.client('s3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + res = client.create_bucket(Bucket=bucket_name) +except botocore.exceptions.EndpointConnectionError: + try: + endpoint='https://%s:%d' % (rgw_host, 443) + client = boto3.client('s3', + endpoint_url=endpoint, + verify=False, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + res = client.create_bucket(Bucket=bucket_name) + except botocore.exceptions.EndpointConnectionError: + endpoint='http://%s:%d' % (rgw_host, 8000) + client = boto3.client('s3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + res = client.create_bucket(Bucket=bucket_name) + +print("endpoint is %s" % endpoint) + +if versioned_bucket: + res = client.put_bucket_versioning( + Bucket=bucket_name, + VersioningConfiguration={ + 'MFADelete': 'Disabled', + 'Status': 'Enabled'} + ) + +key = "mpu_test4" +nparts = 2 +ndups = 11 +do_reupload = True + +part_path = "/tmp/mp_part_5m" +subprocess.run(["dd", "if=/dev/urandom", "of=" + part_path, "bs=1M", "count=5"], check=True) + +f = open(part_path, 'rb') + +res = client.create_multipart_upload(Bucket=bucket_name, Key=key) +mpu_id = res["UploadId"] + +print("start UploadId=%s" % (mpu_id)) + +parts = [] +parts2 = [] + +for ix in range(0,nparts): + part_num = ix + 1 + f.seek(0) + res = client.upload_part(Body=f, Bucket=bucket_name, Key=key, + UploadId=mpu_id, PartNumber=part_num) + # save + etag = res['ETag'] + part = {'ETag': etag, 'PartNumber': part_num} + print("phase 1 uploaded part %s" % part) + parts.append(part) + +if do_reupload: + # just re-upload part 1 + part_num = 1 + for ix in range(0,ndups): + f.seek(0) + res = client.upload_part(Body=f, Bucket=bucket_name, Key=key, + UploadId=mpu_id, PartNumber=part_num) + etag = res['ETag'] + part = {'ETag': etag, 'PartNumber': part_num} + print ("phase 2 uploaded part %s" % part) + + # save + etag = res['ETag'] + part = {'ETag': etag, 'PartNumber': part_num} + parts2.append(part) + +if complete_mpu: + print("completing multipart upload, parts=%s" % parts) + res = client.complete_multipart_upload( + Bucket=bucket_name, Key=key, UploadId=mpu_id, + MultipartUpload={'Parts': parts}) +else: + print("aborting multipart upload, parts=%s" % parts) + res = client.abort_multipart_upload( + Bucket=bucket_name, Key=key, UploadId=mpu_id) + +# clean up +subprocess.run(["rm", "-f", part_path], check=True) diff --git a/qa/workunits/rgw/test_rgw_s3_mp_reupload.sh b/qa/workunits/rgw/test_rgw_s3_mp_reupload.sh new file mode 100755 index 000000000..5d73fd048 --- /dev/null +++ b/qa/workunits/rgw/test_rgw_s3_mp_reupload.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +# INITIALIZATION + +mydir=$(dirname $0) +data_pool=default.rgw.buckets.data +orphan_list_out=/tmp/orphan_list.$$ +radoslist_out=/tmp/radoslist.$$ +rados_ls_out=/tmp/rados_ls.$$ +diff_out=/tmp/diff.$$ + +rgw_host="$(hostname --fqdn)" +echo "INFO: fully qualified domain name: $rgw_host" + +export RGW_ACCESS_KEY="0555b35654ad1656d804" +export RGW_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==" +export RGW_HOST="${RGW_HOST:-$rgw_host}" + +# random argument determines if multipart is aborted or completed 50/50 +outcome=$((RANDOM % 2)) +if [ $outcome -eq 0 ] ;then + echo "== TESTING *ABORTING* MULTIPART UPLOAD WITH RE-UPLOADS ==" +else + echo "== TESTING *COMPLETING* MULTIPART UPLOAD WITH RE-UPLOADS ==" +fi + +# random argument determines if multipart is aborted or completed 50/50 +versioning=$((RANDOM % 2)) +if [ $versioning -eq 0 ] ;then + echo "== TESTING NON-VERSIONED BUCKET ==" +else + echo "== TESTING VERSIONED BUCKET ==" +fi + +# create a randomized bucket name +bucket="reupload-bkt-$((RANDOM % 899999 + 100000))" + + +# SET UP PYTHON VIRTUAL ENVIRONMENT + +# install boto3 +python3 -m venv $mydir +source $mydir/bin/activate +pip install pip --upgrade +pip install boto3 + + +# CREATE RGW USER IF NECESSARY + +if radosgw-admin user info --access-key $RGW_ACCESS_KEY 2>/dev/null ;then + echo INFO: user already exists +else + echo INFO: creating user + radosgw-admin user create --uid testid \ + --access-key $RGW_ACCESS_KEY \ + --secret $RGW_SECRET_KEY \ + --display-name 'M. Tester' \ + --email tester@ceph.com 2>/dev/null +fi + + +# RUN REUPLOAD TEST + +$mydir/bin/python3 ${mydir}/test_rgw_s3_mp_reupload.py $bucket $outcome $versioning + + +# ANALYZE FOR ERRORS +# (NOTE: for now we're choosing not to use the rgw-orphan-list tool) + +# force garbage collection to remove extra parts +radosgw-admin gc process --include-all 2>/dev/null + +marker=$(radosgw-admin metadata get bucket:$bucket 2>/dev/null | grep bucket_id | sed 's/.*: "\(.*\)".*/\1/') + +# determine expected rados objects +radosgw-admin bucket radoslist --bucket=$bucket 2>/dev/null | sort >$radoslist_out +echo "radosgw-admin bucket radoslist:" +cat $radoslist_out + +# determine found rados objects +rados ls -p $data_pool 2>/dev/null | grep "^$marker" | sort >$rados_ls_out +echo "rados ls:" +cat $rados_ls_out + +# compare expected and found +diff $radoslist_out $rados_ls_out >$diff_out +if [ $(cat $diff_out | wc -l) -ne 0 ] ;then + error=1 + echo "ERROR: Found differences between expected and actual rados objects for test bucket." + echo " note: indicators: '>' found but not expected; '<' expected but not found." + cat $diff_out +fi + + +# CLEAN UP + +deactivate + +rm -f $orphan_list_out $radoslist_out $rados_ls_out $diff_out + + +# PRODUCE FINAL RESULTS + +if [ -n "$error" ] ;then + echo "== FAILED ==" + exit 1 +fi + +echo "== PASSED ==" +exit 0 diff --git a/qa/workunits/rgw/test_rgw_throttle.sh b/qa/workunits/rgw/test_rgw_throttle.sh new file mode 100755 index 000000000..f637b8f08 --- /dev/null +++ b/qa/workunits/rgw/test_rgw_throttle.sh @@ -0,0 +1,5 @@ +#!/bin/sh -e + +ceph_test_rgw_throttle + +exit 0 diff --git a/qa/workunits/rgw/test_rgw_versioning.py b/qa/workunits/rgw/test_rgw_versioning.py new file mode 100755 index 000000000..fc69e138d --- /dev/null +++ b/qa/workunits/rgw/test_rgw_versioning.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import logging as log +import json +import uuid +import botocore +import time +from common import exec_cmd, create_user, boto_connect +from botocore.config import Config + +""" +Tests behavior of bucket versioning. +""" +# The test cases in this file have been annotated for inventory. +# To extract the inventory (in csv format) use the command: +# +# grep '^ *# TESTCASE' | sed 's/^ *# TESTCASE //' +# +# + +""" Constants """ +USER = 'versioning-tester' +DISPLAY_NAME = 'Versioning Testing' +ACCESS_KEY = 'LTA662PVVDTDWX6M2AB0' +SECRET_KEY = 'pvtchqajgzqx5581t6qbddbkj0bgf3a69qdkjcea' +BUCKET_NAME = 'versioning-bucket' +DATA_POOL = 'default.rgw.buckets.data' + +def main(): + """ + execute versioning tests + """ + create_user(USER, DISPLAY_NAME, ACCESS_KEY, SECRET_KEY) + + connection = boto_connect(ACCESS_KEY, SECRET_KEY, Config(retries = { + 'total_max_attempts': 1, + })) + + # pre-test cleanup + try: + bucket = connection.Bucket(BUCKET_NAME) + bucket.objects.all().delete() + bucket.object_versions.all().delete() + bucket.delete() + except botocore.exceptions.ClientError as e: + if not e.response['Error']['Code'] == 'NoSuchBucket': + raise + + bucket = connection.create_bucket(Bucket=BUCKET_NAME) + connection.BucketVersioning(BUCKET_NAME).enable() + + # reproducer for bug from https://tracker.ceph.com/issues/59663 + # TESTCASE 'verify that index entries and OLH objects are cleaned up after redundant deletes' + log.debug('TEST: verify that index entries and OLH objects are cleaned up after redundant deletes\n') + key = str(uuid.uuid4()) + resp = bucket.Object(key).delete() + assert 'DeleteMarker' in resp, 'DeleteMarker key not present in response' + assert resp['DeleteMarker'], 'DeleteMarker value not True in response' + assert 'VersionId' in resp, 'VersionId key not present in response' + version_id = resp['VersionId'] + bucket.Object(key).delete() + connection.ObjectVersion(bucket.name, key, version_id).delete() + # bucket index should now be empty + out = exec_cmd(f'radosgw-admin bi list --bucket {BUCKET_NAME}') + json_out = json.loads(out.replace(b'\x80', b'0x80')) + assert len(json_out) == 0, 'bucket index was not empty after all objects were deleted' + + (_out, ret) = exec_cmd(f'rados -p {DATA_POOL} ls | grep {key}', check_retcode=False) + assert ret != 0, 'olh object was not cleaned up' + + # TESTCASE 'verify that index entries and OLH objects are cleaned up after index linking error' + log.debug('TEST: verify that index entries and OLH objects are cleaned up after index linking error\n') + key = str(uuid.uuid4()) + try: + exec_cmd('ceph config set client rgw_debug_inject_set_olh_err 2') + time.sleep(1) + bucket.Object(key).delete() + finally: + exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err') + out = exec_cmd(f'radosgw-admin bi list --bucket {BUCKET_NAME}') + json_out = json.loads(out.replace(b'\x80', b'0x80')) + assert len(json_out) == 0, 'bucket index was not empty after op failed' + (_out, ret) = exec_cmd(f'rados -p {DATA_POOL} ls | grep {key}', check_retcode=False) + assert ret != 0, 'olh object was not cleaned up' + + # TESTCASE 'verify that original null object version is intact after failed olh upgrade' + log.debug('TEST: verify that original null object version is intact after failed olh upgrade\n') + connection.BucketVersioning(BUCKET_NAME).suspend() + key = str(uuid.uuid4()) + put_resp = bucket.put_object(Key=key, Body=b"data") + connection.BucketVersioning(BUCKET_NAME).enable() + try: + exec_cmd('ceph config set client rgw_debug_inject_set_olh_err 2') + time.sleep(1) + # expected to fail due to the above error injection + bucket.put_object(Key=key, Body=b"new data") + except Exception as e: + log.debug(e) + finally: + exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err') + get_resp = bucket.Object(key).get() + assert put_resp.e_tag == get_resp['ETag'], 'get did not return null version with correct etag' + + # Clean up + log.debug("Deleting bucket {}".format(BUCKET_NAME)) + bucket.object_versions.all().delete() + bucket.delete() + +main() +log.info("Completed bucket versioning tests") diff --git a/qa/workunits/suites/blogbench.sh b/qa/workunits/suites/blogbench.sh new file mode 100755 index 000000000..a05d8d21c --- /dev/null +++ b/qa/workunits/suites/blogbench.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e + +echo "getting blogbench" +wget http://download.ceph.com/qa/blogbench-1.0.tar.bz2 +#cp /home/gregf/src/blogbench-1.0.tar.bz2 . +tar -xvf blogbench-1.0.tar.bz2 +cd blogbench-1.0/ +echo "making blogbench" +./configure +make +cd src +mkdir blogtest_in +echo "running blogbench" +./blogbench -d blogtest_in diff --git a/qa/workunits/suites/bonnie.sh b/qa/workunits/suites/bonnie.sh new file mode 100755 index 000000000..b60cc6a5e --- /dev/null +++ b/qa/workunits/suites/bonnie.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e + +bonnie_bin=`which bonnie++` +[ $? -eq 1 ] && bonnie_bin=/usr/sbin/bonnie++ + +uid_flags="" +[ "`id -u`" == "0" ] && uid_flags="-u root" + +$bonnie_bin $uid_flags -n 100 diff --git a/qa/workunits/suites/cephfs_journal_tool_smoke.sh b/qa/workunits/suites/cephfs_journal_tool_smoke.sh new file mode 100755 index 000000000..3fe01ed63 --- /dev/null +++ b/qa/workunits/suites/cephfs_journal_tool_smoke.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +set -e +set -x + +export BIN="${BIN:-cephfs-journal-tool --rank=cephfs:0}" +export JOURNAL_FILE=/tmp/journal.bin +export JSON_OUTPUT=/tmp/json.tmp +export BINARY_OUTPUT=/tmp/binary.tmp + +if [ -d $BINARY_OUTPUT ] ; then + rm -rf $BINARY_OUTPUT +fi + +# Check that the import/export stuff really works as expected +# first because it's used as the reset method between +# following checks. +echo "Testing that export/import cycle preserves state" +HEADER_STATE=`$BIN header get` +EVENT_LIST=`$BIN event get list` +$BIN journal export $JOURNAL_FILE +$BIN journal import $JOURNAL_FILE +NEW_HEADER_STATE=`$BIN header get` +NEW_EVENT_LIST=`$BIN event get list` + +if [ ! "$HEADER_STATE" = "$NEW_HEADER_STATE" ] ; then + echo "Import failed to preserve header state" + echo $HEADER_STATE + echo $NEW_HEADER_STATE + exit -1 +fi + +if [ ! "$EVENT_LIST" = "$NEW_EVENT_LIST" ] ; then + echo "Import failed to preserve event state" + echo $EVENT_LIST + echo $NEW_EVENT_LIST + exit -1 +fi + +echo "Testing 'journal' commands..." + +# Simplest thing: print the vital statistics of the journal +$BIN journal inspect +$BIN header get + +# Make a copy of the journal in its original state +$BIN journal export $JOURNAL_FILE +if [ ! -s $JOURNAL_FILE ] ; then + echo "Export to $JOURNAL_FILE failed" + exit -1 +fi + +# Can we execute a journal reset? +$BIN journal reset +$BIN journal inspect +$BIN header get + +echo "Rolling back journal to original state..." +$BIN journal import $JOURNAL_FILE + +echo "Testing 'header' commands..." +$BIN header get +$BIN header set write_pos 123 +$BIN header set expire_pos 123 +$BIN header set trimmed_pos 123 + +echo "Rolling back journal to original state..." +$BIN journal import $JOURNAL_FILE + +echo "Testing 'event' commands..." +$BIN event get summary +$BIN event get --type=UPDATE --path=/ --inode=0 --frag=0x100 summary +$BIN event get json --path $JSON_OUTPUT +if [ ! -s $JSON_OUTPUT ] ; then + echo "Export to $JSON_OUTPUT failed" + exit -1 +fi +$BIN event get binary --path $BINARY_OUTPUT +if [ ! -s $BINARY_OUTPUT ] ; then + echo "Export to $BINARY_OUTPUT failed" + exit -1 +fi +$BIN event recover_dentries summary +$BIN event splice summary + +# Tests finish. +# Metadata objects have been modified by the 'event recover_dentries' command. +# Journal is no long consistent with respect to metadata objects (especially inotable). +# To ensure mds successfully replays its journal, we need to do journal reset. +$BIN journal reset +cephfs-table-tool all reset session + diff --git a/qa/workunits/suites/dbench-short.sh b/qa/workunits/suites/dbench-short.sh new file mode 100755 index 000000000..b0da02275 --- /dev/null +++ b/qa/workunits/suites/dbench-short.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e + +dbench 1 diff --git a/qa/workunits/suites/dbench.sh b/qa/workunits/suites/dbench.sh new file mode 100755 index 000000000..32c893b45 --- /dev/null +++ b/qa/workunits/suites/dbench.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +dbench 1 +dbench 10 diff --git a/qa/workunits/suites/ffsb.sh b/qa/workunits/suites/ffsb.sh new file mode 100755 index 000000000..bf95a05c4 --- /dev/null +++ b/qa/workunits/suites/ffsb.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -e + +mydir=`dirname $0` + +# try it again if the clone is slow and the second time +trap -- 'retry' EXIT +retry() { + rm -rf ffsb + # double the timeout value + timeout 3600 git clone https://git.ceph.com/ffsb.git --depth 1 +} +rm -rf ffsb +timeout 1800 git clone https://git.ceph.com/ffsb.git --depth 1 +trap - EXIT + +cd ffsb +./configure +make +cd .. +mkdir tmp +cd tmp + +for f in $mydir/*.ffsb +do + ../ffsb/ffsb $f +done +cd .. +rm -r tmp ffsb* + diff --git a/qa/workunits/suites/fio.sh b/qa/workunits/suites/fio.sh new file mode 100755 index 000000000..ee69de81c --- /dev/null +++ b/qa/workunits/suites/fio.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -x + +gen_fio_file() { + iter=$1 + f=$2 + cat > randio-$$-${iter}.fio <<EOF +[randio] +blocksize_range=32m:128m +blocksize_unaligned=1 +filesize=10G:20G +readwrite=randrw +runtime=300 +size=20G +filename=${f} +EOF +} + +sudo apt-get -y install fio +for i in $(seq 1 20); do + fcount=$(ls donetestfile* 2>/dev/null | wc -l) + donef="foo" + fiof="bar" + if test ${fcount} -gt 0; then + # choose random file + r=$[ ${RANDOM} % ${fcount} ] + testfiles=( $(ls donetestfile*) ) + donef=${testfiles[${r}]} + fiof=$(echo ${donef} | sed -e "s|done|fio|") + gen_fio_file $i ${fiof} + else + fiof=fiotestfile.$$.$i + donef=donetestfile.$$.$i + gen_fio_file $i ${fiof} + fi + + sudo rm -f ${donef} + sudo fio randio-$$-$i.fio + sudo ln ${fiof} ${donef} + ls -la +done diff --git a/qa/workunits/suites/fsstress.sh b/qa/workunits/suites/fsstress.sh new file mode 100755 index 000000000..e5da5b439 --- /dev/null +++ b/qa/workunits/suites/fsstress.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -ex + +mkdir -p fsstress +pushd fsstress +wget -q -O ltp-full.tgz http://download.ceph.com/qa/ltp-full-20091231.tgz +tar xzf ltp-full.tgz +pushd ltp-full-20091231/testcases/kernel/fs/fsstress +make +BIN=$(readlink -f fsstress) +popd +popd + +T=$(mktemp -d -p .) +"$BIN" -d "$T" -l 1 -n 1000 -p 10 -v +rm -rf -- "$T" diff --git a/qa/workunits/suites/fsx.sh b/qa/workunits/suites/fsx.sh new file mode 100755 index 000000000..0d5ba3a58 --- /dev/null +++ b/qa/workunits/suites/fsx.sh @@ -0,0 +1,16 @@ +#!/bin/sh -x + +set -e + +git clone https://git.ceph.com/xfstests-dev.git +cd xfstests-dev +git checkout 12973fc04fd10d4af086901e10ffa8e48866b735 +make -j4 +cd .. +cp xfstests-dev/ltp/fsx . + +OPTIONS="-z" # don't use zero range calls; not supported by cephfs + +./fsx $OPTIONS 1MB -N 50000 -p 10000 -l 1048576 +./fsx $OPTIONS 10MB -N 50000 -p 10000 -l 10485760 +./fsx $OPTIONS 100MB -N 50000 -p 10000 -l 104857600 diff --git a/qa/workunits/suites/fsync-tester.sh b/qa/workunits/suites/fsync-tester.sh new file mode 100755 index 000000000..6e32786ea --- /dev/null +++ b/qa/workunits/suites/fsync-tester.sh @@ -0,0 +1,13 @@ +#!/bin/sh -x + +set -e + +# To skirt around GPL compatibility issues: +wget http://download.ceph.com/qa/fsync-tester.c +gcc -D_GNU_SOURCE fsync-tester.c -o fsync-tester + +./fsync-tester + +echo $PATH +whereis lsof +lsof diff --git a/qa/workunits/suites/iogen.sh b/qa/workunits/suites/iogen.sh new file mode 100755 index 000000000..d92b87083 --- /dev/null +++ b/qa/workunits/suites/iogen.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -e + +echo "getting iogen" +wget http://download.ceph.com/qa/iogen_3.1p0.tar +tar -xvzf iogen_3.1p0.tar +cd iogen_3.1p0 +echo "making iogen" +make +echo "running iogen" +./iogen -n 5 -s 2g +echo "sleep for 10 min" +sleep 600 +echo "stopping iogen" +./iogen -k + +echo "OK" diff --git a/qa/workunits/suites/iozone-sync.sh b/qa/workunits/suites/iozone-sync.sh new file mode 100755 index 000000000..a37962d30 --- /dev/null +++ b/qa/workunits/suites/iozone-sync.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +# basic tests of O_SYNC, O_DSYNC, O_RSYNC +# test O_SYNC +iozone -c -e -s 512M -r 1M -t 1 -F osync1 -i 0 -i 1 -o +# test O_DSYNC +iozone -c -e -s 512M -r 1M -t 1 -F odsync1 -i 0 -i 1 -+D +# test O_RSYNC +iozone -c -e -s 512M -r 1M -t 1 -F orsync1 -i 0 -i 1 -+r + +# test same file with O_SYNC in one process, buffered in the other +# the sync test starts first, so the buffered test should blow +# past it and +iozone -c -e -s 512M -r 1M -t 1 -F osync2 -i 0 -i 1 -o & +sleep 1 +iozone -c -e -s 512M -r 256K -t 1 -F osync2 -i 0 +wait $! + +# test same file with O_SYNC from different threads +iozone -c -e -s 512M -r 1M -t 2 -F osync3 -i 2 -o diff --git a/qa/workunits/suites/iozone.sh b/qa/workunits/suites/iozone.sh new file mode 100755 index 000000000..7dc50cb0b --- /dev/null +++ b/qa/workunits/suites/iozone.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -ex + +iozone -c -e -s 1024M -r 16K -t 1 -F f1 -i 0 -i 1 +iozone -c -e -s 1024M -r 1M -t 1 -F f2 -i 0 -i 1 +iozone -c -e -s 10240M -r 1M -t 1 -F f3 -i 0 -i 1 diff --git a/qa/workunits/suites/pjd.sh b/qa/workunits/suites/pjd.sh new file mode 100755 index 000000000..bd72f77f2 --- /dev/null +++ b/qa/workunits/suites/pjd.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -e + +wget http://download.ceph.com/qa/pjd-fstest-20090130-RC-aclfixes.tgz +tar zxvf pjd*.tgz +cd pjd-fstest-20090130-RC +make clean +make +cd .. +mkdir tmp +cd tmp +# must be root! +sudo prove -r -v --exec 'bash -x' ../pjd*/tests +cd .. +rm -rf tmp pjd* + diff --git a/qa/workunits/suites/random_write.32.ffsb b/qa/workunits/suites/random_write.32.ffsb new file mode 100644 index 000000000..ba83e470f --- /dev/null +++ b/qa/workunits/suites/random_write.32.ffsb @@ -0,0 +1,48 @@ +# Large file random writes. +# 1024 files, 100MB per file. + +time=300 # 5 min +alignio=1 + +[filesystem0] + location=. + num_files=128 + min_filesize=104857600 # 100 MB + max_filesize=104857600 + reuse=1 +[end0] + +[threadgroup0] + num_threads=32 + + write_random=1 + write_weight=1 + + write_size=5242880 # 5 MB + write_blocksize=4096 + + [stats] + enable_stats=1 + enable_range=1 + + msec_range 0.00 0.01 + msec_range 0.01 0.02 + msec_range 0.02 0.05 + msec_range 0.05 0.10 + msec_range 0.10 0.20 + msec_range 0.20 0.50 + msec_range 0.50 1.00 + msec_range 1.00 2.00 + msec_range 2.00 5.00 + msec_range 5.00 10.00 + msec_range 10.00 20.00 + msec_range 20.00 50.00 + msec_range 50.00 100.00 + msec_range 100.00 200.00 + msec_range 200.00 500.00 + msec_range 500.00 1000.00 + msec_range 1000.00 2000.00 + msec_range 2000.00 5000.00 + msec_range 5000.00 10000.00 + [end] +[end0] diff --git a/qa/workunits/test_telemetry_pacific.sh b/qa/workunits/test_telemetry_pacific.sh new file mode 100755 index 000000000..a971f5883 --- /dev/null +++ b/qa/workunits/test_telemetry_pacific.sh @@ -0,0 +1,23 @@ +#!/bin/bash -ex + +# Set up ident details for cluster +ceph config set mgr mgr/telemetry/channel_ident true +ceph config set mgr mgr/telemetry/organization 'ceph-qa' +ceph config set mgr mgr/telemetry/description 'upgrade test cluster' + +# Opt-in +ceph telemetry on --license sharing-1-0 + +# Check last_opt_revision +LAST_OPT_REVISION=$(ceph config get mgr mgr/telemetry/last_opt_revision) +if [ $LAST_OPT_REVISION -ne 3 ]; then + echo "last_opt_revision is incorrect." + exit 1 +fi + +# Check reports +ceph telemetry show +ceph telemetry show-device +ceph telemetry show-all + +echo OK diff --git a/qa/workunits/test_telemetry_pacific_x.sh b/qa/workunits/test_telemetry_pacific_x.sh new file mode 100755 index 000000000..0e4a832db --- /dev/null +++ b/qa/workunits/test_telemetry_pacific_x.sh @@ -0,0 +1,59 @@ +#!/bin/bash -ex + +# Assert that we're still opted in +LAST_OPT_REVISION=$(ceph config get mgr mgr/telemetry/last_opt_revision) +if [ $LAST_OPT_REVISION -ne 3 ]; then + echo "last_opt_revision is incorrect" + exit 1 +fi + +# Check the warning: +STATUS=$(ceph -s) +if ! [[ $STATUS == *"Telemetry requires re-opt-in"* ]] +then + echo "STATUS does not contain re-opt-in warning" + exit 1 +fi + +# Check new collections +COLLECTIONS=$(ceph telemetry collection ls) +NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics") +for col in ${NEW_COLLECTIONS[@]}; do + if ! [[ $COLLECTIONS == *$col* ]]; + then + echo "COLLECTIONS does not contain" "'"$col"'." + exit 1 + fi +done + +# Run preview commands +ceph telemetry preview +ceph telemetry preview-device +ceph telemetry preview-all + +# Opt in to new collections +ceph telemetry on --license sharing-1-0 +ceph telemetry enable channel perf + +# Check the warning: +timeout=60 +STATUS=$(ceph -s) +until [[ $STATUS != *"Telemetry requires re-opt-in"* ]] || [ $timeout -le 0 ]; do + STATUS=$(ceph -s) + sleep 1 + timeout=$(( timeout - 1 )) +done +if [ $timeout -le 0 ]; then + echo "STATUS should not contain re-opt-in warning at this point" + exit 1 +fi + +# Run show commands +ceph telemetry show +ceph telemetry show-device +ceph telemetry show + +# Opt out +ceph telemetry off + +echo OK diff --git a/qa/workunits/test_telemetry_quincy.sh b/qa/workunits/test_telemetry_quincy.sh new file mode 100755 index 000000000..e8b07ec13 --- /dev/null +++ b/qa/workunits/test_telemetry_quincy.sh @@ -0,0 +1,44 @@ +#!/bin/bash -ex + +# Set up ident details for cluster +ceph config set mgr mgr/telemetry/channel_ident true +ceph config set mgr mgr/telemetry/organization 'ceph-qa' +ceph config set mgr mgr/telemetry/description 'upgrade test cluster' + + +#Run preview commands +ceph telemetry preview +ceph telemetry preview-device +ceph telemetry preview-all + +# Assert that new collections are available +COLLECTIONS=$(ceph telemetry collection ls) +NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics") +for col in ${NEW_COLLECTIONS[@]}; do + if ! [[ $COLLECTIONS == *$col* ]]; + then + echo "COLLECTIONS does not contain" "'"$col"'." + exit 1 + fi +done + +# Opt-in +ceph telemetry on --license sharing-1-0 + +# Enable perf channel +ceph telemetry enable channel perf + +# For quincy, the last_opt_revision remains at 1 since last_opt_revision +# was phased out for fresh installs of quincy. +LAST_OPT_REVISION=$(ceph config get mgr mgr/telemetry/last_opt_revision) +if [ $LAST_OPT_REVISION -ne 1 ]; then + echo "last_opt_revision is incorrect" + exit 1 +fi + +# Run show commands +ceph telemetry show +ceph telemetry show-device +ceph telemetry show-all + +echo OK diff --git a/qa/workunits/test_telemetry_quincy_x.sh b/qa/workunits/test_telemetry_quincy_x.sh new file mode 100755 index 000000000..4734132d0 --- /dev/null +++ b/qa/workunits/test_telemetry_quincy_x.sh @@ -0,0 +1,40 @@ +#!/bin/bash -ex + +# For quincy, the last_opt_revision remains at 1 since last_opt_revision +# was phased out for fresh installs of quincy. +LAST_OPT_REVISION=$(ceph config get mgr mgr/telemetry/last_opt_revision) +if [ $LAST_OPT_REVISION -ne 1 ]; then + echo "last_opt_revision is incorrect" + exit 1 +fi + +# Check the warning: +ceph -s + +COLLECTIONS=$(ceph telemetry collection ls) +NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics") +for col in ${NEW_COLLECTIONS[@]}; do + if ! [[ $COLLECTIONS == *$col* ]]; + then + echo "COLLECTIONS does not contain" "'"$col"'." + exit 1 + fi +done + +#Run preview commands +ceph telemetry preview +ceph telemetry preview-device +ceph telemetry preview-all + +# Opt in to new collections +# Currently, no new collections between latest quincy and reef (dev) + +# Run show commands +ceph telemetry show +ceph telemetry show-device +ceph telemetry show + +# Opt out +ceph telemetry off + +echo OK diff --git a/qa/workunits/true.sh b/qa/workunits/true.sh new file mode 100755 index 000000000..296ef781c --- /dev/null +++ b/qa/workunits/true.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +true diff --git a/qa/workunits/windows/libvirt_vm/autounattend.xml b/qa/workunits/windows/libvirt_vm/autounattend.xml new file mode 100644 index 000000000..c3cdf3171 --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/autounattend.xml @@ -0,0 +1,157 @@ +<?xml version="1.0" encoding="utf-8"?> +<unattend xmlns="urn:schemas-microsoft-com:unattend"> + + <settings pass="windowsPE"> + + <component name="Microsoft-Windows-International-Core-WinPE" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <SetupUILanguage> + <UILanguage>en-US</UILanguage> + </SetupUILanguage> + <SystemLocale>en-US</SystemLocale> + <UILanguage>en-US</UILanguage> + <UserLocale>en-US</UserLocale> + </component> + + <component name="Microsoft-Windows-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + + <DiskConfiguration> + <WillShowUI>OnError</WillShowUI> + <Disk wcm:action="add"> + <CreatePartitions> + <CreatePartition wcm:action="add"> + <Order>1</Order> + <Size>100</Size> + <Type>Primary</Type> + </CreatePartition> + <CreatePartition wcm:action="add"> + <Order>2</Order> + <Extend>true</Extend> + <Type>Primary</Type> + </CreatePartition> + </CreatePartitions> + <ModifyPartitions> + <ModifyPartition wcm:action="add"> + <Active>true</Active> + <Label>Boot</Label> + <Format>NTFS</Format> + <Order>1</Order> + <PartitionID>1</PartitionID> + </ModifyPartition> + <ModifyPartition wcm:action="add"> + <Format>NTFS</Format> + <Order>2</Order> + <PartitionID>2</PartitionID> + <Label>System</Label> + </ModifyPartition> + </ModifyPartitions> + <DiskID>0</DiskID> + <WillWipeDisk>true</WillWipeDisk> + </Disk> + </DiskConfiguration> + + <ImageInstall> + <OSImage> + <InstallTo> + <PartitionID>2</PartitionID> + <DiskID>0</DiskID> + </InstallTo> + <InstallToAvailablePartition>false</InstallToAvailablePartition> + <WillShowUI>OnError</WillShowUI> + <InstallFrom> + <MetaData wcm:action="add"> + <Key>/IMAGE/NAME</Key> + <Value>Windows Server 2019 SERVERSTANDARDCORE</Value> + </MetaData> + </InstallFrom> + </OSImage> + </ImageInstall> + + <UserData> + <!-- Product Key from http://technet.microsoft.com/en-us/library/jj612867.aspx --> + <ProductKey> + <!-- Do not uncomment the Key element if you are using trial ISOs --> + <!-- You must uncomment the Key element (and optionally insert your own key) if you are using retail or volume license ISOs --> + <!-- <Key></Key> --> + <WillShowUI>OnError</WillShowUI> + </ProductKey> + <AcceptEula>true</AcceptEula> + </UserData> + + </component> + + <component name="Microsoft-Windows-PnpCustomizationsWinPE" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <DriverPaths> + <PathAndCredentials wcm:action="add" wcm:keyValue="1"> + <Path>E:\NetKVM\2k19\amd64\</Path> + </PathAndCredentials> + <PathAndCredentials wcm:action="add" wcm:keyValue="2"> + <Path>E:\viostor\2k19\amd64\</Path> + </PathAndCredentials> + <PathAndCredentials wcm:action="add" wcm:keyValue="3"> + <Path>E:\vioserial\2k19\amd64\</Path> + </PathAndCredentials> + </DriverPaths> + </component> + + </settings> + + <settings pass="oobeSystem"> + <component name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + + <VisualEffects> + <FontSmoothing>ClearType</FontSmoothing> + </VisualEffects> + + <UserAccounts> + <!-- + Password to be used only during initial provisioning. + Must be reset with final Sysprep. + --> + <AdministratorPassword> + <Value>Passw0rd</Value> + <PlainText>true</PlainText> + </AdministratorPassword> + </UserAccounts> + + <AutoLogon> + <Password> + <Value>Passw0rd</Value> + <PlainText>true</PlainText> + </Password> + <Enabled>true</Enabled> + <Username>Administrator</Username> + </AutoLogon> + + <ComputerName>*</ComputerName> + + <OOBE> + <NetworkLocation>Work</NetworkLocation> + <HideEULAPage>true</HideEULAPage> + <ProtectYourPC>3</ProtectYourPC> + <SkipMachineOOBE>true</SkipMachineOOBE> + <SkipUserOOBE>true</SkipUserOOBE> + </OOBE> + + <FirstLogonCommands> + + <SynchronousCommand wcm:action="add"> + <CommandLine>%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell -NoLogo -NonInteractive -ExecutionPolicy RemoteSigned -File A:\first-logon.ps1</CommandLine> + <Order>1</Order> + </SynchronousCommand> + + </FirstLogonCommands> + + </component> + + </settings> + + <settings pass="specialize"> + + <component name="Microsoft-Windows-Shell-Setup" processorArchitecture="amd64" publicKeyToken="31bf3856ad364e35" language="neutral" versionScope="nonSxS" xmlns:wcm="http://schemas.microsoft.com/WMIConfig/2002/State" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <TimeZone>UTC</TimeZone> + <ComputerName>*</ComputerName> + </component> + + </settings> + +</unattend> diff --git a/qa/workunits/windows/libvirt_vm/first-logon.ps1 b/qa/workunits/windows/libvirt_vm/first-logon.ps1 new file mode 100644 index 000000000..654b836bb --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/first-logon.ps1 @@ -0,0 +1,42 @@ +$ErrorActionPreference = "Stop" + +. "${PSScriptRoot}\utils.ps1" + +$VIRTIO_WIN_PATH = "E:\" + +# Install QEMU quest agent +Write-Output "Installing QEMU guest agent" +$p = Start-Process -FilePath "msiexec.exe" -ArgumentList @("/i", "${VIRTIO_WIN_PATH}\guest-agent\qemu-ga-x86_64.msi", "/qn") -NoNewWindow -PassThru -Wait +if($p.ExitCode) { + Throw "The QEMU guest agent installation failed. Exit code: $($p.ExitCode)" +} +Write-Output "Successfully installed QEMU guest agent" + +# Install OpenSSH server +Start-ExecuteWithRetry { + Get-WindowsCapability -Online -Name OpenSSH* | Add-WindowsCapability -Online +} + +# Start OpenSSH server +Set-Service -Name "sshd" -StartupType Automatic +Start-Service -Name "sshd" + +# Set PowerShell as default SSH shell +New-ItemProperty -PropertyType String -Force -Name DefaultShell -Path "HKLM:\SOFTWARE\OpenSSH" -Value (Get-Command powershell.exe).Source + +# Create SSH firewall rule +New-NetFirewallRule -Name "sshd" -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 + +# Authorize the SSH key +$authorizedKeysFile = Join-Path $env:ProgramData "ssh\administrators_authorized_keys" +Set-Content -Path $authorizedKeysFile -Value (Get-Content "${PSScriptRoot}\id_rsa.pub") -Encoding ascii +$acl = Get-Acl $authorizedKeysFile +$acl.SetAccessRuleProtection($true, $false) +$administratorsRule = New-Object system.security.accesscontrol.filesystemaccessrule("Administrators", "FullControl", "Allow") +$systemRule = New-Object system.security.accesscontrol.filesystemaccessrule("SYSTEM", "FullControl", "Allow") +$acl.SetAccessRule($administratorsRule) +$acl.SetAccessRule($systemRule) +$acl | Set-Acl + +# Reboot the machine to complete first logon process +Restart-Computer -Force -Confirm:$false diff --git a/qa/workunits/windows/libvirt_vm/setup.ps1 b/qa/workunits/windows/libvirt_vm/setup.ps1 new file mode 100644 index 000000000..550fb274e --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/setup.ps1 @@ -0,0 +1,43 @@ +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$PYTHON3_URL = "https://www.python.org/ftp/python/3.10.4/python-3.10.4-amd64.exe" +$FIO_URL = "https://bsdio.com/fio/releases/fio-3.27-x64.msi" +$VC_REDIST_URL = "https://aka.ms/vs/17/release/vc_redist.x64.exe" + +. "${PSScriptRoot}\utils.ps1" + +function Install-VCRedist { + Write-Output "Installing Visual Studio Redistributable x64" + Install-Tool -URL $VC_REDIST_URL -Params @("/quiet", "/norestart") + Write-Output "Successfully installed Visual Studio Redistributable x64" +} + +function Install-Python3 { + Write-Output "Installing Python3" + Install-Tool -URL $PYTHON3_URL -Params @("/quiet", "InstallAllUsers=1", "PrependPath=1") + Add-ToPathEnvVar -Path @("${env:ProgramFiles}\Python310\", "${env:ProgramFiles}\Python310\Scripts\") + Write-Output "Installing pip dependencies" + Start-ExecuteWithRetry { + Invoke-CommandLine "pip3.exe" "install prettytable" + } + Write-Output "Successfully installed Python3" +} + +function Install-FIO { + Write-Output "Installing FIO" + Install-Tool -URL $FIO_URL -Params @("/qn", "/l*v", "$env:TEMP\fio-install.log", "/norestart") + Write-Output "Successfully installed FIO" +} + +Install-VCRedist +Install-Python3 +Install-FIO + +# Pre-append WNBD and Ceph to PATH +Add-ToPathEnvVar -Path @( + "${env:SystemDrive}\wnbd\binaries", + "${env:SystemDrive}\ceph") + +# This will refresh the PATH for new SSH sessions +Restart-Service -Force -Name "sshd" diff --git a/qa/workunits/windows/libvirt_vm/setup.sh b/qa/workunits/windows/libvirt_vm/setup.sh new file mode 100755 index 000000000..51e91ec42 --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/setup.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -ex + +WINDOWS_SERVER_2019_ISO_URL=${WINDOWS_SERVER_2019_ISO_URL:-"https://software-download.microsoft.com/download/pr/17763.737.190906-2324.rs5_release_svc_refresh_SERVER_EVAL_x64FRE_en-us_1.iso"} +VIRTIO_WIN_ISO_URL=${VIRTIO_WIN_ISO_URL:-"https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/virtio-win.iso"} + +DIR="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)" + +# Use build_utils.sh from ceph-build +curl --retry-max-time 30 --retry 10 -L -o ${DIR}/build_utils.sh https://raw.githubusercontent.com/ceph/ceph-build/main/scripts/build_utils.sh +source ${DIR}/build_utils.sh + +# Helper function to restart the Windows VM +function restart_windows_vm() { + echo "Restarting Windows VM" + ssh_exec "cmd.exe /c 'shutdown.exe /r /t 0 & sc.exe stop sshd'" + SECONDS=0 + TIMEOUT=${1:-600} + while true; do + if [[ $SECONDS -gt $TIMEOUT ]]; then + echo "Timeout waiting for the VM to start" + exit 1 + fi + ssh_exec hostname || { + echo "Cannot execute SSH commands yet" + sleep 10 + continue + } + break + done + echo "Windows VM restarted" +} + +# Install libvirt with KVM +retrycmd_if_failure 5 0 5m sudo apt-get update +retrycmd_if_failure 5 0 10m sudo apt-get install -y qemu-kvm libvirt-daemon-system libvirt-clients virtinst + +# Download ISO images +echo "Downloading virtio-win ISO" +retrycmd_if_failure 5 0 30m curl -C - -L $VIRTIO_WIN_ISO_URL -o ${DIR}/virtio-win.iso +echo "Downloading Windows Server 2019 ISO" +retrycmd_if_failure 5 0 60m curl -C - -L $WINDOWS_SERVER_2019_ISO_URL -o ${DIR}/windows-server-2019.iso + +# Create virtual floppy image with the unattended instructions to install Windows Server 2019 +echo "Creating floppy image" +qemu-img create -f raw ${DIR}/floppy.img 1440k +mkfs.msdos -s 1 ${DIR}/floppy.img +mkdir ${DIR}/floppy +sudo mount ${DIR}/floppy.img ${DIR}/floppy +ssh-keygen -b 2048 -t rsa -f ${DIR}/id_rsa -q -N "" +sudo cp \ + ${DIR}/autounattend.xml \ + ${DIR}/first-logon.ps1 \ + ${DIR}/id_rsa.pub \ + ${DIR}/utils.ps1 \ + ${DIR}/setup.ps1 \ + ${DIR}/floppy/ +sudo umount ${DIR}/floppy +rmdir ${DIR}/floppy + +echo "Starting libvirt VM" +qemu-img create -f qcow2 ${DIR}/ceph-win-ltsc2019.qcow2 50G +VM_NAME="ceph-win-ltsc2019" +sudo virt-install \ + --name $VM_NAME \ + --os-variant win2k19 \ + --boot hd,cdrom \ + --virt-type kvm \ + --graphics spice \ + --cpu host \ + --vcpus 4 \ + --memory 4096 \ + --disk ${DIR}/floppy.img,device=floppy \ + --disk ${DIR}/ceph-win-ltsc2019.qcow2,bus=virtio \ + --disk ${DIR}/windows-server-2019.iso,device=cdrom \ + --disk ${DIR}/virtio-win.iso,device=cdrom \ + --network network=default,model=virtio \ + --controller type=virtio-serial \ + --channel unix,target_type=virtio,name=org.qemu.guest_agent.0 \ + --noautoconsol + +export SSH_USER="administrator" +export SSH_KNOWN_HOSTS_FILE="${DIR}/known_hosts" +export SSH_KEY="${DIR}/id_rsa" + +SECONDS=0 +TIMEOUT=1800 +SLEEP_SECS=30 +while true; do + if [[ $SECONDS -gt $TIMEOUT ]]; then + echo "Timeout waiting for the VM to start" + exit 1 + fi + VM_IP=$(sudo virsh domifaddr --source agent --interface Ethernet --full $VM_NAME | grep ipv4 | awk '{print $4}' | cut -d '/' -f1) || { + echo "Retrying in $SLEEP_SECS seconds" + sleep $SLEEP_SECS + continue + } + ssh-keyscan -H $VM_IP &> $SSH_KNOWN_HOSTS_FILE || { + echo "SSH is not reachable yet" + sleep $SLEEP_SECS + continue + } + SSH_ADDRESS=$VM_IP ssh_exec hostname || { + echo "Cannot execute SSH commands yet" + sleep $SLEEP_SECS + continue + } + break +done +export SSH_ADDRESS=$VM_IP + +scp_upload ${DIR}/utils.ps1 /utils.ps1 +scp_upload ${DIR}/setup.ps1 /setup.ps1 +SSH_TIMEOUT=1h ssh_exec /setup.ps1 + +cd $DIR + +# Get the helper script to download Chacra builds +retrycmd_if_failure 10 5 1m curl -L -o ./get-chacra-bin.py https://raw.githubusercontent.com/ceph/ceph-win32-tests/main/get-bin.py +chmod +x ./get-chacra-bin.py + +# Download latest WNBD build from Chacra +retrycmd_if_failure 10 0 10m ./get-chacra-bin.py --project wnbd --filename wnbd.zip +scp_upload wnbd.zip /wnbd.zip +ssh_exec tar.exe xzvf /wnbd.zip -C / + +# Install WNBD driver +ssh_exec Import-Certificate -FilePath /wnbd/driver/wnbd.cer -Cert Cert:\\LocalMachine\\Root +ssh_exec Import-Certificate -FilePath /wnbd/driver/wnbd.cer -Cert Cert:\\LocalMachine\\TrustedPublisher +ssh_exec /wnbd/binaries/wnbd-client.exe install-driver /wnbd/driver/wnbd.inf +restart_windows_vm +ssh_exec wnbd-client.exe -v + +# Download Ceph Windows build from Chacra +CEPH_REPO_FILE="/etc/apt/sources.list.d/ceph.list" +PROJECT=$(cat $CEPH_REPO_FILE | cut -d ' ' -f3 | tr '\/', ' ' | awk '{print $4}') +BRANCH=$(cat $CEPH_REPO_FILE | cut -d ' ' -f3 | tr '\/', ' ' | awk '{print $5}') +SHA1=$(cat $CEPH_REPO_FILE | cut -d ' ' -f3 | tr '\/', ' ' | awk '{print $6}') +retrycmd_if_failure 10 0 10m ./get-chacra-bin.py --project $PROJECT --branchname $BRANCH --sha1 $SHA1 --filename ceph.zip + +# Install Ceph on Windows +SSH_TIMEOUT=5m scp_upload ./ceph.zip /ceph.zip +SSH_TIMEOUT=10m ssh_exec tar.exe xzvf /ceph.zip -C / +ssh_exec "New-Service -Name ceph-rbd -BinaryPathName 'c:\ceph\rbd-wnbd.exe service'" +ssh_exec Start-Service -Name ceph-rbd +ssh_exec rbd.exe -v + +# Setup Ceph configs and directories +ssh_exec mkdir -force /etc/ceph, /var/run/ceph, /var/log/ceph +for i in $(ls /etc/ceph); do + scp_upload /etc/ceph/$i /etc/ceph/$i +done + +cat << EOF > ${DIR}/connection_info.sh +export SSH_USER="${SSH_USER}" +export SSH_KNOWN_HOSTS_FILE="${SSH_KNOWN_HOSTS_FILE}" +export SSH_KEY="${SSH_KEY}" +export SSH_ADDRESS="${SSH_ADDRESS}" +EOF + +echo "Windows Server 2019 libvirt testing VM is ready" diff --git a/qa/workunits/windows/libvirt_vm/utils.ps1 b/qa/workunits/windows/libvirt_vm/utils.ps1 new file mode 100644 index 000000000..f29ab79f5 --- /dev/null +++ b/qa/workunits/windows/libvirt_vm/utils.ps1 @@ -0,0 +1,130 @@ +function Invoke-CommandLine { + Param( + [Parameter(Mandatory=$true)] + [String]$Command, + [String]$Arguments, + [Int[]]$AllowedExitCodes=@(0) + ) + & $Command $Arguments.Split(" ") + if($LASTEXITCODE -notin $AllowedExitCodes) { + Throw "$Command $Arguments returned a non zero exit code ${LASTEXITCODE}." + } +} + +function Start-ExecuteWithRetry { + Param( + [Parameter(Mandatory=$true)] + [ScriptBlock]$ScriptBlock, + [Int]$MaxRetryCount=10, + [Int]$RetryInterval=3, + [String]$RetryMessage, + [Array]$ArgumentList=@() + ) + $currentErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $retryCount = 0 + while ($true) { + try { + $res = Invoke-Command -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList + $ErrorActionPreference = $currentErrorActionPreference + return $res + } catch [System.Exception] { + $retryCount++ + if ($retryCount -gt $MaxRetryCount) { + $ErrorActionPreference = $currentErrorActionPreference + Throw $_ + } else { + $prefixMsg = "Retry(${retryCount}/${MaxRetryCount})" + if($RetryMessage) { + Write-Host "${prefixMsg} - $RetryMessage" + } elseif($_) { + Write-Host "${prefixMsg} - $($_.ToString())" + } + Start-Sleep $RetryInterval + } + } + } +} + +function Start-FileDownload { + Param( + [Parameter(Mandatory=$true)] + [String]$URL, + [Parameter(Mandatory=$true)] + [String]$Destination, + [Int]$RetryCount=10 + ) + Write-Output "Downloading $URL to $Destination" + Start-ExecuteWithRetry ` + -ScriptBlock { Invoke-CommandLine -Command "curl.exe" -Arguments "-L -s -o $Destination $URL" } ` + -MaxRetryCount $RetryCount ` + -RetryMessage "Failed to download '${URL}'. Retrying" + Write-Output "Successfully downloaded." +} + +function Add-ToPathEnvVar { + Param( + [Parameter(Mandatory=$true)] + [String[]]$Path, + [Parameter(Mandatory=$false)] + [ValidateSet([System.EnvironmentVariableTarget]::User, [System.EnvironmentVariableTarget]::Machine)] + [System.EnvironmentVariableTarget]$Target=[System.EnvironmentVariableTarget]::Machine + ) + $pathEnvVar = [Environment]::GetEnvironmentVariable("PATH", $Target).Split(';') + $currentSessionPath = $env:PATH.Split(';') + foreach($p in $Path) { + if($p -notin $pathEnvVar) { + $pathEnvVar += $p + } + if($p -notin $currentSessionPath) { + $currentSessionPath += $p + } + } + $env:PATH = $currentSessionPath -join ';' + $newPathEnvVar = $pathEnvVar -join ';' + [Environment]::SetEnvironmentVariable("PATH", $newPathEnvVar, $Target) +} + +function Install-Tool { + [CmdletBinding(DefaultParameterSetName = "URL")] + Param( + [Parameter(Mandatory=$true, ParameterSetName = "URL")] + [String]$URL, + [Parameter(Mandatory=$true, ParameterSetName = "LocalPath")] + [String]$LocalPath, + [Parameter(ParameterSetName = "URL")] + [Parameter(ParameterSetName = "LocalPath")] + [String[]]$Params=@(), + [Parameter(ParameterSetName = "URL")] + [Parameter(ParameterSetName = "LocalPath")] + [Int[]]$AllowedExitCodes=@(0) + ) + PROCESS { + $installerPath = $LocalPath + if($PSCmdlet.ParameterSetName -eq "URL") { + $installerPath = Join-Path $env:TEMP $URL.Split('/')[-1] + Start-FileDownload -URL $URL -Destination $installerPath + } + Write-Output "Installing ${installerPath}" + $kwargs = @{ + "FilePath" = $installerPath + "ArgumentList" = $Params + "NoNewWindow" = $true + "PassThru" = $true + "Wait" = $true + } + if((Get-ChildItem $installerPath).Extension -eq '.msi') { + $kwargs["FilePath"] = "msiexec.exe" + $kwargs["ArgumentList"] = @("/i", $installerPath) + $Params + } + $p = Start-Process @kwargs + if($p.ExitCode -notin $AllowedExitCodes) { + Throw "Installation failed. Exit code: $($p.ExitCode)" + } + if($PSCmdlet.ParameterSetName -eq "URL") { + Start-ExecuteWithRetry ` + -ScriptBlock { Remove-Item -Force -Path $installerPath -ErrorAction Stop } ` + -RetryMessage "Failed to remove ${installerPath}. Retrying" + } + } +} diff --git a/qa/workunits/windows/run-tests.ps1 b/qa/workunits/windows/run-tests.ps1 new file mode 100644 index 000000000..6d818f426 --- /dev/null +++ b/qa/workunits/windows/run-tests.ps1 @@ -0,0 +1,29 @@ +$ProgressPreference = "SilentlyContinue" +$ErrorActionPreference = "Stop" + +$scriptLocation = [System.IO.Path]::GetDirectoryName( + $myInvocation.MyCommand.Definition) + +$testRbdWnbd = "$scriptLocation/test_rbd_wnbd.py" + +function safe_exec() { + # Powershell doesn't check the command exit code, we'll need to + # do it ourselves. Also, in case of native commands, it treats stderr + # output as an exception, which is why we'll have to capture it. + cmd /c "$args 2>&1" + if ($LASTEXITCODE) { + throw "Command failed: $args" + } +} + +safe_exec python.exe $testRbdWnbd --test-name RbdTest --iterations 100 +safe_exec python.exe $testRbdWnbd --test-name RbdFioTest --iterations 100 +safe_exec python.exe $testRbdWnbd --test-name RbdStampTest --iterations 100 + +# It can take a while to setup the partition (~10s), we'll use fewer iterations. +safe_exec python.exe $testRbdWnbd --test-name RbdFsTest --iterations 4 +safe_exec python.exe $testRbdWnbd --test-name RbdFsFioTest --iterations 4 +safe_exec python.exe $testRbdWnbd --test-name RbdFsStampTest --iterations 4 + +safe_exec python.exe $testRbdWnbd ` + --test-name RbdResizeFioTest --image-size-mb 64 diff --git a/qa/workunits/windows/run-tests.sh b/qa/workunits/windows/run-tests.sh new file mode 100644 index 000000000..b582491c5 --- /dev/null +++ b/qa/workunits/windows/run-tests.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -ex + +DIR="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)" + +source ${DIR}/libvirt_vm/build_utils.sh +source ${DIR}/libvirt_vm/connection_info.sh + +# Run the Windows tests +scp_upload ${DIR} /windows-workunits +SSH_TIMEOUT=30m ssh_exec powershell.exe -File /windows-workunits/run-tests.ps1 diff --git a/qa/workunits/windows/test_rbd_wnbd.py b/qa/workunits/windows/test_rbd_wnbd.py new file mode 100644 index 000000000..f22810e2e --- /dev/null +++ b/qa/workunits/windows/test_rbd_wnbd.py @@ -0,0 +1,919 @@ +import argparse +import collections +import functools +import json +import logging +import math +import os +import prettytable +import random +import subprocess +import time +import threading +import typing +import uuid +from concurrent import futures + +LOG = logging.getLogger() + +parser = argparse.ArgumentParser(description='rbd-wnbd tests') +parser.add_argument('--test-name', + help='The test to be run.', + default="RbdFioTest") +parser.add_argument('--iterations', + help='Total number of test iterations', + default=1, type=int) +parser.add_argument('--concurrency', + help='The number of tests to run in parallel', + default=4, type=int) +parser.add_argument('--fio-iterations', + help='Total number of benchmark iterations per disk.', + default=1, type=int) +parser.add_argument('--fio-workers', + help='Total number of fio workers per disk.', + default=1, type=int) +parser.add_argument('--fio-depth', + help='The number of concurrent asynchronous operations ' + 'executed per disk', + default=64, type=int) +parser.add_argument('--fio-verify', + help='The mechanism used to validate the written ' + 'data. Examples: crc32c, md5, sha1, null, etc. ' + 'If set to null, the written data will not be ' + 'verified.', + default='crc32c') +parser.add_argument('--bs', + help='Benchmark block size.', + default="2M") +parser.add_argument('--op', + help='Benchmark operation. ' + 'Examples: read, randwrite, rw, etc.', + default="rw") +parser.add_argument('--image-prefix', + help='The image name prefix.', + default="cephTest-") +parser.add_argument('--image-size-mb', + help='The image size in megabytes.', + default=1024, type=int) +parser.add_argument('--map-timeout', + help='Image map timeout.', + default=60, type=int) +parser.add_argument('--skip-enabling-disk', action='store_true', + help='If set, the disk will not be turned online and the ' + 'read-only flag will not be removed. Useful when ' + 'the SAN policy is set to "onlineAll".') +parser.add_argument('--verbose', action='store_true', + help='Print info messages.') +parser.add_argument('--debug', action='store_true', + help='Print debug messages.') +parser.add_argument('--stop-on-error', action='store_true', + help='Stop testing when hitting errors.') +parser.add_argument('--skip-cleanup-on-error', action='store_true', + help='Skip cleanup when hitting errors.') + + +class CephTestException(Exception): + msg_fmt = "An exception has been encountered." + + def __init__(self, message: str = None, **kwargs): + self.kwargs = kwargs + if not message: + message = self.msg_fmt % kwargs + self.message = message + super(CephTestException, self).__init__(message) + + +class CommandFailed(CephTestException): + msg_fmt = ( + "Command failed: %(command)s. " + "Return code: %(returncode)s. " + "Stdout: %(stdout)s. Stderr: %(stderr)s.") + + +class CephTestTimeout(CephTestException): + msg_fmt = "Operation timeout." + + +def setup_logging(log_level: int = logging.INFO): + handler = logging.StreamHandler() + handler.setLevel(log_level) + + log_fmt = '[%(asctime)s] %(levelname)s - %(message)s' + formatter = logging.Formatter(log_fmt) + handler.setFormatter(formatter) + + LOG.addHandler(handler) + LOG.setLevel(logging.DEBUG) + + +def retry_decorator(timeout: int = 60, + retry_interval: int = 2, + silent_interval: int = 10, + additional_details: str = "", + retried_exceptions: + typing.Union[ + typing.Type[Exception], + collections.abc.Iterable[ + typing.Type[Exception]]] = Exception): + def wrapper(f: typing.Callable[..., typing.Any]): + @functools.wraps(f) + def inner(*args, **kwargs): + tstart: float = time.time() + elapsed: float = 0 + exc = None + details = additional_details or "%s failed" % f.__qualname__ + + while elapsed < timeout or not timeout: + try: + return f(*args, **kwargs) + except retried_exceptions as ex: + exc = ex + elapsed = time.time() - tstart + if elapsed > silent_interval: + level = logging.WARNING + else: + level = logging.DEBUG + LOG.log(level, + "Exception: %s. Additional details: %s. " + "Time elapsed: %d. Timeout: %d", + ex, details, elapsed, timeout) + + time.sleep(retry_interval) + elapsed = time.time() - tstart + + msg = ( + "Operation timed out. Exception: %s. Additional details: %s. " + "Time elapsed: %d. Timeout: %d.") + raise CephTestTimeout( + msg % (exc, details, elapsed, timeout)) + return inner + return wrapper + + +def execute(*args, **kwargs): + LOG.debug("Executing: %s", args) + result = subprocess.run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs) + LOG.debug("Command %s returned %d.", args, result.returncode) + if result.returncode: + exc = CommandFailed( + command=args, returncode=result.returncode, + stdout=result.stdout, stderr=result.stderr) + LOG.error(exc) + raise exc + return result + + +def ps_execute(*args, **kwargs): + # Disable PS progress bar, causes issues when invoked remotely. + prefix = "$global:ProgressPreference = 'SilentlyContinue' ; " + return execute( + "powershell.exe", "-NonInteractive", + "-Command", prefix, *args, **kwargs) + + +def array_stats(array: list): + mean = sum(array) / len(array) if len(array) else 0 + variance = (sum((i - mean) ** 2 for i in array) / len(array) + if len(array) else 0) + std_dev = math.sqrt(variance) + sorted_array = sorted(array) + + return { + 'min': min(array) if len(array) else 0, + 'max': max(array) if len(array) else 0, + 'sum': sum(array) if len(array) else 0, + 'mean': mean, + 'median': sorted_array[len(array) // 2] if len(array) else 0, + 'max_90': sorted_array[int(len(array) * 0.9)] if len(array) else 0, + 'min_90': sorted_array[int(len(array) * 0.1)] if len(array) else 0, + 'variance': variance, + 'std_dev': std_dev, + 'count': len(array) + } + + +class Tracer: + data: collections.OrderedDict = collections.OrderedDict() + lock = threading.Lock() + + @classmethod + def trace(cls, func): + def wrapper(*args, **kwargs): + tstart = time.time() + exc_str = None + + # Preserve call order + with cls.lock: + if func.__qualname__ not in cls.data: + cls.data[func.__qualname__] = list() + + try: + return func(*args, **kwargs) + except Exception as exc: + exc_str = str(exc) + raise + finally: + tend = time.time() + + with cls.lock: + cls.data[func.__qualname__] += [{ + "duration": tend - tstart, + "error": exc_str, + }] + + return wrapper + + @classmethod + def get_results(cls): + stats = collections.OrderedDict() + for f in cls.data.keys(): + stats[f] = array_stats([i['duration'] for i in cls.data[f]]) + errors = [] + for i in cls.data[f]: + if i['error']: + errors.append(i['error']) + + stats[f]['errors'] = errors + return stats + + @classmethod + def print_results(cls): + r = cls.get_results() + + table = prettytable.PrettyTable(title="Duration (s)") + table.field_names = [ + "function", "min", "max", "total", + "mean", "median", "std_dev", + "max 90%", "min 90%", "count", "errors"] + table.float_format = ".4" + for f, s in r.items(): + table.add_row([f, s['min'], s['max'], s['sum'], + s['mean'], s['median'], s['std_dev'], + s['max_90'], s['min_90'], + s['count'], len(s['errors'])]) + print(table) + + +class RbdImage(object): + def __init__(self, + name: str, + size_mb: int, + is_shared: bool = True, + disk_number: int = -1, + mapped: bool = False): + self.name = name + self.size_mb = size_mb + self.is_shared = is_shared + self.disk_number = disk_number + self.mapped = mapped + self.removed = False + self.drive_letter = "" + + @classmethod + @Tracer.trace + def create(cls, + name: str, + size_mb: int = 1024, + is_shared: bool = True): + LOG.info("Creating image: %s. Size: %s.", name, "%sM" % size_mb) + cmd = ["rbd", "create", name, "--size", "%sM" % size_mb] + if is_shared: + cmd += ["--image-shared"] + execute(*cmd) + + return RbdImage(name, size_mb, is_shared) + + @Tracer.trace + def get_disk_number(self, + timeout: int = 60, + retry_interval: int = 2): + @retry_decorator( + retried_exceptions=CephTestException, + timeout=timeout, + retry_interval=retry_interval) + def _get_disk_number(): + LOG.info("Retrieving disk number: %s", self.name) + + result = execute("rbd-wnbd", "show", self.name, "--format=json") + disk_info = json.loads(result.stdout) + disk_number = disk_info["disk_number"] + if disk_number > 0: + LOG.debug("Image %s disk number: %d", self.name, disk_number) + return disk_number + + raise CephTestException( + f"Could not get disk number: {self.name}.") + + return _get_disk_number() + + @Tracer.trace + def _wait_for_disk(self, + timeout: int = 60, + retry_interval: int = 2): + @retry_decorator( + retried_exceptions=(FileNotFoundError, OSError), + additional_details="the mapped disk isn't available yet", + timeout=timeout, + retry_interval=retry_interval) + def wait_for_disk(): + LOG.debug("Waiting for disk to be accessible: %s %s", + self.name, self.path) + + with open(self.path, 'rb'): + pass + + return wait_for_disk() + + @property + def path(self): + return f"\\\\.\\PhysicalDrive{self.disk_number}" + + @Tracer.trace + @retry_decorator(additional_details="couldn't clear disk read-only flag") + def set_writable(self): + ps_execute( + "Set-Disk", "-Number", str(self.disk_number), + "-IsReadOnly", "$false") + + @Tracer.trace + @retry_decorator(additional_details="couldn't bring the disk online") + def set_online(self): + ps_execute( + "Set-Disk", "-Number", str(self.disk_number), + "-IsOffline", "$false") + + @Tracer.trace + def map(self, timeout: int = 60): + LOG.info("Mapping image: %s", self.name) + tstart = time.time() + + execute("rbd-wnbd", "map", self.name) + self.mapped = True + + self.disk_number = self.get_disk_number(timeout=timeout) + + elapsed = time.time() - tstart + self._wait_for_disk(timeout=timeout - elapsed) + + @Tracer.trace + def unmap(self): + if self.mapped: + LOG.info("Unmapping image: %s", self.name) + execute("rbd-wnbd", "unmap", self.name) + self.mapped = False + + @Tracer.trace + def remove(self): + if not self.removed: + LOG.info("Removing image: %s", self.name) + execute("rbd", "rm", self.name) + self.removed = True + + def cleanup(self): + try: + self.unmap() + finally: + self.remove() + + @Tracer.trace + @retry_decorator() + def _init_disk(self): + cmd = f"Get-Disk -Number {self.disk_number} | Initialize-Disk" + ps_execute(cmd) + + @Tracer.trace + @retry_decorator() + def _create_partition(self): + cmd = (f"Get-Disk -Number {self.disk_number} | " + "New-Partition -AssignDriveLetter -UseMaximumSize") + ps_execute(cmd) + + @Tracer.trace + @retry_decorator() + def _format_volume(self): + cmd = ( + f"(Get-Partition -DiskNumber {self.disk_number}" + " | ? { $_.DriveLetter }) | Format-Volume -Force -Confirm:$false") + ps_execute(cmd) + + @Tracer.trace + @retry_decorator() + def _get_drive_letter(self): + cmd = (f"(Get-Partition -DiskNumber {self.disk_number}" + " | ? { $_.DriveLetter }).DriveLetter") + result = ps_execute(cmd) + + # The PowerShell command will place a null character if no drive letter + # is available. For example, we can receive "\x00\r\n". + self.drive_letter = result.stdout.decode().strip() + if not self.drive_letter.isalpha() or len(self.drive_letter) != 1: + raise CephTestException( + "Invalid drive letter received: %s" % self.drive_letter) + + @Tracer.trace + def init_fs(self): + if not self.mapped: + raise CephTestException("Unable to create fs, image not mapped.") + + LOG.info("Initializing fs, image: %s.", self.name) + + self._init_disk() + self._create_partition() + self._format_volume() + self._get_drive_letter() + + @Tracer.trace + def get_fs_capacity(self): + if not self.drive_letter: + raise CephTestException("No drive letter available") + + cmd = f"(Get-Volume -DriveLetter {self.drive_letter}).Size" + result = ps_execute(cmd) + + return int(result.stdout.decode().strip()) + + @Tracer.trace + def resize(self, new_size_mb, allow_shrink=False): + LOG.info( + "Resizing image: %s. New size: %s MB, old size: %s MB", + self.name, new_size_mb, self.size_mb) + + cmd = ["rbd", "resize", self.name, + "--size", f"{new_size_mb}M", "--no-progress"] + if allow_shrink: + cmd.append("--allow-shrink") + + execute(*cmd) + + self.size_mb = new_size_mb + + @Tracer.trace + def get_disk_size(self): + """Retrieve the virtual disk size (bytes) reported by Windows.""" + cmd = f"(Get-Disk -Number {self.disk_number}).Size" + result = ps_execute(cmd) + + disk_size = result.stdout.decode().strip() + if not disk_size.isdigit(): + raise CephTestException( + "Invalid disk size received: %s" % disk_size) + + return int(disk_size) + + @Tracer.trace + @retry_decorator(timeout=30) + def wait_for_disk_resize(self): + # After resizing the rbd image, the daemon is expected to receive + # the notification, inform the WNBD driver and then trigger a disk + # rescan (IOCTL_DISK_UPDATE_PROPERTIES). This might take a few seconds, + # so we'll need to do some polling. + disk_size = self.get_disk_size() + disk_size_mb = disk_size // (1 << 20) + + if disk_size_mb != self.size_mb: + raise CephTestException( + "The disk size hasn't been updated yet. Retrieved size: " + f"{disk_size_mb}MB. Expected size: {self.size_mb}MB.") + + +class RbdTest(object): + image: RbdImage + + requires_disk_online = False + requires_disk_write = False + + def __init__(self, + image_prefix: str = "cephTest-", + image_size_mb: int = 1024, + map_timeout: int = 60, + **kwargs): + self.image_size_mb = image_size_mb + self.image_name = image_prefix + str(uuid.uuid4()) + self.map_timeout = map_timeout + self.skip_enabling_disk = kwargs.get("skip_enabling_disk") + + @Tracer.trace + def initialize(self): + self.image = RbdImage.create( + self.image_name, + self.image_size_mb) + self.image.map(timeout=self.map_timeout) + + if not self.skip_enabling_disk: + if self.requires_disk_write: + self.image.set_writable() + + if self.requires_disk_online: + self.image.set_online() + + def run(self): + pass + + def cleanup(self): + if self.image: + self.image.cleanup() + + @classmethod + def print_results(cls, + title: str = "Test results", + description: str = None): + pass + + +class RbdFsTestMixin(object): + # Windows disks must be turned online before accessing partitions. + requires_disk_online = True + requires_disk_write = True + + @Tracer.trace + def initialize(self): + super(RbdFsTestMixin, self).initialize() + + self.image.init_fs() + + def get_subpath(self, *args): + drive_path = f"{self.image.drive_letter}:\\" + return os.path.join(drive_path, *args) + + +class RbdFsTest(RbdFsTestMixin, RbdTest): + pass + + +class RbdFioTest(RbdTest): + data: typing.DefaultDict[str, typing.List[typing.Dict[str, str]]] = ( + collections.defaultdict(list)) + lock = threading.Lock() + + def __init__(self, + *args, + fio_size_mb: int = None, + iterations: int = 1, + workers: int = 1, + bs: str = "2M", + iodepth: int = 64, + op: str = "rw", + verify: str = "crc32c", + **kwargs): + + super(RbdFioTest, self).__init__(*args, **kwargs) + + self.fio_size_mb = fio_size_mb or self.image_size_mb + self.iterations = iterations + self.workers = workers + self.bs = bs + self.iodepth = iodepth + self.op = op + if op not in ("read", "randread"): + self.requires_disk_write = True + self.verify = verify + + def process_result(self, raw_fio_output: str): + result = json.loads(raw_fio_output) + with self.lock: + for job in result["jobs"]: + # Fio doesn't support trim on Windows + for op in ['read', 'write']: + if op in job: + self.data[op].append({ + 'error': job['error'], + 'io_bytes': job[op]['io_bytes'], + 'bw_bytes': job[op]['bw_bytes'], + 'runtime': job[op]['runtime'] / 1000, # seconds + 'total_ios': job[op]['short_ios'], + 'short_ios': job[op]['short_ios'], + 'dropped_ios': job[op]['short_ios'], + 'clat_ns_min': job[op]['clat_ns']['min'], + 'clat_ns_max': job[op]['clat_ns']['max'], + 'clat_ns_mean': job[op]['clat_ns']['mean'], + 'clat_ns_stddev': job[op]['clat_ns']['stddev'], + 'clat_ns_10': job[op].get('clat_ns', {}) + .get('percentile', {}) + .get('10.000000', 0), + 'clat_ns_90': job[op].get('clat_ns', {}) + .get('percentile', {}) + .get('90.000000', 0) + }) + + def _get_fio_path(self): + return self.image.path + + @Tracer.trace + def _run_fio(self, fio_size_mb=None): + LOG.info("Starting FIO test.") + cmd = [ + "fio", "--thread", "--output-format=json", + "--randrepeat=%d" % self.iterations, + "--direct=1", "--name=test", + "--bs=%s" % self.bs, "--iodepth=%s" % self.iodepth, + "--size=%sM" % (fio_size_mb or self.fio_size_mb), + "--readwrite=%s" % self.op, + "--numjobs=%s" % self.workers, + "--filename=%s" % self._get_fio_path(), + ] + if self.verify: + cmd += ["--verify=%s" % self.verify] + result = execute(*cmd) + LOG.info("Completed FIO test.") + self.process_result(result.stdout) + + @Tracer.trace + def run(self): + self._run_fio() + + @classmethod + def print_results(cls, + title: str = "Benchmark results", + description: str = None): + if description: + title = "%s (%s)" % (title, description) + + for op in cls.data.keys(): + op_title = "%s op=%s" % (title, op) + + table = prettytable.PrettyTable(title=op_title) + table.field_names = ["stat", "min", "max", "mean", + "median", "std_dev", + "max 90%", "min 90%", "total"] + table.float_format = ".4" + + op_data = cls.data[op] + + s = array_stats([float(i["bw_bytes"]) / 1000_000 for i in op_data]) + table.add_row(["bandwidth (MB/s)", + s['min'], s['max'], s['mean'], + s['median'], s['std_dev'], + s['max_90'], s['min_90'], 'N/A']) + + s = array_stats([float(i["runtime"]) for i in op_data]) + table.add_row(["duration (s)", + s['min'], s['max'], s['mean'], + s['median'], s['std_dev'], + s['max_90'], s['min_90'], s['sum']]) + + s = array_stats([i["error"] for i in op_data]) + table.add_row(["errors", + s['min'], s['max'], s['mean'], + s['median'], s['std_dev'], + s['max_90'], s['min_90'], s['sum']]) + + s = array_stats([i["short_ios"] for i in op_data]) + table.add_row(["incomplete IOs", + s['min'], s['max'], s['mean'], + s['median'], s['std_dev'], + s['max_90'], s['min_90'], s['sum']]) + + s = array_stats([i["dropped_ios"] for i in op_data]) + table.add_row(["dropped IOs", + s['min'], s['max'], s['mean'], + s['median'], s['std_dev'], + s['max_90'], s['min_90'], s['sum']]) + + clat_min = array_stats([i["clat_ns_min"] for i in op_data]) + clat_max = array_stats([i["clat_ns_max"] for i in op_data]) + clat_mean = array_stats([i["clat_ns_mean"] for i in op_data]) + clat_stddev = math.sqrt( + sum([float(i["clat_ns_stddev"]) ** 2 for i in op_data]) / len(op_data) + if len(op_data) else 0) + clat_10 = array_stats([i["clat_ns_10"] for i in op_data]) + clat_90 = array_stats([i["clat_ns_90"] for i in op_data]) + # For convenience, we'll convert it from ns to seconds. + table.add_row(["completion latency (s)", + clat_min['min'] / 1e+9, + clat_max['max'] / 1e+9, + clat_mean['mean'] / 1e+9, + clat_mean['median'] / 1e+9, + clat_stddev / 1e+9, + clat_10['mean'] / 1e+9, + clat_90['mean'] / 1e+9, + clat_mean['sum'] / 1e+9]) + print(table) + + +class RbdResizeFioTest(RbdFioTest): + """Image resize test. + + This test extends and then shrinks the image, performing FIO tests to + validate the resized image. + """ + + @Tracer.trace + def run(self): + self.image.resize(self.image_size_mb * 2) + self.image.wait_for_disk_resize() + + self._run_fio(fio_size_mb=self.image_size_mb * 2) + + self.image.resize(self.image_size_mb // 2, allow_shrink=True) + self.image.wait_for_disk_resize() + + self._run_fio(fio_size_mb=self.image_size_mb // 2) + + # Just like rbd-nbd, rbd-wnbd is masking out-of-bounds errors. + # For this reason, we don't have a negative test that writes + # passed the disk boundary. + + +class RbdFsFioTest(RbdFsTestMixin, RbdFioTest): + def initialize(self): + super(RbdFsFioTest, self).initialize() + + if not self.fio_size_mb or self.fio_size_mb == self.image_size_mb: + # Out of caution, we'll use up to 80% of the FS by default + self.fio_size_mb = int( + self.image.get_fs_capacity() * 0.8 / (1024 * 1024)) + + @staticmethod + def _fio_escape_path(path): + # FIO allows specifying multiple files separated by colon. + # This means that ":" has to be escaped, so + # F:\filename becomes F\:\filename. + return path.replace(":", "\\:") + + def _get_fio_path(self): + return self._fio_escape_path(self.get_subpath("test-fio")) + + +class RbdStampTest(RbdTest): + requires_disk_write = True + + _write_open_mode = "rb+" + _read_open_mode = "rb" + _expect_path_exists = True + + @staticmethod + def _rand_float(min_val: float, max_val: float): + return min_val + (random.random() * max_val - min_val) + + def _get_stamp(self): + buff = self.image_name.encode() + padding = 512 - len(buff) + buff += b'\0' * padding + return buff + + def _get_stamp_path(self): + return self.image.path + + @Tracer.trace + def _write_stamp(self): + with open(self._get_stamp_path(), self._write_open_mode) as disk: + stamp = self._get_stamp() + disk.write(stamp) + + @Tracer.trace + def _read_stamp(self): + with open(self._get_stamp_path(), self._read_open_mode) as disk: + return disk.read(len(self._get_stamp())) + + @Tracer.trace + def run(self): + if self._expect_path_exists: + # Wait up to 5 seconds and then check the disk, ensuring that + # nobody else wrote to it. This is particularly useful when + # running a high number of tests in parallel, ensuring that + # we aren't writing to the wrong disk. + time.sleep(self._rand_float(0, 5)) + + stamp = self._read_stamp() + assert stamp == b'\0' * len(self._get_stamp()) + + self._write_stamp() + + stamp = self._read_stamp() + assert stamp == self._get_stamp() + + +class RbdFsStampTest(RbdFsTestMixin, RbdStampTest): + _write_open_mode = "wb" + _expect_path_exists = False + + def _get_stamp_path(self): + return self.get_subpath("test-stamp") + + +class TestRunner(object): + def __init__(self, + test_cls: typing.Type[RbdTest], + test_params: dict = {}, + iterations: int = 1, + workers: int = 1, + stop_on_error: bool = False, + cleanup_on_error: bool = True): + self.test_cls = test_cls + self.test_params = test_params + self.iterations = iterations + self.workers = workers + self.executor = futures.ThreadPoolExecutor(max_workers=workers) + self.lock = threading.Lock() + self.completed = 0 + self.errors = 0 + self.stopped = False + self.stop_on_error = stop_on_error + self.cleanup_on_error = cleanup_on_error + + @Tracer.trace + def run(self): + tasks = [] + for i in range(self.iterations): + task = self.executor.submit(self.run_single_test) + tasks.append(task) + + LOG.info("Waiting for %d tests to complete.", self.iterations) + for task in tasks: + task.result() + + def run_single_test(self): + failed = False + if self.stopped: + return + + try: + test = self.test_cls(**self.test_params) + test.initialize() + test.run() + except KeyboardInterrupt: + LOG.warning("Received Ctrl-C.") + self.stopped = True + except Exception as ex: + failed = True + if self.stop_on_error: + self.stopped = True + with self.lock: + self.errors += 1 + LOG.exception( + "Test exception: %s. Total exceptions: %d", + ex, self.errors) + finally: + if not failed or self.cleanup_on_error: + try: + test.cleanup() + except KeyboardInterrupt: + LOG.warning("Received Ctrl-C.") + self.stopped = True + # Retry the cleanup + test.cleanup() + except Exception: + LOG.exception("Test cleanup failed.") + + with self.lock: + self.completed += 1 + LOG.info("Completed tests: %d. Pending: %d", + self.completed, self.iterations - self.completed) + + +TESTS: typing.Dict[str, typing.Type[RbdTest]] = { + 'RbdTest': RbdTest, + 'RbdFioTest': RbdFioTest, + 'RbdResizeFioTest': RbdResizeFioTest, + 'RbdStampTest': RbdStampTest, + # FS tests + 'RbdFsTest': RbdFsTest, + 'RbdFsFioTest': RbdFsFioTest, + 'RbdFsStampTest': RbdFsStampTest, +} + +if __name__ == '__main__': + args = parser.parse_args() + + log_level = logging.WARNING + if args.verbose: + log_level = logging.INFO + if args.debug: + log_level = logging.DEBUG + setup_logging(log_level) + + test_params = dict( + image_size_mb=args.image_size_mb, + image_prefix=args.image_prefix, + bs=args.bs, + op=args.op, + verify=args.fio_verify, + iodepth=args.fio_depth, + map_timeout=args.map_timeout, + skip_enabling_disk=args.skip_enabling_disk, + ) + + try: + test_cls = TESTS[args.test_name] + except KeyError: + raise CephTestException("Unkown test: {}".format(args.test_name)) + + runner = TestRunner( + test_cls, + test_params=test_params, + iterations=args.iterations, + workers=args.concurrency, + stop_on_error=args.stop_on_error, + cleanup_on_error=not args.skip_cleanup_on_error) + runner.run() + + Tracer.print_results() + test_cls.print_results( + description="count: %d, concurrency: %d" % + (args.iterations, args.concurrency)) + + assert runner.errors == 0, f"encountered {runner.errors} error(s)." |