diff options
Diffstat (limited to 'qa/workunits/rgw/test_rgw_orphan_list.sh')
| -rwxr-xr-x | qa/workunits/rgw/test_rgw_orphan_list.sh | 519 |
1 files changed, 519 insertions, 0 deletions
diff --git a/qa/workunits/rgw/test_rgw_orphan_list.sh b/qa/workunits/rgw/test_rgw_orphan_list.sh new file mode 100755 index 000000000..34d550cea --- /dev/null +++ b/qa/workunits/rgw/test_rgw_orphan_list.sh @@ -0,0 +1,519 @@ +#!/usr/bin/env bash + +# set -x +set -e + +# if defined, debug messages will be displayed and prepended with the string +# debug="DEBUG" + +huge_size=5100 # in megabytes +big_size=7 # in megabytes + +huge_obj=/tmp/huge_obj.temp.$$ +big_obj=/tmp/big_obj.temp.$$ +empty_obj=/tmp/empty_obj.temp.$$ + +fifo=/tmp/orphan-fifo.$$ +awscli_dir=${HOME}/awscli_temp +export PATH=${PATH}:${awscli_dir} + +rgw_host=$(hostname --fqdn) +if echo "$rgw_host" | grep -q '\.' ; then + : +else + host_domain=".front.sepia.ceph.com" + echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain" + rgw_host="${rgw_host}${host_domain}" +fi +rgw_port=80 + +echo "Fully Qualified Domain Name: $rgw_host" + +success() { + echo OK. + exit 0 +} + +######################################################################## +# INSTALL AND CONFIGURE TOOLING + +install_awscli() { + # NB: this does verify authenticity and integrity of downloaded + # file; see + # https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html + here="$(pwd)" + cd "$HOME" + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + mkdir -p $awscli_dir + ./aws/install -i $awscli_dir + cd "$here" +} + +uninstall_awscli() { + here="$(pwd)" + cd "$HOME" + rm -rf $awscli_dir ./aws awscliv2.zip + cd "$here" +} + +sudo yum -y install s3cmd +sudo yum -y install python3-setuptools +sudo yum -y install python3-pip +sudo pip3 install --upgrade setuptools +sudo pip3 install python-swiftclient + +# get ready for transition from s3cmd to awscli +if false ;then + install_awscli + aws --version + uninstall_awscli +fi + +s3config=/tmp/s3config.$$ + +# do not include the port when it is 80; the host base is used in the +# v4 signature and it needs to follow this convention for signatures +# to match +if [ "$rgw_port" -ne 80 ] ;then + s3_host_base="${rgw_host}:${rgw_port}" +else + s3_host_base="$rgw_host" +fi + +cat >${s3config} <<EOF +[default] +host_base = $s3_host_base +access_key = 0555b35654ad1656d804 +secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== +bucket_location = us-east-1 +check_ssl_certificate = True +check_ssl_hostname = True +default_mime_type = binary/octet-stream +delete_removed = False +dry_run = False +enable_multipart = True +encoding = UTF-8 +encrypt = False +follow_symlinks = False +force = False +guess_mime_type = True +host_bucket = anything.with.three.dots +multipart_chunk_size_mb = 15 +multipart_max_chunks = 10000 +recursive = False +recv_chunk = 65536 +send_chunk = 65536 +signature_v2 = False +socket_timeout = 300 +use_https = False +use_mime_magic = True +verbosity = WARNING +EOF + + +# set up swift authentication +export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0 +export ST_USER=test:tester +export ST_KEY=testing + +create_users() { + # Create S3 user + local akey='0555b35654ad1656d804' + local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==' + radosgw-admin user create --uid testid \ + --access-key $akey --secret $skey \ + --display-name 'M. Tester' --email tester@ceph.com + + # Create Swift user + radosgw-admin user create --subuser=test:tester \ + --display-name=Tester-Subuser --key-type=swift \ + --secret=testing --access=full +} + +myswift() { + if [ -n "$debug" ] ;then + echo "${debug}: swift --verbose --debug $@" + fi + swift --verbose --debug "$@" + local code=$? + if [ $code -ne 0 ] ;then + echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@"" + exit $code + fi +} + +mys3cmd() { + if [ -n "$debug" ] ;then + echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@" + fi + s3cmd --config=${s3config} --verbose --debug "$@" + local code=$? + if [ $code -ne 0 ] ;then + echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@"" + exit $code + fi +} + +mys3uploadkill() { + if [ $# -ne 5 ] ;then + echo "$0: error expecting 5 arguments" + exit 1 + fi + + local_file="$1" + remote_bkt="$2" + remote_obj="$3" + fifo="$4" + stop_part="$5" + + mkfifo $fifo + + s3cmd --config=${s3config} put $local_file \ + s3://${remote_bkt}/${remote_obj} \ + --progress \ + --multipart-chunk-size-mb=5 >$fifo & + set +e # don't allow errors to stop script + while read line ;do + echo "$line" | grep --quiet "part $stop_part " + if [ ${PIPESTATUS[1]} -eq 0 ] ;then + kill -9 $(jobs -p) + break + fi + done <$fifo + set -e + + rm -f $fifo +} + +mys3upload() { + obj=$1 + bucket=$2 + dest_obj=$3 + + mys3cmd put -q $obj s3://${bucket}/$dest_obj +} + +######################################################################## +# PREP + +create_users +dd if=/dev/urandom of=$big_obj bs=1M count=${big_size} +dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size} +touch $empty_obj + +quick_tests() { + echo TRY A SWIFT COMMAND + myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test + myswift list + myswift list swift-plain-ctr + + echo TRY A RADOSGW-ADMIN COMMAND + radosgw-admin bucket list # make sure rgw is up and running +} + +######################################################################## +# S3 TESTS + +#################################### +# regular multipart test + +mys3cmd mb s3://multipart-bkt +mys3upload $huge_obj multipart-bkt multipart-obj +mys3cmd ls +mys3cmd ls s3://multipart-bkt + +#################################### +# multipart test with incomplete uploads + +bkt="incomplete-mp-bkt-1" + +mys3cmd mb s3://$bkt + +mys3uploadkill $huge_obj $bkt incomplete-mp-obj-c $fifo 20 + +# generate an incomplete multipart with more than 1,000 parts +mys3uploadkill $huge_obj $bkt incomplete-mp-obj-b $fifo 1005 + +# generate more than 1000 incomplet multiparts +for c in $(seq 1005) ;do + mys3uploadkill $huge_obj $bkt incomplete-mp-obj-c-$c $fifo 3 +done + +#################################### +# resharded bucket + +bkt=resharded-bkt-1 + +mys3cmd mb s3://$bkt + +for f in $(seq 8) ; do + dest_obj="reshard-obj-${f}" + mys3cmd put -q $big_obj s3://${bkt}/$dest_obj +done + +radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it +radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it + +#################################### +# versioned bucket + +if true ;then + echo "WARNING: versioned bucket test currently turned off" +else + bkt=versioned-bkt-1 + + mys3cmd mb s3://$bkt + + # bucket-enable-versioning $bkt + + for f in $(seq 3) ;do + for g in $(seq 10) ;do + dest_obj="versioned-obj-${g}" + mys3cmd put -q $big_obj s3://${bkt}/$dest_obj + done + done + + for g in $(seq 1 2 10) ;do + dest_obj="versioned-obj-${g}" + mys3cmd rm s3://${bkt}/$dest_obj + done +fi + +############################################################ +# copy small objects + +o_bkt="orig-bkt-1" +d_bkt="copy-bkt-1" +mys3cmd mb s3://$o_bkt + +for f in $(seq 4) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1 +mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3 + +for f in $(seq 5 6) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj +done + +############################################################ +# copy small objects and delete original + +o_bkt="orig-bkt-2" +d_bkt="copy-bkt-2" + +mys3cmd mb s3://$o_bkt + +for f in $(seq 4) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1 +mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3 + +for f in $(seq 5 6) ;do + dest_obj="orig-obj-$f" + mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj +done + +mys3cmd rb --recursive s3://${o_bkt} + +############################################################ +# copy multipart objects + +o_bkt="orig-mp-bkt-3" +d_bkt="copy-mp-bkt-3" + +mys3cmd mb s3://$o_bkt + +for f in $(seq 2) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \ + s3://${d_bkt}/copied-multipart-obj-1 + +for f in $(seq 5 5) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj +done + + +############################################################ +# copy multipart objects and delete original + +o_bkt="orig-mp-bkt-4" +d_bkt="copy-mp-bkt-4" + +mys3cmd mb s3://$o_bkt + +for f in $(seq 2) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj +done + +mys3cmd mb s3://$d_bkt + +mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \ + s3://${d_bkt}/copied-multipart-obj-1 + +for f in $(seq 5 5) ;do + dest_obj="orig-multipart-obj-$f" + mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj +done + +mys3cmd rb --recursive s3://$o_bkt + +######################################################################## +# SWIFT TESTS + +# 600MB +segment_size=629145600 + +############################################################ +# plain test + +for f in $(seq 4) ;do + myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f +done + +############################################################ +# zero-len test + +myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/ +myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1 +myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1 +myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1 + +############################################################ +# dlo test + +# upload in 300MB segments +myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \ + -S $segment_size + +############################################################ +# slo test + +# upload in 300MB segments +myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \ + -S $segment_size --use-slo + +############################################################ +# large object copy test + +# upload in 300MB segments +o_ctr=swift-orig-ctr +o_obj=slo-orig-obj-1 +d_ctr=swift-copy-ctr +d_obj=slo-copy-obj-1 +myswift upload $o_ctr $big_obj --object-name $o_obj + +myswift copy --destination /${d_ctr}/${d_obj} \ + $o_ctr $o_obj + +myswift delete $o_ctr $o_obj + +############################################################ +# huge dlo object copy test + +o_ctr=swift-orig-dlo-ctr-1 +o_obj=dlo-orig-dlo-obj-1 +d_ctr=swift-copy-dlo-ctr-1 +d_obj=dlo-copy-dlo-obj-1 + +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size + +myswift copy --destination /${d_ctr}/${d_obj} \ + $o_ctr $o_obj + +############################################################ +# huge dlo object copy and orig delete + +o_ctr=swift-orig-dlo-ctr-2 +o_obj=dlo-orig-dlo-obj-2 +d_ctr=swift-copy-dlo-ctr-2 +d_obj=dlo-copy-dlo-obj-2 + +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size + +myswift copy --destination /${d_ctr}/${d_obj} \ + $o_ctr $o_obj + +myswift delete $o_ctr $o_obj + +############################################################ +# huge slo object copy test + +o_ctr=swift-orig-slo-ctr-1 +o_obj=slo-orig-slo-obj-1 +d_ctr=swift-copy-slo-ctr-1 +d_obj=slo-copy-slo-obj-1 +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size --use-slo + +myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj + +############################################################ +# huge slo object copy test and orig delete + +o_ctr=swift-orig-slo-ctr-2 +o_obj=slo-orig-slo-obj-2 +d_ctr=swift-copy-slo-ctr-2 +d_obj=slo-copy-slo-obj-2 +myswift upload $o_ctr $huge_obj --object-name $o_obj \ + -S $segment_size --use-slo + +myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj + +myswift delete $o_ctr $o_obj + +######################################################################## +# FORCE GARBAGE COLLECTION + +sleep 6 # since for testing age at which gc can happen is 5 secs +radosgw-admin gc process --include-all + + +######################################## +# DO ORPHAN LIST + +pool="default.rgw.buckets.data" + +rgw-orphan-list $pool + +# we only expect there to be one output file, but loop just in case +ol_error="" +for f in orphan-list-*.out ; do + if [ -s "$f" ] ;then # if file non-empty + ol_error="${ol_error}:$f" + echo "One ore more orphans found in $f:" + cat "$f" + fi +done + +if [ -n "$ol_error" ] ;then + echo "ERROR: orphans found when none expected" + exit 1 +fi + +######################################################################## +# CLEAN UP + +rm -f $empty_obj $big_obj $huge_obj $s3config + +success |