Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

208 files changed, 34737 insertions, 0 deletions

diff --git a/qa/workunits/Makefile b/qa/workunits/Makefile
new file mode 100644
index 00000000..f75f5dfd
--- /dev/null
+++ b/qa/workunits/Makefile
@@ -0,0 +1,4 @@
+DIRS = direct_io fs
+
+all:
+	for d in $(DIRS) ; do ( cd $$d ; $(MAKE) all ) ; done
diff --git a/qa/workunits/caps/mon_commands.sh b/qa/workunits/caps/mon_commands.sh
new file mode 100755
index 00000000..5b5bce62
--- /dev/null
+++ b/qa/workunits/caps/mon_commands.sh
@@ -0,0 +1,25 @@
+#!/bin/sh -ex
+
+ceph-authtool --create-keyring k --gen-key -p --name client.xx
+ceph auth add -i k client.xx mon "allow command foo; allow command bar *; allow command baz ...; allow command foo add * mon allow\\ rwx osd allow\\ *"
+
+( ceph -k k -n client.xx foo      || true ) | grep 'unrecog'
+( ceph -k k -n client.xx foo ooo  || true ) | grep 'Access denied'
+( ceph -k k -n client.xx fo       || true ) | grep 'Access denied'
+( ceph -k k -n client.xx fooo     || true ) | grep 'Access denied'
+
+( ceph -k k -n client.xx bar       || true ) | grep 'Access denied'
+( ceph -k k -n client.xx bar a     || true ) | grep 'unrecog'
+( ceph -k k -n client.xx bar a b c || true ) | grep 'Access denied'
+( ceph -k k -n client.xx ba        || true ) | grep 'Access denied'
+( ceph -k k -n client.xx barr      || true ) | grep 'Access denied'
+
+( ceph -k k -n client.xx baz     || true ) | grep -v 'Access denied'
+( ceph -k k -n client.xx baz a   || true ) | grep -v 'Access denied'
+( ceph -k k -n client.xx baz a b || true ) | grep -v 'Access denied'
+
+( ceph -k k -n client.xx foo add osd.1 -i k mon 'allow rwx' osd 'allow *' || true ) | grep 'unrecog'
+( ceph -k k -n client.xx foo add osd a b c -i k mon 'allow rwx' osd 'allow *' || true ) | grep 'Access denied'
+( ceph -k k -n client.xx foo add osd a b c -i k mon 'allow *' || true ) | grep 'Access denied'
+
+echo OK
\ No newline at end of file
diff --git a/qa/workunits/ceph-helpers-root.sh b/qa/workunits/ceph-helpers-root.sh
new file mode 100755
index 00000000..dc81b2b3
--- /dev/null
+++ b/qa/workunits/ceph-helpers-root.sh
@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2015 Red Hat <contact@redhat.com>
+#
+# Author: Loic Dachary <loic@dachary.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library Public License for more details.
+#
+
+#######################################################################
+
+function distro_id() {
+    source /etc/os-release
+    echo $ID
+}
+
+function distro_version() {
+    source /etc/os-release
+    echo $VERSION
+}
+
+function install() {
+    for package in "$@" ; do
+        install_one $package
+    done
+}
+
+function install_one() {
+    case $(distro_id) in
+        ubuntu|debian|devuan)
+            sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y "$@"
+            ;;
+        centos|fedora|rhel)
+            sudo yum install -y "$@"
+            ;;
+        opensuse*|suse|sles)
+            sudo zypper --non-interactive install "$@"
+            ;;
+        *)
+            echo "$(distro_id) is unknown, $@ will have to be installed manually."
+            ;;
+    esac
+}
+
+function install_cmake3_on_centos7 {
+    source /etc/os-release
+    local MAJOR_VERSION="$(echo $VERSION_ID | cut -d. -f1)"
+    sudo yum-config-manager --add-repo https://dl.fedoraproject.org/pub/epel/$MAJOR_VERSION/x86_64/
+    sudo yum install --nogpgcheck -y epel-release
+    sudo rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-$MAJOR_VERSION
+    sudo yum install -y cmake3
+}
+
+function install_pkg_on_ubuntu {
+    local project=$1
+    shift
+    local sha1=$1
+    shift
+    local codename=$1
+    shift
+    local force=$1
+    shift
+    local pkgs=$@
+    local missing_pkgs
+    if [ $force = "force" ]; then
+	missing_pkgs="$@"
+    else
+	for pkg in $pkgs; do
+	    if ! dpkg -s $pkg &> /dev/null; then
+		missing_pkgs+=" $pkg"
+	    fi
+	done
+    fi
+    if test -n "$missing_pkgs"; then
+	local shaman_url="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo"
+	sudo curl --silent --location $shaman_url --output /etc/apt/sources.list.d/$project.list
+	sudo env DEBIAN_FRONTEND=noninteractive apt-get update -y -o Acquire::Languages=none -o Acquire::Translation=none || true
+	sudo env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y $missing_pkgs
+    fi
+}
+
+#######################################################################
+
+function control_osd() {
+    local action=$1
+    local id=$2
+
+    sudo systemctl $action ceph-osd@$id
+
+    return 0
+}
+
+#######################################################################
+
+function pool_read_write() {
+    local size=${1:-1}
+    local dir=/tmp
+    local timeout=360
+    local test_pool=test_pool
+
+    ceph osd pool delete $test_pool $test_pool --yes-i-really-really-mean-it || return 1
+    ceph osd pool create $test_pool 4 || return 1
+    ceph osd pool set $test_pool size $size || return 1
+    ceph osd pool set $test_pool min_size $size || return 1
+    ceph osd pool application enable $test_pool rados
+
+    echo FOO > $dir/BAR
+    timeout $timeout rados --pool $test_pool put BAR $dir/BAR || return 1
+    timeout $timeout rados --pool $test_pool get BAR $dir/BAR.copy || return 1
+    diff $dir/BAR $dir/BAR.copy || return 1
+    ceph osd pool delete $test_pool $test_pool --yes-i-really-really-mean-it || return 1
+}
+
+#######################################################################
+
+set -x
+
+"$@"
diff --git a/qa/workunits/ceph-tests/ceph-admin-commands.sh b/qa/workunits/ceph-tests/ceph-admin-commands.sh
new file mode 100755
index 00000000..4a9f0a66
--- /dev/null
+++ b/qa/workunits/ceph-tests/ceph-admin-commands.sh
@@ -0,0 +1,10 @@
+#!/bin/sh -ex
+
+ceph -s
+rados lspools
+rbd ls
+# check that the monitors work
+ceph osd set nodown
+ceph osd unset nodown
+
+exit 0
diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
new file mode 100755
index 00000000..1929d296
--- /dev/null
+++ b/qa/workunits/cephtool/test.sh
@@ -0,0 +1,2920 @@
+#!/usr/bin/env bash
+# -*- mode:shell-script; tab-width:8; sh-basic-offset:2; indent-tabs-mode:t -*-
+# vim: ts=8 sw=8 ft=bash smarttab
+set -x
+
+source $(dirname $0)/../../standalone/ceph-helpers.sh
+
+set -e
+set -o functrace
+PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
+SUDO=${SUDO:-sudo}
+export CEPH_DEV=1
+
+function get_admin_socket()
+{
+  local client=$1
+
+  if test -n "$CEPH_ASOK_DIR";
+  then
+    echo $(get_asok_dir)/$client.asok
+  else
+    local cluster=$(echo $CEPH_ARGS | sed  -r 's/.*--cluster[[:blank:]]*([[:alnum:]]*).*/\1/')
+    echo "/var/run/ceph/$cluster-$client.asok"
+  fi
+}
+
+function check_no_osd_down()
+{
+    ! ceph osd dump | grep ' down '
+}
+
+function wait_no_osd_down()
+{
+  max_run=300
+  for i in $(seq 1 $max_run) ; do
+    if ! check_no_osd_down ; then
+      echo "waiting for osd(s) to come back up ($i/$max_run)"
+      sleep 1
+    else
+      break
+    fi
+  done
+  check_no_osd_down
+}
+
+function expect_false()
+{
+	set -x
+	if "$@"; then return 1; else return 0; fi
+}
+
+function expect_true()
+{
+	set -x
+	if ! "$@"; then return 1; else return 0; fi
+}
+
+TEMP_DIR=$(mktemp -d ${TMPDIR-/tmp}/cephtool.XXX)
+trap "rm -fr $TEMP_DIR" 0
+
+TMPFILE=$(mktemp $TEMP_DIR/test_invalid.XXX)
+
+#
+# retry_eagain max cmd args ...
+#
+# retry cmd args ... if it exits on error and its output contains the
+# string EAGAIN, at most $max times
+#
+function retry_eagain()
+{
+    local max=$1
+    shift
+    local status
+    local tmpfile=$TEMP_DIR/retry_eagain.$$
+    local count
+    for count in $(seq 1 $max) ; do
+        status=0
+        "$@" > $tmpfile 2>&1 || status=$?
+        if test $status = 0 || 
+            ! grep --quiet EAGAIN $tmpfile ; then
+            break
+        fi
+        sleep 1
+    done
+    if test $count = $max ; then
+        echo retried with non zero exit status, $max times: "$@" >&2
+    fi
+    cat $tmpfile
+    rm $tmpfile
+    return $status
+}
+
+#
+# map_enxio_to_eagain cmd arg ...
+#
+# add EAGAIN to the output of cmd arg ... if the output contains
+# ENXIO.
+#
+function map_enxio_to_eagain()
+{
+    local status=0
+    local tmpfile=$TEMP_DIR/map_enxio_to_eagain.$$
+
+    "$@" > $tmpfile 2>&1 || status=$?
+    if test $status != 0 &&
+        grep --quiet ENXIO $tmpfile ; then
+        echo "EAGAIN added by $0::map_enxio_to_eagain" >> $tmpfile
+    fi
+    cat $tmpfile
+    rm $tmpfile
+    return $status
+}
+
+function check_response()
+{
+	expected_string=$1
+	retcode=$2
+	expected_retcode=$3
+	if [ "$expected_retcode" -a $retcode != $expected_retcode ] ; then
+		echo "return code invalid: got $retcode, expected $expected_retcode" >&2
+		exit 1
+	fi
+
+	if ! grep --quiet -- "$expected_string" $TMPFILE ; then 
+		echo "Didn't find $expected_string in output" >&2
+		cat $TMPFILE >&2
+		exit 1
+	fi
+}
+
+function get_config_value_or_die()
+{
+  local target config_opt raw val
+
+  target=$1
+  config_opt=$2
+
+  raw="`$SUDO ceph daemon $target config get $config_opt 2>/dev/null`"
+  if [[ $? -ne 0 ]]; then
+    echo "error obtaining config opt '$config_opt' from '$target': $raw"
+    exit 1
+  fi
+
+  raw=`echo $raw | sed -e 's/[{} "]//g'`
+  val=`echo $raw | cut -f2 -d:`
+
+  echo "$val"
+  return 0
+}
+
+function expect_config_value()
+{
+  local target config_opt expected_val val
+  target=$1
+  config_opt=$2
+  expected_val=$3
+
+  val=$(get_config_value_or_die $target $config_opt)
+
+  if [[ "$val" != "$expected_val" ]]; then
+    echo "expected '$expected_val', got '$val'"
+    exit 1
+  fi
+}
+
+function ceph_watch_start()
+{
+    local whatch_opt=--watch
+
+    if [ -n "$1" ]; then
+	whatch_opt=--watch-$1
+	if [ -n "$2" ]; then
+	    whatch_opt+=" --watch-channel $2"
+	fi
+    fi
+
+    CEPH_WATCH_FILE=${TEMP_DIR}/CEPH_WATCH_$$
+    ceph $whatch_opt > $CEPH_WATCH_FILE &
+    CEPH_WATCH_PID=$!
+
+    # wait until the "ceph" client is connected and receiving
+    # log messages from monitor
+    for i in `seq 3`; do
+        grep -q "cluster" $CEPH_WATCH_FILE && break
+        sleep 1
+    done
+}
+
+function ceph_watch_wait()
+{
+    local regexp=$1
+    local timeout=30
+
+    if [ -n "$2" ]; then
+	timeout=$2
+    fi
+
+    for i in `seq ${timeout}`; do
+	grep -q "$regexp" $CEPH_WATCH_FILE && break
+	sleep 1
+    done
+
+    kill $CEPH_WATCH_PID
+
+    if ! grep "$regexp" $CEPH_WATCH_FILE; then
+	echo "pattern ${regexp} not found in watch file. Full watch file content:" >&2
+	cat $CEPH_WATCH_FILE >&2
+	return 1
+    fi
+}
+
+function test_mon_injectargs()
+{
+  ceph tell osd.0 injectargs --no-osd_enable_op_tracker
+  ceph tell osd.0 config get osd_enable_op_tracker | grep false
+  ceph tell osd.0 injectargs '--osd_enable_op_tracker --osd_op_history_duration 500'
+  ceph tell osd.0 config get osd_enable_op_tracker | grep true
+  ceph tell osd.0 config get osd_op_history_duration | grep 500
+  ceph tell osd.0 injectargs --no-osd_enable_op_tracker
+  ceph tell osd.0 config get osd_enable_op_tracker | grep false
+  ceph tell osd.0 injectargs -- --osd_enable_op_tracker
+  ceph tell osd.0 config get osd_enable_op_tracker | grep true
+  ceph tell osd.0 injectargs -- '--osd_enable_op_tracker --osd_op_history_duration 600'
+  ceph tell osd.0 config get osd_enable_op_tracker | grep true
+  ceph tell osd.0 config get osd_op_history_duration | grep 600
+
+  ceph tell osd.0 injectargs -- '--osd_deep_scrub_interval 2419200'
+  ceph tell osd.0 config get osd_deep_scrub_interval | grep 2419200
+
+  ceph tell osd.0 injectargs -- '--mon_probe_timeout 2'
+  ceph tell osd.0 config get mon_probe_timeout | grep 2
+
+  ceph tell osd.0 injectargs -- '--mon-lease 6'
+  ceph tell osd.0 config get mon_lease | grep 6
+
+  # osd-scrub-auto-repair-num-errors is an OPT_U32, so -1 is not a valid setting
+  expect_false ceph tell osd.0 injectargs --osd-scrub-auto-repair-num-errors -1 >& $TMPFILE || return 1
+  check_response "Error EINVAL: Parse error setting osd_scrub_auto_repair_num_errors to '-1' using injectargs"
+
+  expect_failure $TEMP_DIR "Option --osd_op_history_duration requires an argument" \
+                 ceph tell osd.0 injectargs -- '--osd_op_history_duration'
+
+}
+
+function test_mon_injectargs_SI()
+{
+  # Test SI units during injectargs and 'config set'
+  # We only aim at testing the units are parsed accordingly
+  # and don't intend to test whether the options being set
+  # actually expect SI units to be passed.
+  # Keep in mind that all integer based options that are not based on bytes
+  # (i.e., INT, LONG, U32, U64) will accept SI unit modifiers and be parsed to
+  # base 10.
+  initial_value=$(get_config_value_or_die "mon.a" "mon_pg_warn_min_objects")
+  $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10
+  expect_config_value "mon.a" "mon_pg_warn_min_objects" 10
+  $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10K
+  expect_config_value "mon.a" "mon_pg_warn_min_objects" 10000
+  $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 1G
+  expect_config_value "mon.a" "mon_pg_warn_min_objects" 1000000000
+  $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects 10F > $TMPFILE || true
+  check_response "'10F': (22) Invalid argument"
+  # now test with injectargs
+  ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10'
+  expect_config_value "mon.a" "mon_pg_warn_min_objects" 10
+  ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10K'
+  expect_config_value "mon.a" "mon_pg_warn_min_objects" 10000
+  ceph tell mon.a injectargs '--mon_pg_warn_min_objects 1G'
+  expect_config_value "mon.a" "mon_pg_warn_min_objects" 1000000000
+  expect_false ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10F'
+  expect_false ceph tell mon.a injectargs '--mon_globalid_prealloc -1'
+  $SUDO ceph daemon mon.a config set mon_pg_warn_min_objects $initial_value
+}
+
+function test_mon_injectargs_IEC()
+{
+  # Test IEC units during injectargs and 'config set'
+  # We only aim at testing the units are parsed accordingly
+  # and don't intend to test whether the options being set
+  # actually expect IEC units to be passed.
+  # Keep in mind that all integer based options that are based on bytes
+  # (i.e., INT, LONG, U32, U64) will accept IEC unit modifiers, as well as SI
+  # unit modifiers (for backwards compatibility and convenience) and be parsed
+  # to base 2.
+  initial_value=$(get_config_value_or_die "mon.a" "mon_data_size_warn")
+  $SUDO ceph daemon mon.a config set mon_data_size_warn 15000000000
+  expect_config_value "mon.a" "mon_data_size_warn" 15000000000
+  $SUDO ceph daemon mon.a config set mon_data_size_warn 15G
+  expect_config_value "mon.a" "mon_data_size_warn" 16106127360
+  $SUDO ceph daemon mon.a config set mon_data_size_warn 16Gi
+  expect_config_value "mon.a" "mon_data_size_warn" 17179869184
+  $SUDO ceph daemon mon.a config set mon_data_size_warn 10F > $TMPFILE || true
+  check_response "'10F': (22) Invalid argument"
+  # now test with injectargs
+  ceph tell mon.a injectargs '--mon_data_size_warn 15000000000'
+  expect_config_value "mon.a" "mon_data_size_warn" 15000000000
+  ceph tell mon.a injectargs '--mon_data_size_warn 15G'
+  expect_config_value "mon.a" "mon_data_size_warn" 16106127360
+  ceph tell mon.a injectargs '--mon_data_size_warn 16Gi'
+  expect_config_value "mon.a" "mon_data_size_warn" 17179869184
+  expect_false ceph tell mon.a injectargs '--mon_data_size_warn 10F'
+  $SUDO ceph daemon mon.a config set mon_data_size_warn $initial_value
+}
+
+function test_tiering_agent()
+{
+  local slow=slow_eviction
+  local fast=fast_eviction
+  ceph osd pool create $slow  1 1
+  ceph osd pool application enable $slow rados
+  ceph osd pool create $fast  1 1
+  ceph osd tier add $slow $fast
+  ceph osd tier cache-mode $fast writeback
+  ceph osd tier set-overlay $slow $fast
+  ceph osd pool set $fast hit_set_type bloom
+  rados -p $slow put obj1 /etc/group
+  ceph osd pool set $fast target_max_objects  1
+  ceph osd pool set $fast hit_set_count 1
+  ceph osd pool set $fast hit_set_period 5
+  # wait for the object to be evicted from the cache
+  local evicted
+  evicted=false
+  for i in `seq 1 300` ; do
+      if ! rados -p $fast ls | grep obj1 ; then
+          evicted=true
+          break
+      fi
+      sleep 1
+  done
+  $evicted # assert
+  # the object is proxy read and promoted to the cache
+  rados -p $slow get obj1 - >/dev/null
+  # wait for the promoted object to be evicted again
+  evicted=false
+  for i in `seq 1 300` ; do
+      if ! rados -p $fast ls | grep obj1 ; then
+          evicted=true
+          break
+      fi
+      sleep 1
+  done
+  $evicted # assert
+  ceph osd tier remove-overlay $slow
+  ceph osd tier remove $slow $fast
+  ceph osd pool delete $fast $fast --yes-i-really-really-mean-it
+  ceph osd pool delete $slow $slow --yes-i-really-really-mean-it
+}
+
+function test_tiering_1()
+{
+  # tiering
+  ceph osd pool create slow 2
+  ceph osd pool application enable slow rados
+  ceph osd pool create slow2 2
+  ceph osd pool application enable slow2 rados
+  ceph osd pool create cache 2
+  ceph osd pool create cache2 2
+  ceph osd tier add slow cache
+  ceph osd tier add slow cache2
+  expect_false ceph osd tier add slow2 cache
+  # test some state transitions
+  ceph osd tier cache-mode cache writeback
+  expect_false ceph osd tier cache-mode cache forward
+  ceph osd tier cache-mode cache forward --yes-i-really-mean-it
+  expect_false ceph osd tier cache-mode cache readonly
+  ceph osd tier cache-mode cache readonly --yes-i-really-mean-it
+  expect_false ceph osd tier cache-mode cache forward
+  ceph osd tier cache-mode cache forward --yes-i-really-mean-it
+  ceph osd tier cache-mode cache none
+  ceph osd tier cache-mode cache writeback
+  ceph osd tier cache-mode cache proxy
+  ceph osd tier cache-mode cache writeback
+  expect_false ceph osd tier cache-mode cache none
+  expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it
+  # test with dirty objects in the tier pool
+  # tier pool currently set to 'writeback'
+  rados -p cache put /etc/passwd /etc/passwd
+  flush_pg_stats
+  # 1 dirty object in pool 'cache'
+  ceph osd tier cache-mode cache proxy
+  expect_false ceph osd tier cache-mode cache none
+  expect_false ceph osd tier cache-mode cache readonly --yes-i-really-mean-it
+  ceph osd tier cache-mode cache writeback
+  # remove object from tier pool
+  rados -p cache rm /etc/passwd
+  rados -p cache cache-flush-evict-all
+  flush_pg_stats
+  # no dirty objects in pool 'cache'
+  ceph osd tier cache-mode cache proxy
+  ceph osd tier cache-mode cache none
+  ceph osd tier cache-mode cache readonly --yes-i-really-mean-it
+  TRIES=0
+  while ! ceph osd pool set cache pg_num 3 --yes-i-really-mean-it 2>$TMPFILE
+  do
+    grep 'currently creating pgs' $TMPFILE
+    TRIES=$(( $TRIES + 1 ))
+    test $TRIES -ne 60
+    sleep 3
+  done
+  expect_false ceph osd pool set cache pg_num 4
+  ceph osd tier cache-mode cache none
+  ceph osd tier set-overlay slow cache
+  expect_false ceph osd tier set-overlay slow cache2
+  expect_false ceph osd tier remove slow cache
+  ceph osd tier remove-overlay slow
+  ceph osd tier set-overlay slow cache2
+  ceph osd tier remove-overlay slow
+  ceph osd tier remove slow cache
+  ceph osd tier add slow2 cache
+  expect_false ceph osd tier set-overlay slow cache
+  ceph osd tier set-overlay slow2 cache
+  ceph osd tier remove-overlay slow2
+  ceph osd tier remove slow2 cache
+  ceph osd tier remove slow cache2
+
+  # make sure a non-empty pool fails
+  rados -p cache2 put /etc/passwd /etc/passwd
+  while ! ceph df | grep cache2 | grep ' 1 ' ; do
+    echo waiting for pg stats to flush
+    sleep 2
+  done
+  expect_false ceph osd tier add slow cache2
+  ceph osd tier add slow cache2 --force-nonempty
+  ceph osd tier remove slow cache2
+
+  ceph osd pool ls | grep cache2
+  ceph osd pool ls -f json-pretty | grep cache2
+  ceph osd pool ls detail | grep cache2
+  ceph osd pool ls detail -f json-pretty | grep cache2
+
+  ceph osd pool delete slow slow --yes-i-really-really-mean-it
+  ceph osd pool delete slow2 slow2 --yes-i-really-really-mean-it
+  ceph osd pool delete cache cache --yes-i-really-really-mean-it
+  ceph osd pool delete cache2 cache2 --yes-i-really-really-mean-it
+}
+
+function test_tiering_2()
+{
+  # make sure we can't clobber snapshot state
+  ceph osd pool create snap_base 2
+  ceph osd pool application enable snap_base rados
+  ceph osd pool create snap_cache 2
+  ceph osd pool mksnap snap_cache snapname
+  expect_false ceph osd tier add snap_base snap_cache
+  ceph osd pool delete snap_base snap_base --yes-i-really-really-mean-it
+  ceph osd pool delete snap_cache snap_cache --yes-i-really-really-mean-it
+}
+
+function test_tiering_3()
+{
+  # make sure we can't create snapshot on tier
+  ceph osd pool create basex 2
+  ceph osd pool application enable basex rados
+  ceph osd pool create cachex 2
+  ceph osd tier add basex cachex
+  expect_false ceph osd pool mksnap cache snapname
+  ceph osd tier remove basex cachex
+  ceph osd pool delete basex basex --yes-i-really-really-mean-it
+  ceph osd pool delete cachex cachex --yes-i-really-really-mean-it
+}
+
+function test_tiering_4()
+{
+  # make sure we can't create an ec pool tier
+  ceph osd pool create eccache 2 2 erasure
+  expect_false ceph osd set-require-min-compat-client bobtail
+  ceph osd pool create repbase 2
+  ceph osd pool application enable repbase rados
+  expect_false ceph osd tier add repbase eccache
+  ceph osd pool delete repbase repbase --yes-i-really-really-mean-it
+  ceph osd pool delete eccache eccache --yes-i-really-really-mean-it
+}
+
+function test_tiering_5()
+{
+  # convenient add-cache command
+  ceph osd pool create slow 2
+  ceph osd pool application enable slow rados
+  ceph osd pool create cache3 2
+  ceph osd tier add-cache slow cache3 1024000
+  ceph osd dump | grep cache3 | grep bloom | grep 'false_positive_probability: 0.05' | grep 'target_bytes 1024000' | grep '1200s x4'
+  ceph osd tier remove slow cache3 2> $TMPFILE || true
+  check_response "EBUSY: tier pool 'cache3' is the overlay for 'slow'; please remove-overlay first"
+  ceph osd tier remove-overlay slow
+  ceph osd tier remove slow cache3
+  ceph osd pool ls | grep cache3
+  ceph osd pool delete cache3 cache3 --yes-i-really-really-mean-it
+  ! ceph osd pool ls | grep cache3 || exit 1
+  ceph osd pool delete slow slow --yes-i-really-really-mean-it
+}
+
+function test_tiering_6()
+{
+  # check add-cache whether work
+  ceph osd pool create datapool 2
+  ceph osd pool application enable datapool rados
+  ceph osd pool create cachepool 2
+  ceph osd tier add-cache datapool cachepool 1024000
+  ceph osd tier cache-mode cachepool writeback
+  rados -p datapool put object /etc/passwd
+  rados -p cachepool stat object
+  rados -p cachepool cache-flush object
+  rados -p datapool stat object
+  ceph osd tier remove-overlay datapool
+  ceph osd tier remove datapool cachepool
+  ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it
+  ceph osd pool delete datapool datapool --yes-i-really-really-mean-it
+}
+
+function test_tiering_7()
+{
+  # protection against pool removal when used as tiers
+  ceph osd pool create datapool 2
+  ceph osd pool application enable datapool rados
+  ceph osd pool create cachepool 2
+  ceph osd tier add-cache datapool cachepool 1024000
+  ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it 2> $TMPFILE || true
+  check_response "EBUSY: pool 'cachepool' is a tier of 'datapool'"
+  ceph osd pool delete datapool datapool --yes-i-really-really-mean-it 2> $TMPFILE || true
+  check_response "EBUSY: pool 'datapool' has tiers cachepool"
+  ceph osd tier remove-overlay datapool
+  ceph osd tier remove datapool cachepool
+  ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it
+  ceph osd pool delete datapool datapool --yes-i-really-really-mean-it
+}
+
+function test_tiering_8()
+{
+  ## check health check
+  ceph osd set notieragent
+  ceph osd pool create datapool 2
+  ceph osd pool application enable datapool rados
+  ceph osd pool create cache4 2
+  ceph osd tier add-cache datapool cache4 1024000
+  ceph osd tier cache-mode cache4 writeback
+  tmpfile=$(mktemp|grep tmp)
+  dd if=/dev/zero of=$tmpfile  bs=4K count=1
+  ceph osd pool set cache4 target_max_objects 200
+  ceph osd pool set cache4 target_max_bytes 1000000
+  rados -p cache4 put foo1 $tmpfile
+  rados -p cache4 put foo2 $tmpfile
+  rm -f $tmpfile
+  flush_pg_stats
+  ceph df | grep datapool | grep ' 2 '
+  ceph osd tier remove-overlay datapool
+  ceph osd tier remove datapool cache4
+  ceph osd pool delete cache4 cache4 --yes-i-really-really-mean-it
+  ceph osd pool delete datapool datapool --yes-i-really-really-mean-it
+  ceph osd unset notieragent
+}
+
+function test_tiering_9()
+{
+  # make sure 'tier remove' behaves as we expect
+  # i.e., removing a tier from a pool that's not its base pool only
+  # results in a 'pool foo is now (or already was) not a tier of bar'
+  #
+  ceph osd pool create basepoolA 2
+  ceph osd pool application enable basepoolA rados
+  ceph osd pool create basepoolB 2
+  ceph osd pool application enable basepoolB rados
+  poolA_id=$(ceph osd dump | grep 'pool.*basepoolA' | awk '{print $2;}')
+  poolB_id=$(ceph osd dump | grep 'pool.*basepoolB' | awk '{print $2;}')
+
+  ceph osd pool create cache5 2
+  ceph osd pool create cache6 2
+  ceph osd tier add basepoolA cache5
+  ceph osd tier add basepoolB cache6
+  ceph osd tier remove basepoolB cache5 2>&1 | grep 'not a tier of'
+  ceph osd dump | grep "pool.*'cache5'" 2>&1 | grep "tier_of[ \t]\+$poolA_id"
+  ceph osd tier remove basepoolA cache6 2>&1 | grep 'not a tier of'
+  ceph osd dump | grep "pool.*'cache6'" 2>&1 | grep "tier_of[ \t]\+$poolB_id"
+
+  ceph osd tier remove basepoolA cache5 2>&1 | grep 'not a tier of'
+  ! ceph osd dump | grep "pool.*'cache5'" 2>&1 | grep "tier_of" || exit 1
+  ceph osd tier remove basepoolB cache6 2>&1 | grep 'not a tier of'
+  ! ceph osd dump | grep "pool.*'cache6'" 2>&1 | grep "tier_of" || exit 1
+
+  ! ceph osd dump | grep "pool.*'basepoolA'" 2>&1 | grep "tiers" || exit 1
+  ! ceph osd dump | grep "pool.*'basepoolB'" 2>&1 | grep "tiers" || exit 1
+
+  ceph osd pool delete cache6 cache6 --yes-i-really-really-mean-it
+  ceph osd pool delete cache5 cache5 --yes-i-really-really-mean-it
+  ceph osd pool delete basepoolB basepoolB --yes-i-really-really-mean-it
+  ceph osd pool delete basepoolA basepoolA --yes-i-really-really-mean-it
+}
+
+function test_auth()
+{
+  expect_false ceph auth add client.xx mon 'invalid' osd "allow *"
+  expect_false ceph auth add client.xx mon 'allow *' osd "allow *" invalid "allow *"
+  ceph auth add client.xx mon 'allow *' osd "allow *"
+  ceph auth export client.xx >client.xx.keyring
+  ceph auth add client.xx -i client.xx.keyring
+  rm -f client.xx.keyring
+  ceph auth list | grep client.xx
+  ceph auth ls | grep client.xx
+  ceph auth get client.xx | grep caps | grep mon
+  ceph auth get client.xx | grep caps | grep osd
+  ceph auth get-key client.xx
+  ceph auth print-key client.xx
+  ceph auth print_key client.xx
+  ceph auth caps client.xx osd "allow rw"
+  expect_false sh <<< "ceph auth get client.xx | grep caps | grep mon"
+  ceph auth get client.xx | grep osd | grep "allow rw"
+  ceph auth export | grep client.xx
+  ceph auth export -o authfile
+  ceph auth import -i authfile
+  ceph auth export -o authfile2
+  diff authfile authfile2
+  rm authfile authfile2
+  ceph auth del client.xx
+  expect_false ceph auth get client.xx
+
+  # (almost) interactive mode
+  echo -e 'auth add client.xx mon "allow *" osd "allow *"\n' | ceph
+  ceph auth get client.xx
+  # script mode
+  echo 'auth del client.xx' | ceph
+  expect_false ceph auth get client.xx
+}
+
+function test_auth_profiles()
+{
+  ceph auth add client.xx-profile-ro mon 'allow profile read-only' \
+       mgr 'allow profile read-only'
+  ceph auth add client.xx-profile-rw mon 'allow profile read-write' \
+       mgr 'allow profile read-write'
+  ceph auth add client.xx-profile-rd mon 'allow profile role-definer'
+
+  ceph auth export > client.xx.keyring
+
+  # read-only is allowed all read-only commands (auth excluded)
+  ceph -n client.xx-profile-ro -k client.xx.keyring status
+  ceph -n client.xx-profile-ro -k client.xx.keyring osd dump
+  ceph -n client.xx-profile-ro -k client.xx.keyring pg dump
+  ceph -n client.xx-profile-ro -k client.xx.keyring mon dump
+  # read-only gets access denied for rw commands or auth commands
+  ceph -n client.xx-profile-ro -k client.xx.keyring log foo >& $TMPFILE || true
+  check_response "EACCES: access denied"
+  ceph -n client.xx-profile-ro -k client.xx.keyring osd set noout >& $TMPFILE || true
+  check_response "EACCES: access denied"
+  ceph -n client.xx-profile-ro -k client.xx.keyring auth ls >& $TMPFILE || true
+  check_response "EACCES: access denied"
+
+  # read-write is allowed for all read-write commands (except auth)
+  ceph -n client.xx-profile-rw -k client.xx.keyring status
+  ceph -n client.xx-profile-rw -k client.xx.keyring osd dump
+  ceph -n client.xx-profile-rw -k client.xx.keyring pg dump
+  ceph -n client.xx-profile-rw -k client.xx.keyring mon dump
+  ceph -n client.xx-profile-rw -k client.xx.keyring fs dump
+  ceph -n client.xx-profile-rw -k client.xx.keyring log foo
+  ceph -n client.xx-profile-rw -k client.xx.keyring osd set noout
+  ceph -n client.xx-profile-rw -k client.xx.keyring osd unset noout
+  # read-write gets access denied for auth commands
+  ceph -n client.xx-profile-rw -k client.xx.keyring auth ls >& $TMPFILE || true
+  check_response "EACCES: access denied"
+
+  # role-definer is allowed RWX 'auth' commands and read-only 'mon' commands
+  ceph -n client.xx-profile-rd -k client.xx.keyring auth ls
+  ceph -n client.xx-profile-rd -k client.xx.keyring auth export
+  ceph -n client.xx-profile-rd -k client.xx.keyring auth add client.xx-profile-foo
+  ceph -n client.xx-profile-rd -k client.xx.keyring status
+  ceph -n client.xx-profile-rd -k client.xx.keyring osd dump >& $TMPFILE || true
+  check_response "EACCES: access denied"
+  ceph -n client.xx-profile-rd -k client.xx.keyring pg dump >& $TMPFILE || true
+  check_response "EACCES: access denied"
+  # read-only 'mon' subsystem commands are allowed
+  ceph -n client.xx-profile-rd -k client.xx.keyring mon dump
+  # but read-write 'mon' commands are not
+  ceph -n client.xx-profile-rd -k client.xx.keyring mon add foo 1.1.1.1 >& $TMPFILE || true
+  check_response "EACCES: access denied"
+  ceph -n client.xx-profile-rd -k client.xx.keyring fs dump >& $TMPFILE || true
+  check_response "EACCES: access denied"
+  ceph -n client.xx-profile-rd -k client.xx.keyring log foo >& $TMPFILE || true
+  check_response "EACCES: access denied"
+  ceph -n client.xx-profile-rd -k client.xx.keyring osd set noout >& $TMPFILE || true
+  check_response "EACCES: access denied"
+
+  ceph -n client.xx-profile-rd -k client.xx.keyring auth del client.xx-profile-ro
+  ceph -n client.xx-profile-rd -k client.xx.keyring auth del client.xx-profile-rw
+  
+  # add a new role-definer with the existing role-definer
+  ceph -n client.xx-profile-rd -k client.xx.keyring \
+    auth add client.xx-profile-rd2 mon 'allow profile role-definer'
+  ceph -n client.xx-profile-rd -k client.xx.keyring \
+    auth export > client.xx.keyring.2
+  # remove old role-definer using the new role-definer
+  ceph -n client.xx-profile-rd2 -k client.xx.keyring.2 \
+    auth del client.xx-profile-rd
+  # remove the remaining role-definer with admin
+  ceph auth del client.xx-profile-rd2
+  rm -f client.xx.keyring client.xx.keyring.2
+}
+
+function test_mon_caps()
+{
+  ceph-authtool --create-keyring $TEMP_DIR/ceph.client.bug.keyring
+  chmod +r  $TEMP_DIR/ceph.client.bug.keyring
+  ceph-authtool  $TEMP_DIR/ceph.client.bug.keyring -n client.bug --gen-key
+  ceph auth add client.bug -i  $TEMP_DIR/ceph.client.bug.keyring
+
+  # pass --no-mon-config since we are looking for the permission denied error
+  rados lspools --no-mon-config --keyring $TEMP_DIR/ceph.client.bug.keyring -n client.bug >& $TMPFILE || true
+  cat $TMPFILE
+  check_response "Permission denied"
+
+  rm -rf $TEMP_DIR/ceph.client.bug.keyring
+  ceph auth del client.bug
+  ceph-authtool --create-keyring $TEMP_DIR/ceph.client.bug.keyring
+  chmod +r  $TEMP_DIR/ceph.client.bug.keyring
+  ceph-authtool  $TEMP_DIR/ceph.client.bug.keyring -n client.bug --gen-key
+  ceph-authtool -n client.bug --cap mon '' $TEMP_DIR/ceph.client.bug.keyring
+  ceph auth add client.bug -i  $TEMP_DIR/ceph.client.bug.keyring
+  rados lspools --no-mon-config --keyring $TEMP_DIR/ceph.client.bug.keyring -n client.bug >& $TMPFILE || true
+  check_response "Permission denied"  
+}
+
+function test_mon_misc()
+{
+  # with and without verbosity
+  ceph osd dump | grep '^epoch'
+  ceph --concise osd dump | grep '^epoch'
+
+  ceph osd df | grep 'MIN/MAX VAR'
+  osd_class=$(ceph osd crush get-device-class 0)
+  ceph osd df tree class $osd_class | grep 'osd.0'
+  ceph osd crush rm-device-class 0
+  # create class first in case old device class may
+  # have already been automatically destroyed
+  ceph osd crush class create $osd_class
+  ceph osd df tree class $osd_class | expect_false grep 'osd.0'
+  ceph osd crush set-device-class $osd_class 0
+  ceph osd df tree name osd.0 | grep 'osd.0'
+
+  # df
+  ceph df > $TMPFILE
+  grep RAW $TMPFILE
+  grep -v DIRTY $TMPFILE
+  ceph df detail > $TMPFILE
+  grep DIRTY $TMPFILE
+  ceph df --format json > $TMPFILE
+  grep 'total_bytes' $TMPFILE
+  grep -v 'dirty' $TMPFILE
+  ceph df detail --format json > $TMPFILE
+  grep 'rd_bytes' $TMPFILE
+  grep 'dirty' $TMPFILE
+  ceph df --format xml | grep '<total_bytes>'
+  ceph df detail --format xml | grep '<rd_bytes>'
+
+  ceph fsid
+  ceph health
+  ceph health detail
+  ceph health --format json-pretty
+  ceph health detail --format xml-pretty
+
+  ceph time-sync-status
+
+  ceph node ls
+  for t in mon osd mds mgr ; do
+      ceph node ls $t
+  done
+
+  ceph_watch_start
+  mymsg="this is a test log message $$.$(date)"
+  ceph log "$mymsg"
+  ceph log last | grep "$mymsg"
+  ceph log last 100 | grep "$mymsg"
+  ceph_watch_wait "$mymsg"
+
+  ceph mgr dump
+  ceph mgr module ls
+  ceph mgr module enable restful
+  expect_false ceph mgr module enable foodne
+  ceph mgr module enable foodne --force
+  ceph mgr module disable foodne
+  ceph mgr module disable foodnebizbangbash
+
+  ceph mon metadata a
+  ceph mon metadata
+  ceph mon count-metadata ceph_version
+  ceph mon versions
+
+  ceph mgr metadata
+  ceph mgr versions
+  ceph mgr count-metadata ceph_version
+
+  ceph versions
+
+  ceph node ls
+}
+
+function check_mds_active()
+{
+    fs_name=$1
+    ceph fs get $fs_name | grep active
+}
+
+function wait_mds_active()
+{
+  fs_name=$1
+  max_run=300
+  for i in $(seq 1 $max_run) ; do
+      if ! check_mds_active $fs_name ; then
+          echo "waiting for an active MDS daemon ($i/$max_run)"
+          sleep 5
+      else
+          break
+      fi
+  done
+  check_mds_active $fs_name
+}
+
+function get_mds_gids()
+{
+    fs_name=$1
+    ceph fs get $fs_name --format=json | python -c "import json; import sys; print ' '.join([m['gid'].__str__() for m in json.load(sys.stdin)['mdsmap']['info'].values()])"
+}
+
+function fail_all_mds()
+{
+  fs_name=$1
+  ceph fs set $fs_name cluster_down true
+  mds_gids=$(get_mds_gids $fs_name)
+  for mds_gid in $mds_gids ; do
+      ceph mds fail $mds_gid
+  done
+  if check_mds_active $fs_name ; then
+      echo "An active MDS remains, something went wrong"
+      ceph fs get $fs_name
+      exit -1
+  fi
+
+}
+
+function remove_all_fs()
+{
+  existing_fs=$(ceph fs ls --format=json | python -c "import json; import sys; print ' '.join([fs['name'] for fs in json.load(sys.stdin)])")
+  for fs_name in $existing_fs ; do
+      echo "Removing fs ${fs_name}..."
+      fail_all_mds $fs_name
+      echo "Removing existing filesystem '${fs_name}'..."
+      ceph fs rm $fs_name --yes-i-really-mean-it
+      echo "Removed '${fs_name}'."
+  done
+}
+
+# So that tests requiring MDS can skip if one is not configured
+# in the cluster at all
+function mds_exists()
+{
+    ceph auth ls | grep "^mds"
+}
+
+# some of the commands are just not idempotent.
+function without_test_dup_command()
+{
+  if [ -z ${CEPH_CLI_TEST_DUP_COMMAND+x} ]; then
+    $@
+  else
+    local saved=${CEPH_CLI_TEST_DUP_COMMAND}
+    unset CEPH_CLI_TEST_DUP_COMMAND
+    $@
+    CEPH_CLI_TEST_DUP_COMMAND=saved
+  fi
+}
+
+function test_mds_tell()
+{
+  local FS_NAME=cephfs
+  if ! mds_exists ; then
+      echo "Skipping test, no MDS found"
+      return
+  fi
+
+  remove_all_fs
+  ceph osd pool create fs_data 10
+  ceph osd pool create fs_metadata 10
+  ceph fs new $FS_NAME fs_metadata fs_data
+  wait_mds_active $FS_NAME
+
+  # Test injectargs by GID
+  old_mds_gids=$(get_mds_gids $FS_NAME)
+  echo Old GIDs: $old_mds_gids
+
+  for mds_gid in $old_mds_gids ; do
+      ceph tell mds.$mds_gid injectargs "--debug-mds 20"
+  done
+  expect_false ceph tell mds.a injectargs mds_max_file_recover -1
+
+  # Test respawn by rank
+  without_test_dup_command ceph tell mds.0 respawn
+  new_mds_gids=$old_mds_gids
+  while [ $new_mds_gids -eq $old_mds_gids ] ; do
+      sleep 5
+      new_mds_gids=$(get_mds_gids $FS_NAME)
+  done
+  echo New GIDs: $new_mds_gids
+
+  # Test respawn by ID
+  without_test_dup_command ceph tell mds.a respawn
+  new_mds_gids=$old_mds_gids
+  while [ $new_mds_gids -eq $old_mds_gids ] ; do
+      sleep 5
+      new_mds_gids=$(get_mds_gids $FS_NAME)
+  done
+  echo New GIDs: $new_mds_gids
+
+  remove_all_fs
+  ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it
+  ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it
+}
+
+function test_mon_mds()
+{
+  local FS_NAME=cephfs
+  remove_all_fs
+
+  ceph osd pool create fs_data 10
+  ceph osd pool create fs_metadata 10
+  ceph fs new $FS_NAME fs_metadata fs_data
+
+  ceph fs set $FS_NAME cluster_down true
+  ceph fs set $FS_NAME cluster_down false
+
+  ceph mds compat rm_incompat 4
+  ceph mds compat rm_incompat 4
+
+  # We don't want any MDSs to be up, their activity can interfere with
+  # the "current_epoch + 1" checking below if they're generating updates
+  fail_all_mds $FS_NAME
+
+  ceph mds compat show
+  ceph fs dump
+  ceph fs get $FS_NAME
+  for mds_gid in $(get_mds_gids $FS_NAME) ; do
+      ceph mds metadata $mds_id
+  done
+  ceph mds metadata
+  ceph mds versions
+  ceph mds count-metadata os
+
+  # XXX mds fail, but how do you undo it?
+  mdsmapfile=$TEMP_DIR/mdsmap.$$
+  current_epoch=$(ceph fs dump -o $mdsmapfile --no-log-to-stderr 2>&1 | grep epoch | sed 's/.*epoch //')
+  [ -s $mdsmapfile ]
+  rm $mdsmapfile
+
+  ceph osd pool create data2 10
+  ceph osd pool create data3 10
+  data2_pool=$(ceph osd dump | grep "pool.*'data2'" | awk '{print $2;}')
+  data3_pool=$(ceph osd dump | grep "pool.*'data3'" | awk '{print $2;}')
+  ceph fs add_data_pool cephfs $data2_pool
+  ceph fs add_data_pool cephfs $data3_pool
+  ceph fs add_data_pool cephfs 100 >& $TMPFILE || true
+  check_response "Error ENOENT"
+  ceph fs add_data_pool cephfs foobarbaz >& $TMPFILE || true
+  check_response "Error ENOENT"
+  ceph fs rm_data_pool cephfs $data2_pool
+  ceph fs rm_data_pool cephfs $data3_pool
+  ceph osd pool delete data2 data2 --yes-i-really-really-mean-it
+  ceph osd pool delete data3 data3 --yes-i-really-really-mean-it
+  ceph fs set cephfs max_mds 4
+  ceph fs set cephfs max_mds 3
+  ceph fs set cephfs max_mds 256
+  expect_false ceph fs set cephfs max_mds 257
+  ceph fs set cephfs max_mds 4
+  ceph fs set cephfs max_mds 256
+  expect_false ceph fs set cephfs max_mds 257
+  expect_false ceph fs set cephfs max_mds asdf
+  expect_false ceph fs set cephfs inline_data true
+  ceph fs set cephfs inline_data true --yes-i-really-mean-it
+  ceph fs set cephfs inline_data yes --yes-i-really-mean-it
+  ceph fs set cephfs inline_data 1 --yes-i-really-mean-it
+  expect_false ceph fs set cephfs inline_data --yes-i-really-mean-it
+  ceph fs set cephfs inline_data false
+  ceph fs set cephfs inline_data no
+  ceph fs set cephfs inline_data 0
+  expect_false ceph fs set cephfs inline_data asdf
+  ceph fs set cephfs max_file_size 1048576
+  expect_false ceph fs set cephfs max_file_size 123asdf
+
+  expect_false ceph fs set cephfs allow_new_snaps
+  ceph fs set cephfs allow_new_snaps true
+  ceph fs set cephfs allow_new_snaps 0
+  ceph fs set cephfs allow_new_snaps false
+  ceph fs set cephfs allow_new_snaps no
+  expect_false ceph fs set cephfs allow_new_snaps taco
+
+  # we should never be able to add EC pools as data or metadata pools
+  # create an ec-pool...
+  ceph osd pool create mds-ec-pool 10 10 erasure
+  set +e
+  ceph fs add_data_pool cephfs mds-ec-pool 2>$TMPFILE
+  check_response 'erasure-code' $? 22
+  set -e
+  ec_poolnum=$(ceph osd dump | grep "pool.* 'mds-ec-pool" | awk '{print $2;}')
+  data_poolnum=$(ceph osd dump | grep "pool.* 'fs_data" | awk '{print $2;}')
+  metadata_poolnum=$(ceph osd dump | grep "pool.* 'fs_metadata" | awk '{print $2;}')
+
+  fail_all_mds $FS_NAME
+
+  set +e
+  # Check that rmfailed requires confirmation
+  expect_false ceph mds rmfailed 0
+  ceph mds rmfailed 0 --yes-i-really-mean-it
+  set -e
+
+  # Check that `fs new` is no longer permitted
+  expect_false ceph fs new cephfs $metadata_poolnum $data_poolnum --yes-i-really-mean-it 2>$TMPFILE
+
+  # Check that 'fs reset' runs
+  ceph fs reset $FS_NAME --yes-i-really-mean-it
+
+  # Check that creating a second FS fails by default
+  ceph osd pool create fs_metadata2 10
+  ceph osd pool create fs_data2 10
+  set +e
+  expect_false ceph fs new cephfs2 fs_metadata2 fs_data2
+  set -e
+
+  # Check that setting enable_multiple enables creation of second fs
+  ceph fs flag set enable_multiple true --yes-i-really-mean-it
+  ceph fs new cephfs2 fs_metadata2 fs_data2
+
+  # Clean up multi-fs stuff
+  fail_all_mds cephfs2
+  ceph fs rm cephfs2 --yes-i-really-mean-it
+  ceph osd pool delete fs_metadata2 fs_metadata2 --yes-i-really-really-mean-it
+  ceph osd pool delete fs_data2 fs_data2 --yes-i-really-really-mean-it
+
+  fail_all_mds $FS_NAME
+
+  # Clean up to enable subsequent fs new tests
+  ceph fs rm $FS_NAME --yes-i-really-mean-it
+
+  set +e
+  ceph fs new $FS_NAME fs_metadata mds-ec-pool --force 2>$TMPFILE
+  check_response 'erasure-code' $? 22
+  ceph fs new $FS_NAME mds-ec-pool fs_data 2>$TMPFILE
+  check_response 'erasure-code' $? 22
+  ceph fs new $FS_NAME mds-ec-pool mds-ec-pool 2>$TMPFILE
+  check_response 'erasure-code' $? 22
+  set -e
+
+  # ... new create a cache tier in front of the EC pool...
+  ceph osd pool create mds-tier 2
+  ceph osd tier add mds-ec-pool mds-tier
+  ceph osd tier set-overlay mds-ec-pool mds-tier
+  tier_poolnum=$(ceph osd dump | grep "pool.* 'mds-tier" | awk '{print $2;}')
+
+  # Use of a readonly tier should be forbidden
+  ceph osd tier cache-mode mds-tier readonly --yes-i-really-mean-it
+  set +e
+  ceph fs new $FS_NAME fs_metadata mds-ec-pool --force 2>$TMPFILE
+  check_response 'has a write tier (mds-tier) that is configured to forward' $? 22
+  set -e
+
+  # Use of a writeback tier should enable FS creation
+  ceph osd tier cache-mode mds-tier writeback
+  ceph fs new $FS_NAME fs_metadata mds-ec-pool --force
+
+  # While a FS exists using the tiered pools, I should not be allowed
+  # to remove the tier
+  set +e
+  ceph osd tier remove-overlay mds-ec-pool 2>$TMPFILE
+  check_response 'in use by CephFS' $? 16
+  ceph osd tier remove mds-ec-pool mds-tier 2>$TMPFILE
+  check_response 'in use by CephFS' $? 16
+  set -e
+
+  fail_all_mds $FS_NAME
+  ceph fs rm $FS_NAME --yes-i-really-mean-it
+
+  # ... but we should be forbidden from using the cache pool in the FS directly.
+  set +e
+  ceph fs new $FS_NAME fs_metadata mds-tier --force 2>$TMPFILE
+  check_response 'in use as a cache tier' $? 22
+  ceph fs new $FS_NAME mds-tier fs_data 2>$TMPFILE
+  check_response 'in use as a cache tier' $? 22
+  ceph fs new $FS_NAME mds-tier mds-tier 2>$TMPFILE
+  check_response 'in use as a cache tier' $? 22
+  set -e
+
+  # Clean up tier + EC pools
+  ceph osd tier remove-overlay mds-ec-pool
+  ceph osd tier remove mds-ec-pool mds-tier
+
+  # Create a FS using the 'cache' pool now that it's no longer a tier
+  ceph fs new $FS_NAME fs_metadata mds-tier --force
+
+  # We should be forbidden from using this pool as a tier now that
+  # it's in use for CephFS
+  set +e
+  ceph osd tier add mds-ec-pool mds-tier 2>$TMPFILE
+  check_response 'in use by CephFS' $? 16
+  set -e
+
+  fail_all_mds $FS_NAME
+  ceph fs rm $FS_NAME --yes-i-really-mean-it
+
+  # We should be permitted to use an EC pool with overwrites enabled
+  # as the data pool...
+  ceph osd pool set mds-ec-pool allow_ec_overwrites true
+  ceph fs new $FS_NAME fs_metadata mds-ec-pool --force 2>$TMPFILE
+  fail_all_mds $FS_NAME
+  ceph fs rm $FS_NAME --yes-i-really-mean-it
+
+  # ...but not as the metadata pool
+  set +e
+  ceph fs new $FS_NAME mds-ec-pool fs_data 2>$TMPFILE
+  check_response 'erasure-code' $? 22
+  set -e
+
+  ceph osd pool delete mds-ec-pool mds-ec-pool --yes-i-really-really-mean-it
+
+  # Create a FS and check that we can subsequently add a cache tier to it
+  ceph fs new $FS_NAME fs_metadata fs_data --force
+
+  # Adding overlay to FS pool should be permitted, RADOS clients handle this.
+  ceph osd tier add fs_metadata mds-tier
+  ceph osd tier cache-mode mds-tier writeback
+  ceph osd tier set-overlay fs_metadata mds-tier
+
+  # Removing tier should be permitted because the underlying pool is
+  # replicated (#11504 case)
+  ceph osd tier cache-mode mds-tier proxy
+  ceph osd tier remove-overlay fs_metadata
+  ceph osd tier remove fs_metadata mds-tier
+  ceph osd pool delete mds-tier mds-tier --yes-i-really-really-mean-it
+
+  # Clean up FS
+  fail_all_mds $FS_NAME
+  ceph fs rm $FS_NAME --yes-i-really-mean-it
+
+
+
+  ceph mds stat
+  # ceph mds tell mds.a getmap
+  # ceph mds rm
+  # ceph mds rmfailed
+  # ceph mds set_state
+
+  ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it
+  ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it
+}
+
+function test_mon_mds_metadata()
+{
+  local nmons=$(ceph tell 'mon.*' version | grep -c 'version')
+  test "$nmons" -gt 0
+
+  ceph fs dump |
+  sed -nEe "s/^([0-9]+):.*'([a-z])' mds\\.([0-9]+)\\..*/\\1 \\2 \\3/p" |
+  while read gid id rank; do
+    ceph mds metadata ${gid} | grep '"hostname":'
+    ceph mds metadata ${id} | grep '"hostname":'
+    ceph mds metadata ${rank} | grep '"hostname":'
+
+    local n=$(ceph tell 'mon.*' mds metadata ${id} | grep -c '"hostname":')
+    test "$n" -eq "$nmons"
+  done
+
+  expect_false ceph mds metadata UNKNOWN
+}
+
+function test_mon_mon()
+{
+  # print help message
+  ceph --help mon
+  # no mon add/remove
+  ceph mon dump
+  ceph mon getmap -o $TEMP_DIR/monmap.$$
+  [ -s $TEMP_DIR/monmap.$$ ]
+  # ceph mon tell
+  ceph mon_status
+
+  # test mon features
+  ceph mon feature ls
+  ceph mon feature set kraken --yes-i-really-mean-it
+  expect_false ceph mon feature set abcd
+  expect_false ceph mon feature set abcd --yes-i-really-mean-it
+
+  # test mon stat
+  # don't check output, just ensure it does not fail.
+  ceph mon stat
+  ceph mon stat -f json | jq '.'
+}
+
+function gen_secrets_file()
+{
+  # lets assume we can have the following types
+  #  all - generates both cephx and lockbox, with mock dm-crypt key
+  #  cephx - only cephx
+  #  no_cephx - lockbox and dm-crypt, no cephx
+  #  no_lockbox - dm-crypt and cephx, no lockbox
+  #  empty - empty file
+  #  empty_json - correct json, empty map
+  #  bad_json - bad json :)
+  #
+  local t=$1
+  if [[ -z "$t" ]]; then
+    t="all"
+  fi
+
+  fn=$(mktemp $TEMP_DIR/secret.XXXXXX)
+  echo $fn
+  if [[ "$t" == "empty" ]]; then
+    return 0
+  fi
+
+  echo "{" > $fn
+  if [[ "$t" == "bad_json" ]]; then
+    echo "asd: ; }" >> $fn
+    return 0
+  elif [[ "$t" == "empty_json" ]]; then
+    echo "}" >> $fn
+    return 0
+  fi
+
+  cephx_secret="\"cephx_secret\": \"$(ceph-authtool --gen-print-key)\""
+  lb_secret="\"cephx_lockbox_secret\": \"$(ceph-authtool --gen-print-key)\""
+  dmcrypt_key="\"dmcrypt_key\": \"$(ceph-authtool --gen-print-key)\""
+
+  if [[ "$t" == "all" ]]; then
+    echo "$cephx_secret,$lb_secret,$dmcrypt_key" >> $fn
+  elif [[ "$t" == "cephx" ]]; then
+    echo "$cephx_secret" >> $fn
+  elif [[ "$t" == "no_cephx" ]]; then
+    echo "$lb_secret,$dmcrypt_key" >> $fn
+  elif [[ "$t" == "no_lockbox" ]]; then
+    echo "$cephx_secret,$dmcrypt_key" >> $fn
+  else
+    echo "unknown gen_secrets_file() type \'$fn\'"
+    return 1
+  fi
+  echo "}" >> $fn
+  return 0
+}
+
+function test_mon_osd_create_destroy()
+{
+  ceph osd new 2>&1 | grep 'EINVAL'
+  ceph osd new '' -1 2>&1 | grep 'EINVAL'
+  ceph osd new '' 10 2>&1 | grep 'EINVAL'
+
+  old_maxosd=$(ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
+
+  old_osds=$(ceph osd ls)
+  num_osds=$(ceph osd ls | wc -l)
+
+  uuid=$(uuidgen)
+  id=$(ceph osd new $uuid 2>/dev/null)
+
+  for i in $old_osds; do
+    [[ "$i" != "$id" ]]
+  done
+
+  ceph osd find $id
+
+  id2=`ceph osd new $uuid 2>/dev/null`
+
+  [[ $id2 == $id ]]
+
+  ceph osd new $uuid $id
+
+  id3=$(ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
+  ceph osd new $uuid $((id3+1)) 2>&1 | grep EEXIST
+
+  uuid2=$(uuidgen)
+  id2=$(ceph osd new $uuid2)
+  ceph osd find $id2
+  [[ "$id2" != "$id" ]]
+
+  ceph osd new $uuid $id2 2>&1 | grep EEXIST
+  ceph osd new $uuid2 $id2
+
+  # test with secrets
+  empty_secrets=$(gen_secrets_file "empty")
+  empty_json=$(gen_secrets_file "empty_json")
+  all_secrets=$(gen_secrets_file "all")
+  cephx_only=$(gen_secrets_file "cephx")
+  no_cephx=$(gen_secrets_file "no_cephx")
+  no_lockbox=$(gen_secrets_file "no_lockbox")
+  bad_json=$(gen_secrets_file "bad_json")
+
+  # empty secrets should be idempotent
+  new_id=$(ceph osd new $uuid $id -i $empty_secrets)
+  [[ "$new_id" == "$id" ]]
+
+  # empty json, thus empty secrets
+  new_id=$(ceph osd new $uuid $id -i $empty_json)
+  [[ "$new_id" == "$id" ]]
+
+  ceph osd new $uuid $id -i $all_secrets 2>&1 | grep 'EEXIST'
+
+  ceph osd rm $id
+  ceph osd rm $id2
+  ceph osd setmaxosd $old_maxosd
+
+  ceph osd new $uuid -i $no_cephx 2>&1 | grep 'EINVAL'
+  ceph osd new $uuid -i $no_lockbox 2>&1 | grep 'EINVAL'
+
+  osds=$(ceph osd ls)
+  id=$(ceph osd new $uuid -i $all_secrets)
+  for i in $osds; do
+    [[ "$i" != "$id" ]]
+  done
+
+  ceph osd find $id
+
+  # validate secrets and dm-crypt are set
+  k=$(ceph auth get-key osd.$id --format=json-pretty 2>/dev/null | jq '.key')
+  s=$(cat $all_secrets | jq '.cephx_secret')
+  [[ $k == $s ]]
+  k=$(ceph auth get-key client.osd-lockbox.$uuid --format=json-pretty 2>/dev/null | \
+      jq '.key')
+  s=$(cat $all_secrets | jq '.cephx_lockbox_secret')
+  [[ $k == $s ]]
+  ceph config-key exists dm-crypt/osd/$uuid/luks
+
+  osds=$(ceph osd ls)
+  id2=$(ceph osd new $uuid2 -i $cephx_only)
+  for i in $osds; do
+    [[ "$i" != "$id2" ]]
+  done
+
+  ceph osd find $id2
+  k=$(ceph auth get-key osd.$id --format=json-pretty 2>/dev/null | jq '.key')
+  s=$(cat $all_secrets | jq '.cephx_secret')
+  [[ $k == $s ]]
+  expect_false ceph auth get-key client.osd-lockbox.$uuid2
+  expect_false ceph config-key exists dm-crypt/osd/$uuid2/luks
+
+  ceph osd destroy osd.$id2 --yes-i-really-mean-it
+  ceph osd destroy $id2 --yes-i-really-mean-it
+  ceph osd find $id2
+  expect_false ceph auth get-key osd.$id2
+  ceph osd dump | grep osd.$id2 | grep destroyed
+
+  id3=$id2
+  uuid3=$(uuidgen)
+  ceph osd new $uuid3 $id3 -i $all_secrets
+  ceph osd dump | grep osd.$id3 | expect_false grep destroyed
+  ceph auth get-key client.osd-lockbox.$uuid3
+  ceph auth get-key osd.$id3
+  ceph config-key exists dm-crypt/osd/$uuid3/luks
+
+  ceph osd purge-new osd.$id3 --yes-i-really-mean-it
+  expect_false ceph osd find $id2
+  expect_false ceph auth get-key osd.$id2
+  expect_false ceph auth get-key client.osd-lockbox.$uuid3
+  expect_false ceph config-key exists dm-crypt/osd/$uuid3/luks
+  ceph osd purge osd.$id3 --yes-i-really-mean-it
+  ceph osd purge-new osd.$id3 --yes-i-really-mean-it # idempotent
+
+  ceph osd purge osd.$id --yes-i-really-mean-it
+  ceph osd purge 123456 --yes-i-really-mean-it
+  expect_false ceph osd find $id
+  expect_false ceph auth get-key osd.$id
+  expect_false ceph auth get-key client.osd-lockbox.$uuid
+  expect_false ceph config-key exists dm-crypt/osd/$uuid/luks
+
+  rm $empty_secrets $empty_json $all_secrets $cephx_only \
+     $no_cephx $no_lockbox $bad_json
+
+  for i in $(ceph osd ls); do
+    [[ "$i" != "$id" ]]
+    [[ "$i" != "$id2" ]]
+    [[ "$i" != "$id3" ]]
+  done
+
+  [[ "$(ceph osd ls | wc -l)" == "$num_osds" ]]
+  ceph osd setmaxosd $old_maxosd
+
+}
+
+function test_mon_config_key()
+{
+  key=asdfasdfqwerqwreasdfuniquesa123df
+  ceph config-key list | grep -c $key | grep 0
+  ceph config-key get $key | grep -c bar | grep 0
+  ceph config-key set $key bar
+  ceph config-key get $key | grep bar
+  ceph config-key list | grep -c $key | grep 1
+  ceph config-key dump | grep $key | grep bar
+  ceph config-key rm $key
+  expect_false ceph config-key get $key
+  ceph config-key list | grep -c $key | grep 0
+  ceph config-key dump | grep -c $key | grep 0
+}
+
+function test_mon_osd()
+{
+  #
+  # osd blacklist
+  #
+  bl=192.168.0.1:0/1000
+  ceph osd blacklist add $bl
+  ceph osd blacklist ls | grep $bl
+  ceph osd blacklist ls --format=json-pretty  | sed 's/\\\//\//' | grep $bl
+  ceph osd dump --format=json-pretty | grep $bl
+  ceph osd dump | grep $bl
+  ceph osd blacklist rm $bl
+  ceph osd blacklist ls | expect_false grep $bl
+
+  bl=192.168.0.1
+  # test without nonce, invalid nonce
+  ceph osd blacklist add $bl
+  ceph osd blacklist ls | grep $bl
+  ceph osd blacklist rm $bl
+  ceph osd blacklist ls | expect_false grep $bl
+  expect_false "ceph osd blacklist $bl/-1"
+  expect_false "ceph osd blacklist $bl/foo"
+
+  # test with wrong address
+  expect_false "ceph osd blacklist 1234.56.78.90/100"
+
+  # Test `clear`
+  ceph osd blacklist add $bl
+  ceph osd blacklist ls | grep $bl
+  ceph osd blacklist clear
+  ceph osd blacklist ls | expect_false grep $bl
+
+  #
+  # osd crush
+  #
+  ceph osd crush reweight-all
+  ceph osd crush tunables legacy
+  ceph osd crush show-tunables | grep argonaut
+  ceph osd crush tunables bobtail
+  ceph osd crush show-tunables | grep bobtail
+  ceph osd crush tunables firefly
+  ceph osd crush show-tunables | grep firefly
+
+  ceph osd crush set-tunable straw_calc_version 0
+  ceph osd crush get-tunable straw_calc_version | grep 0
+  ceph osd crush set-tunable straw_calc_version 1
+  ceph osd crush get-tunable straw_calc_version | grep 1
+
+  #
+  # require-min-compat-client
+  expect_false ceph osd set-require-min-compat-client dumpling  # firefly tunables
+  ceph osd set-require-min-compat-client luminous
+  ceph osd get-require-min-compat-client | grep luminous
+  ceph osd dump | grep 'require_min_compat_client luminous'
+
+  #
+  # osd scrub
+  #
+
+  # blocking
+  ceph osd scrub 0 --block
+  ceph osd deep-scrub 0 --block
+
+  # how do I tell when these are done?
+  ceph osd scrub 0
+  ceph osd deep-scrub 0
+  ceph osd repair 0
+
+  # pool scrub, force-recovery/backfill
+  pool_names=`rados lspools`
+  for pool_name in $pool_names
+  do
+    ceph osd pool scrub $pool_name
+    ceph osd pool deep-scrub $pool_name
+    ceph osd pool repair $pool_name
+    ceph osd pool force-recovery $pool_name
+    ceph osd pool cancel-force-recovery $pool_name
+    ceph osd pool force-backfill $pool_name
+    ceph osd pool cancel-force-backfill $pool_name
+  done
+
+  for f in noup nodown noin noout noscrub nodeep-scrub nobackfill \
+	  norebalance norecover notieragent full
+  do
+    ceph osd set $f
+    ceph osd unset $f
+  done
+  expect_false ceph osd set bogus
+  expect_false ceph osd unset bogus
+  for f in sortbitwise recover_deletes require_jewel_osds \
+	  require_kraken_osds
+  do
+	expect_false ceph osd set $f
+	expect_false ceph osd unset $f
+  done
+  ceph osd require-osd-release nautilus
+  # can't lower (or use new command for anything but jewel)
+  expect_false ceph osd require-osd-release jewel
+  # these are no-ops but should succeed.
+
+  ceph osd set noup
+  ceph osd down 0
+  ceph osd dump | grep 'osd.0 down'
+  ceph osd unset noup
+  max_run=1000
+  for ((i=0; i < $max_run; i++)); do
+    if ! ceph osd dump | grep 'osd.0 up'; then
+      echo "waiting for osd.0 to come back up ($i/$max_run)"
+      sleep 1
+    else
+      break
+    fi
+  done
+  ceph osd dump | grep 'osd.0 up'
+
+  ceph osd dump | grep 'osd.0 up'
+  # ceph osd find expects the OsdName, so both ints and osd.n should work.
+  ceph osd find 1
+  ceph osd find osd.1
+  expect_false ceph osd find osd.xyz
+  expect_false ceph osd find xyz
+  expect_false ceph osd find 0.1
+  ceph --format plain osd find 1 # falls back to json-pretty
+  if [ `uname` == Linux ]; then
+    ceph osd metadata 1 | grep 'distro'
+    ceph --format plain osd metadata 1 | grep 'distro' # falls back to json-pretty
+  fi
+  ceph osd out 0
+  ceph osd dump | grep 'osd.0.*out'
+  ceph osd in 0
+  ceph osd dump | grep 'osd.0.*in'
+  ceph osd find 0
+
+  ceph osd add-nodown 0 1
+  ceph health detail | grep 'NODOWN'
+  ceph osd rm-nodown 0 1
+  ! ceph health detail | grep 'NODOWN'
+
+  ceph osd out 0 # so we can mark it as noin later
+  ceph osd add-noin 0
+  ceph health detail | grep 'NOIN'
+  ceph osd rm-noin 0
+  ! ceph health detail | grep 'NOIN'
+  ceph osd in 0
+
+  ceph osd add-noout 0
+  ceph health detail | grep 'NOOUT'
+  ceph osd rm-noout 0
+  ! ceph health detail | grep 'NOOUT'
+
+  # test osd id parse
+  expect_false ceph osd add-noup 797er
+  expect_false ceph osd add-nodown u9uwer
+  expect_false ceph osd add-noin 78~15
+
+  expect_false ceph osd rm-noup 1234567
+  expect_false ceph osd rm-nodown fsadf7
+  expect_false ceph osd rm-noout 790-fd
+
+  ids=`ceph osd ls-tree default`
+  for osd in $ids
+  do
+    ceph osd add-nodown $osd
+    ceph osd add-noout $osd
+  done
+  ceph -s | grep 'NODOWN'
+  ceph -s | grep 'NOOUT'
+  ceph osd rm-nodown any
+  ceph osd rm-noout all
+  ! ceph -s | grep 'NODOWN'
+  ! ceph -s | grep 'NOOUT'
+
+  # test crush node flags
+  ceph osd add-noup osd.0
+  ceph osd add-nodown osd.0
+  ceph osd add-noin osd.0
+  ceph osd add-noout osd.0
+  ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep "osd.0"
+  ceph osd rm-noup osd.0
+  ceph osd rm-nodown osd.0
+  ceph osd rm-noin osd.0
+  ceph osd rm-noout osd.0
+  ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep "osd.0"
+
+  ceph osd crush add-bucket foo host root=default
+  ceph osd add-noup foo
+  ceph osd add-nodown foo
+  ceph osd add-noin foo
+  ceph osd add-noout foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags" | grep foo
+  ceph osd rm-noup foo
+  ceph osd rm-nodown foo
+  ceph osd rm-noin foo
+  ceph osd rm-noout foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep foo
+  ceph osd add-noup foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags" | grep foo
+  ceph osd crush rm foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep foo
+
+  ceph osd set-group noup osd.0
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup'
+  ceph osd set-group noup,nodown osd.0
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown'
+  ceph osd set-group noup,nodown,noin osd.0
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin'
+  ceph osd set-group noup,nodown,noin,noout osd.0
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout'
+  ceph osd unset-group noup osd.0
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout'
+  ceph osd unset-group noup,nodown osd.0
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout'
+  ceph osd unset-group noup,nodown,noin osd.0
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown\|noin'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout'
+  ceph osd unset-group noup,nodown,noin,noout osd.0
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown\|noin\|noout'
+
+  ceph osd set-group noup,nodown,noin,noout osd.0 osd.1
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noout'
+  ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".osds[1].state" | grep 'noout'
+  ceph osd unset-group noup,nodown,noin,noout osd.0 osd.1
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup\|nodown\|noin\|noout'
+  ceph osd dump -f json-pretty | jq ".osds[1].state" | expect_false grep 'noup\|nodown\|noin\|noout'
+
+  ceph osd set-group noup all
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | grep 'noup'
+  ceph osd unset-group noup all
+  ceph osd dump -f json-pretty | jq ".osds[0].state" | expect_false grep 'noup'
+
+  # crush node flags
+  ceph osd crush add-bucket foo host root=default
+  ceph osd set-group noup foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup'
+  ceph osd set-group noup,nodown foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown'
+  ceph osd set-group noup,nodown,noin foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin'
+  ceph osd set-group noup,nodown,noin,noout foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout'
+
+  ceph osd unset-group noup foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout'
+  ceph osd unset-group noup,nodown foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup\|nodown'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout'
+  ceph osd unset-group noup,nodown,noin foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup\|nodown\|noin'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout'
+  ceph osd unset-group noup,nodown,noin,noout foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | expect_false grep 'noup\|nodown\|noin\|noout'
+
+  ceph osd set-group noin,noout foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout'
+  ceph osd unset-group noin,noout foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep 'foo'
+
+  ceph osd set-group noup,nodown,noin,noout foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".crush_node_flags.foo" | grep 'noout'
+  ceph osd crush rm foo
+  ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep 'foo'
+
+  # test device class flags
+  osd_0_device_class=$(ceph osd crush get-device-class osd.0)
+  ceph osd set-group noup $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noup'
+  ceph osd set-group noup,nodown $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'nodown'
+  ceph osd set-group noup,nodown,noin $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin'
+  ceph osd set-group noup,nodown,noin,noout $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noup'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout'
+
+  ceph osd unset-group noup $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | expect_false grep 'noup'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'nodown'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout'
+  ceph osd unset-group noup,nodown $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | expect_false grep 'noup\|nodown'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout'
+  ceph osd unset-group noup,nodown,noin $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | expect_false grep 'noup\|nodown\|noin'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout'
+  ceph osd unset-group noup,nodown,noin,noout $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | expect_false grep 'noup\|nodown\|noin\|noout'
+
+  ceph osd set-group noin,noout $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noin'
+  ceph osd dump -f json-pretty | jq ".device_class_flags.$osd_0_device_class" | grep 'noout'
+  ceph osd unset-group noin,noout $osd_0_device_class
+  ceph osd dump -f json-pretty | jq ".crush_node_flags" | expect_false grep $osd_0_device_class
+
+  # make sure mark out preserves weight
+  ceph osd reweight osd.0 .5
+  ceph osd dump | grep ^osd.0 | grep 'weight 0.5'
+  ceph osd out 0
+  ceph osd in 0
+  ceph osd dump | grep ^osd.0 | grep 'weight 0.5'
+
+  ceph osd getmap -o $f
+  [ -s $f ]
+  rm $f
+  save=$(ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//')
+  [ "$save" -gt 0 ]
+  ceph osd setmaxosd $((save - 1)) 2>&1 | grep 'EBUSY'
+  ceph osd setmaxosd 10
+  ceph osd getmaxosd | grep 'max_osd = 10'
+  ceph osd setmaxosd $save
+  ceph osd getmaxosd | grep "max_osd = $save"
+
+  for id in `ceph osd ls` ; do
+    retry_eagain 5 map_enxio_to_eagain ceph tell osd.$id version
+  done
+
+  ceph osd rm 0 2>&1 | grep 'EBUSY'
+
+  local old_osds=$(echo $(ceph osd ls))
+  id=`ceph osd create`
+  ceph osd find $id
+  ceph osd lost $id --yes-i-really-mean-it
+  expect_false ceph osd setmaxosd $id
+  local new_osds=$(echo $(ceph osd ls))
+  for id in $(echo $new_osds | sed -e "s/$old_osds//") ; do
+      ceph osd rm $id
+  done
+
+  uuid=`uuidgen`
+  id=`ceph osd create $uuid`
+  id2=`ceph osd create $uuid`
+  [ "$id" = "$id2" ]
+  ceph osd rm $id
+
+  ceph --help osd
+
+  # reset max_osd.
+  ceph osd setmaxosd $id
+  ceph osd getmaxosd | grep "max_osd = $save"
+  local max_osd=$save
+
+  ceph osd create $uuid 0 2>&1 | grep 'EINVAL'
+  ceph osd create $uuid $((max_osd - 1)) 2>&1 | grep 'EINVAL'
+
+  id=`ceph osd create $uuid $max_osd`
+  [ "$id" = "$max_osd" ]
+  ceph osd find $id
+  max_osd=$((max_osd + 1))
+  ceph osd getmaxosd | grep "max_osd = $max_osd"
+
+  ceph osd create $uuid $((id - 1)) 2>&1 | grep 'EEXIST'
+  ceph osd create $uuid $((id + 1)) 2>&1 | grep 'EEXIST'
+  id2=`ceph osd create $uuid`
+  [ "$id" = "$id2" ]
+  id2=`ceph osd create $uuid $id`
+  [ "$id" = "$id2" ]
+
+  uuid=`uuidgen`
+  local gap_start=$max_osd
+  id=`ceph osd create $uuid $((gap_start + 100))`
+  [ "$id" = "$((gap_start + 100))" ]
+  max_osd=$((id + 1))
+  ceph osd getmaxosd | grep "max_osd = $max_osd"
+
+  ceph osd create $uuid $gap_start 2>&1 | grep 'EEXIST'
+
+  #
+  # When CEPH_CLI_TEST_DUP_COMMAND is set, osd create
+  # is repeated and consumes two osd id, not just one.
+  #
+  local next_osd=$gap_start
+  id=`ceph osd create $(uuidgen)`
+  [ "$id" = "$next_osd" ]
+
+  next_osd=$((id + 1))
+  id=`ceph osd create $(uuidgen) $next_osd`
+  [ "$id" = "$next_osd" ]
+
+  local new_osds=$(echo $(ceph osd ls))
+  for id in $(echo $new_osds | sed -e "s/$old_osds//") ; do
+      [ $id -ge $save ]
+      ceph osd rm $id
+  done
+  ceph osd setmaxosd $save
+
+  ceph osd ls
+  ceph osd pool create data 10
+  ceph osd pool application enable data rados
+  ceph osd lspools | grep data
+  ceph osd map data foo | grep 'pool.*data.*object.*foo.*pg.*up.*acting'
+  ceph osd map data foo namespace| grep 'pool.*data.*object.*namespace/foo.*pg.*up.*acting'
+  ceph osd pool delete data data --yes-i-really-really-mean-it
+
+  ceph osd pause
+  ceph osd dump | grep 'flags.*pauserd,pausewr'
+  ceph osd unpause
+
+  ceph osd tree
+  ceph osd tree up
+  ceph osd tree down
+  ceph osd tree in
+  ceph osd tree out
+  ceph osd tree destroyed
+  ceph osd tree up in
+  ceph osd tree up out
+  ceph osd tree down in
+  ceph osd tree down out
+  ceph osd tree out down
+  expect_false ceph osd tree up down
+  expect_false ceph osd tree up destroyed
+  expect_false ceph osd tree down destroyed
+  expect_false ceph osd tree up down destroyed
+  expect_false ceph osd tree in out
+  expect_false ceph osd tree up foo
+
+  ceph osd metadata
+  ceph osd count-metadata os
+  ceph osd versions
+
+  ceph osd perf
+  ceph osd blocked-by
+
+  ceph osd stat | grep up
+}
+
+function test_mon_crush()
+{
+  f=$TEMP_DIR/map.$$
+  epoch=$(ceph osd getcrushmap -o $f 2>&1 | tail -n1)
+  [ -s $f ]
+  [ "$epoch" -gt 1 ]
+  nextepoch=$(( $epoch + 1 ))
+  echo epoch $epoch nextepoch $nextepoch
+  rm -f $f.epoch
+  expect_false ceph osd setcrushmap $nextepoch -i $f
+  gotepoch=$(ceph osd setcrushmap $epoch -i $f 2>&1 | tail -n1)
+  echo gotepoch $gotepoch
+  [ "$gotepoch" -eq "$nextepoch" ]
+  # should be idempotent
+  gotepoch=$(ceph osd setcrushmap $epoch -i $f 2>&1 | tail -n1)
+  echo epoch $gotepoch
+  [ "$gotepoch" -eq "$nextepoch" ]
+  rm $f
+}
+
+function test_mon_osd_pool()
+{
+  #
+  # osd pool
+  #
+  ceph osd pool create data 10
+  ceph osd pool application enable data rados
+  ceph osd pool mksnap data datasnap
+  rados -p data lssnap | grep datasnap
+  ceph osd pool rmsnap data datasnap
+  expect_false ceph osd pool rmsnap pool_fake snapshot
+  ceph osd pool delete data data --yes-i-really-really-mean-it
+
+  ceph osd pool create data2 10
+  ceph osd pool application enable data2 rados
+  ceph osd pool rename data2 data3
+  ceph osd lspools | grep data3
+  ceph osd pool delete data3 data3 --yes-i-really-really-mean-it
+
+  ceph osd pool create replicated 12 12 replicated
+  ceph osd pool create replicated 12 12 replicated
+  ceph osd pool create replicated 12 12 # default is replicated
+  ceph osd pool create replicated 12    # default is replicated, pgp_num = pg_num
+  ceph osd pool application enable replicated rados
+  # should fail because the type is not the same
+  expect_false ceph osd pool create replicated 12 12 erasure
+  ceph osd lspools | grep replicated
+  ceph osd pool create ec_test 1 1 erasure
+  ceph osd pool application enable ec_test rados
+  set +e
+  ceph osd count-metadata osd_objectstore | grep 'bluestore'
+  if [ $? -eq 1 ]; then # enable ec_overwrites on non-bluestore pools should fail
+      ceph osd pool set ec_test allow_ec_overwrites true >& $TMPFILE
+      check_response "pool must only be stored on bluestore for scrubbing to work" $? 22
+  else
+      ceph osd pool set ec_test allow_ec_overwrites true || return 1
+      expect_false ceph osd pool set ec_test allow_ec_overwrites false
+  fi
+  set -e
+  ceph osd pool delete replicated replicated --yes-i-really-really-mean-it
+  ceph osd pool delete ec_test ec_test --yes-i-really-really-mean-it
+
+  # test create pool with rule
+  ceph osd erasure-code-profile set foo foo
+  ceph osd erasure-code-profile ls | grep foo
+  ceph osd crush rule create-erasure foo foo
+  ceph osd pool create erasure 12 12 erasure foo
+  expect_false ceph osd erasure-code-profile rm foo
+  ceph osd pool delete erasure erasure --yes-i-really-really-mean-it
+  ceph osd crush rule rm foo
+  ceph osd erasure-code-profile rm foo
+
+}
+
+function test_mon_osd_pool_quota()
+{
+  #
+  # test osd pool set/get quota
+  #
+
+  # create tmp pool
+  ceph osd pool create tmp-quota-pool 36
+  ceph osd pool application enable tmp-quota-pool rados
+  #
+  # set erroneous quotas
+  #
+  expect_false ceph osd pool set-quota tmp-quota-pool max_fooness 10
+  expect_false ceph osd pool set-quota tmp-quota-pool max_bytes -1
+  expect_false ceph osd pool set-quota tmp-quota-pool max_objects aaa
+  #
+  # set valid quotas
+  #
+  ceph osd pool set-quota tmp-quota-pool max_bytes 10
+  ceph osd pool set-quota tmp-quota-pool max_objects 10M
+  #
+  # get quotas in json-pretty format
+  #
+  ceph osd pool get-quota tmp-quota-pool --format=json-pretty | \
+    grep '"quota_max_objects":.*10000000'
+  ceph osd pool get-quota tmp-quota-pool --format=json-pretty | \
+    grep '"quota_max_bytes":.*10'
+  #
+  # get quotas
+  #
+  ceph osd pool get-quota tmp-quota-pool | grep 'max bytes.*10 B'
+  ceph osd pool get-quota tmp-quota-pool | grep 'max objects.*10.*M objects'
+  #
+  # set valid quotas with unit prefix
+  #
+  ceph osd pool set-quota tmp-quota-pool max_bytes 10K
+  #
+  # get quotas
+  #
+  ceph osd pool get-quota tmp-quota-pool | grep 'max bytes.*10 Ki'
+  #
+  # set valid quotas with unit prefix
+  #
+  ceph osd pool set-quota tmp-quota-pool max_bytes 10Ki
+  #
+  # get quotas
+  #
+  ceph osd pool get-quota tmp-quota-pool | grep 'max bytes.*10 Ki'
+  #
+  #
+  # reset pool quotas
+  #
+  ceph osd pool set-quota tmp-quota-pool max_bytes 0
+  ceph osd pool set-quota tmp-quota-pool max_objects 0
+  #
+  # test N/A quotas
+  #
+  ceph osd pool get-quota tmp-quota-pool | grep 'max bytes.*N/A'
+  ceph osd pool get-quota tmp-quota-pool | grep 'max objects.*N/A'
+  #
+  # cleanup tmp pool
+  ceph osd pool delete tmp-quota-pool tmp-quota-pool --yes-i-really-really-mean-it
+}
+
+function test_mon_pg()
+{
+  # Make sure we start healthy.
+  wait_for_health_ok
+
+  ceph pg debug unfound_objects_exist
+  ceph pg debug degraded_pgs_exist
+  ceph pg deep-scrub 1.0
+  ceph pg dump
+  ceph pg dump pgs_brief --format=json
+  ceph pg dump pgs --format=json
+  ceph pg dump pools --format=json
+  ceph pg dump osds --format=json
+  ceph pg dump sum --format=json
+  ceph pg dump all --format=json
+  ceph pg dump pgs_brief osds --format=json
+  ceph pg dump pools osds pgs_brief --format=json
+  ceph pg dump_json
+  ceph pg dump_pools_json
+  ceph pg dump_stuck inactive
+  ceph pg dump_stuck unclean
+  ceph pg dump_stuck stale
+  ceph pg dump_stuck undersized
+  ceph pg dump_stuck degraded
+  ceph pg ls
+  ceph pg ls 1
+  ceph pg ls stale
+  expect_false ceph pg ls scrubq
+  ceph pg ls active stale repair recovering
+  ceph pg ls 1 active
+  ceph pg ls 1 active stale
+  ceph pg ls-by-primary osd.0
+  ceph pg ls-by-primary osd.0 1
+  ceph pg ls-by-primary osd.0 active
+  ceph pg ls-by-primary osd.0 active stale
+  ceph pg ls-by-primary osd.0 1 active stale
+  ceph pg ls-by-osd osd.0
+  ceph pg ls-by-osd osd.0 1
+  ceph pg ls-by-osd osd.0 active
+  ceph pg ls-by-osd osd.0 active stale
+  ceph pg ls-by-osd osd.0 1 active stale
+  ceph pg ls-by-pool rbd
+  ceph pg ls-by-pool rbd active stale
+  # can't test this...
+  # ceph pg force_create_pg
+  ceph pg getmap -o $TEMP_DIR/map.$$
+  [ -s $TEMP_DIR/map.$$ ]
+  ceph pg map 1.0 | grep acting
+  ceph pg repair 1.0
+  ceph pg scrub 1.0
+
+  ceph osd set-full-ratio .962
+  ceph osd dump | grep '^full_ratio 0.962'
+  ceph osd set-backfillfull-ratio .912
+  ceph osd dump | grep '^backfillfull_ratio 0.912'
+  ceph osd set-nearfull-ratio .892
+  ceph osd dump | grep '^nearfull_ratio 0.892'
+
+  # Check health status
+  ceph osd set-nearfull-ratio .913
+  ceph health -f json | grep OSD_OUT_OF_ORDER_FULL
+  ceph health detail | grep OSD_OUT_OF_ORDER_FULL
+  ceph osd set-nearfull-ratio .892
+  ceph osd set-backfillfull-ratio .963
+  ceph health -f json | grep OSD_OUT_OF_ORDER_FULL
+  ceph health detail | grep OSD_OUT_OF_ORDER_FULL
+  ceph osd set-backfillfull-ratio .912
+
+  # Check injected full results
+  $SUDO ceph --admin-daemon $(get_admin_socket osd.0) injectfull nearfull
+  wait_for_health "OSD_NEARFULL"
+  ceph health detail | grep "osd.0 is near full"
+  $SUDO ceph --admin-daemon $(get_admin_socket osd.0) injectfull none
+  wait_for_health_ok
+
+  $SUDO ceph --admin-daemon $(get_admin_socket osd.1) injectfull backfillfull
+  wait_for_health "OSD_BACKFILLFULL"
+  ceph health detail | grep "osd.1 is backfill full"
+  $SUDO ceph --admin-daemon $(get_admin_socket osd.1) injectfull none
+  wait_for_health_ok
+
+  $SUDO ceph --admin-daemon $(get_admin_socket osd.2) injectfull failsafe
+  # failsafe and full are the same as far as the monitor is concerned
+  wait_for_health "OSD_FULL"
+  ceph health detail | grep "osd.2 is full"
+  $SUDO ceph --admin-daemon $(get_admin_socket osd.2) injectfull none
+  wait_for_health_ok
+
+  $SUDO ceph --admin-daemon $(get_admin_socket osd.0) injectfull full
+  wait_for_health "OSD_FULL"
+  ceph health detail | grep "osd.0 is full"
+  $SUDO ceph --admin-daemon $(get_admin_socket osd.0) injectfull none
+  wait_for_health_ok
+
+  ceph pg stat | grep 'pgs:'
+  ceph pg 1.0 query
+  ceph tell 1.0 query
+  ceph quorum enter
+  ceph quorum_status
+  ceph report | grep osd_stats
+  ceph status
+  ceph -s
+
+  #
+  # tell osd version
+  #
+  ceph tell osd.0 version
+  expect_false ceph tell osd.9999 version 
+  expect_false ceph tell osd.foo version
+
+  # back to pg stuff
+
+  ceph tell osd.0 dump_pg_recovery_stats | grep Started
+
+  ceph osd reweight 0 0.9
+  expect_false ceph osd reweight 0 -1
+  ceph osd reweight osd.0 1
+
+  ceph osd primary-affinity osd.0 .9
+  expect_false ceph osd primary-affinity osd.0 -2
+  expect_false ceph osd primary-affinity osd.9999 .5
+  ceph osd primary-affinity osd.0 1
+
+  ceph osd pool set rbd size 2
+  ceph osd pg-temp 1.0 0 1
+  ceph osd pg-temp 1.0 osd.1 osd.0
+  expect_false ceph osd pg-temp 1.0 0 1 2
+  expect_false ceph osd pg-temp asdf qwer
+  expect_false ceph osd pg-temp 1.0 asdf
+  ceph osd pg-temp 1.0 # cleanup pg-temp
+
+  ceph pg repeer 1.0
+  expect_false ceph pg repeer 0.0   # pool 0 shouldn't exist anymore
+
+  # don't test ceph osd primary-temp for now
+}
+
+function test_mon_osd_pool_set()
+{
+  TEST_POOL_GETSET=pool_getset
+  ceph osd pool create $TEST_POOL_GETSET 1
+  ceph osd pool application enable $TEST_POOL_GETSET rados
+  ceph osd pool set $TEST_POOL_GETSET pg_autoscale_mode off
+  wait_for_clean
+  ceph osd pool get $TEST_POOL_GETSET all
+
+  for s in pg_num pgp_num size min_size crush_rule; do
+    ceph osd pool get $TEST_POOL_GETSET $s
+  done
+
+  old_size=$(ceph osd pool get $TEST_POOL_GETSET size | sed -e 's/size: //')
+  (( new_size = old_size + 1 ))
+  ceph osd pool set $TEST_POOL_GETSET size $new_size
+  ceph osd pool get $TEST_POOL_GETSET size | grep "size: $new_size"
+  ceph osd pool set $TEST_POOL_GETSET size $old_size
+
+  ceph osd pool create pool_erasure 1 1 erasure
+  ceph osd pool application enable pool_erasure rados
+  wait_for_clean
+  set +e
+  ceph osd pool set pool_erasure size 4444 2>$TMPFILE
+  check_response 'not change the size'
+  set -e
+  ceph osd pool get pool_erasure erasure_code_profile
+
+  for flag in nodelete nopgchange nosizechange write_fadvise_dontneed noscrub nodeep-scrub; do
+      ceph osd pool set $TEST_POOL_GETSET $flag false
+      ceph osd pool get $TEST_POOL_GETSET $flag | grep "$flag: false"
+      ceph osd pool set $TEST_POOL_GETSET $flag true
+      ceph osd pool get $TEST_POOL_GETSET $flag | grep "$flag: true"
+      ceph osd pool set $TEST_POOL_GETSET $flag 1
+      ceph osd pool get $TEST_POOL_GETSET $flag | grep "$flag: true"
+      ceph osd pool set $TEST_POOL_GETSET $flag 0
+      ceph osd pool get $TEST_POOL_GETSET $flag | grep "$flag: false"
+      expect_false ceph osd pool set $TEST_POOL_GETSET $flag asdf
+      expect_false ceph osd pool set $TEST_POOL_GETSET $flag 2
+  done
+
+  ceph osd pool get $TEST_POOL_GETSET scrub_min_interval | expect_false grep '.'
+  ceph osd pool set $TEST_POOL_GETSET scrub_min_interval 123456
+  ceph osd pool get $TEST_POOL_GETSET scrub_min_interval | grep 'scrub_min_interval: 123456'
+  ceph osd pool set $TEST_POOL_GETSET scrub_min_interval 0
+  ceph osd pool get $TEST_POOL_GETSET scrub_min_interval | expect_false grep '.'
+
+  ceph osd pool get $TEST_POOL_GETSET scrub_max_interval | expect_false grep '.'
+  ceph osd pool set $TEST_POOL_GETSET scrub_max_interval 123456
+  ceph osd pool get $TEST_POOL_GETSET scrub_max_interval | grep 'scrub_max_interval: 123456'
+  ceph osd pool set $TEST_POOL_GETSET scrub_max_interval 0
+  ceph osd pool get $TEST_POOL_GETSET scrub_max_interval | expect_false grep '.'
+
+  ceph osd pool get $TEST_POOL_GETSET deep_scrub_interval | expect_false grep '.'
+  ceph osd pool set $TEST_POOL_GETSET deep_scrub_interval 123456
+  ceph osd pool get $TEST_POOL_GETSET deep_scrub_interval | grep 'deep_scrub_interval: 123456'
+  ceph osd pool set $TEST_POOL_GETSET deep_scrub_interval 0
+  ceph osd pool get $TEST_POOL_GETSET deep_scrub_interval | expect_false grep '.'
+
+  ceph osd pool get $TEST_POOL_GETSET recovery_priority | expect_false grep '.'
+  ceph osd pool set $TEST_POOL_GETSET recovery_priority 5 
+  ceph osd pool get $TEST_POOL_GETSET recovery_priority | grep 'recovery_priority: 5'
+  ceph osd pool set $TEST_POOL_GETSET recovery_priority -5
+  ceph osd pool get $TEST_POOL_GETSET recovery_priority | grep 'recovery_priority: -5'
+  ceph osd pool set $TEST_POOL_GETSET recovery_priority 0
+  ceph osd pool get $TEST_POOL_GETSET recovery_priority | expect_false grep '.'
+  expect_false ceph osd pool set $TEST_POOL_GETSET recovery_priority -11
+  expect_false ceph osd pool set $TEST_POOL_GETSET recovery_priority 11
+
+  ceph osd pool get $TEST_POOL_GETSET recovery_op_priority | expect_false grep '.'
+  ceph osd pool set $TEST_POOL_GETSET recovery_op_priority 5 
+  ceph osd pool get $TEST_POOL_GETSET recovery_op_priority | grep 'recovery_op_priority: 5'
+  ceph osd pool set $TEST_POOL_GETSET recovery_op_priority 0
+  ceph osd pool get $TEST_POOL_GETSET recovery_op_priority | expect_false grep '.'
+
+  ceph osd pool get $TEST_POOL_GETSET scrub_priority | expect_false grep '.'
+  ceph osd pool set $TEST_POOL_GETSET scrub_priority 5 
+  ceph osd pool get $TEST_POOL_GETSET scrub_priority | grep 'scrub_priority: 5'
+  ceph osd pool set $TEST_POOL_GETSET scrub_priority 0
+  ceph osd pool get $TEST_POOL_GETSET scrub_priority | expect_false grep '.'
+
+  ceph osd pool set $TEST_POOL_GETSET nopgchange 1
+  expect_false ceph osd pool set $TEST_POOL_GETSET pg_num 10
+  expect_false ceph osd pool set $TEST_POOL_GETSET pgp_num 10
+  ceph osd pool set $TEST_POOL_GETSET nopgchange 0
+  ceph osd pool set $TEST_POOL_GETSET pg_num 10
+  wait_for_clean
+  ceph osd pool set $TEST_POOL_GETSET pgp_num 10
+  expect_false ceph osd pool set $TEST_POOL_GETSET pg_num 0
+  expect_false ceph osd pool set $TEST_POOL_GETSET pgp_num 0
+
+  old_pgs=$(ceph osd pool get $TEST_POOL_GETSET pg_num | sed -e 's/pg_num: //')
+  new_pgs=$(($old_pgs + $(ceph osd stat --format json | jq '.num_osds') * 32))
+  ceph osd pool set $TEST_POOL_GETSET pg_num $new_pgs
+  ceph osd pool set $TEST_POOL_GETSET pgp_num $new_pgs
+  wait_for_clean
+
+  ceph osd pool set $TEST_POOL_GETSET nosizechange 1
+  expect_false ceph osd pool set $TEST_POOL_GETSET size 2
+  expect_false ceph osd pool set $TEST_POOL_GETSET min_size 2
+  ceph osd pool set $TEST_POOL_GETSET nosizechange 0
+  ceph osd pool set $TEST_POOL_GETSET size 2
+  wait_for_clean
+  ceph osd pool set $TEST_POOL_GETSET min_size 2
+  
+  expect_false ceph osd pool set $TEST_POOL_GETSET hashpspool 0
+  ceph osd pool set $TEST_POOL_GETSET hashpspool 0 --yes-i-really-mean-it
+  
+  expect_false ceph osd pool set $TEST_POOL_GETSET hashpspool 1
+  ceph osd pool set $TEST_POOL_GETSET hashpspool 1 --yes-i-really-mean-it
+
+  ceph osd pool get rbd crush_rule | grep 'crush_rule: '
+
+  ceph osd pool get $TEST_POOL_GETSET compression_mode | expect_false grep '.'
+  ceph osd pool set $TEST_POOL_GETSET compression_mode aggressive
+  ceph osd pool get $TEST_POOL_GETSET compression_mode | grep 'aggressive'
+  ceph osd pool set $TEST_POOL_GETSET compression_mode unset
+  ceph osd pool get $TEST_POOL_GETSET compression_mode | expect_false grep '.'
+
+  ceph osd pool get $TEST_POOL_GETSET compression_algorithm | expect_false grep '.'
+  ceph osd pool set $TEST_POOL_GETSET compression_algorithm zlib
+  ceph osd pool get $TEST_POOL_GETSET compression_algorithm | grep 'zlib'
+  ceph osd pool set $TEST_POOL_GETSET compression_algorithm unset
+  ceph osd pool get $TEST_POOL_GETSET compression_algorithm | expect_false grep '.'
+
+  ceph osd pool get $TEST_POOL_GETSET compression_required_ratio | expect_false grep '.'
+  expect_false ceph osd pool set $TEST_POOL_GETSET compression_required_ratio 1.1
+  expect_false ceph osd pool set $TEST_POOL_GETSET compression_required_ratio -.2
+  ceph osd pool set $TEST_POOL_GETSET compression_required_ratio .2
+  ceph osd pool get $TEST_POOL_GETSET compression_required_ratio | grep '.2'
+  ceph osd pool set $TEST_POOL_GETSET compression_required_ratio 0
+  ceph osd pool get $TEST_POOL_GETSET compression_required_ratio | expect_false grep '.'
+
+  ceph osd pool get $TEST_POOL_GETSET csum_type | expect_false grep '.'
+  ceph osd pool set $TEST_POOL_GETSET csum_type crc32c
+  ceph osd pool get $TEST_POOL_GETSET csum_type | grep 'crc32c'
+  ceph osd pool set $TEST_POOL_GETSET csum_type unset
+  ceph osd pool get $TEST_POOL_GETSET csum_type | expect_false grep '.'
+
+  for size in compression_max_blob_size compression_min_blob_size csum_max_block csum_min_block; do
+      ceph osd pool get $TEST_POOL_GETSET $size | expect_false grep '.'
+      ceph osd pool set $TEST_POOL_GETSET $size 100
+      ceph osd pool get $TEST_POOL_GETSET $size | grep '100'
+      ceph osd pool set $TEST_POOL_GETSET $size 0
+      ceph osd pool get $TEST_POOL_GETSET $size | expect_false grep '.'
+  done
+
+  ceph osd pool set $TEST_POOL_GETSET nodelete 1
+  expect_false ceph osd pool delete $TEST_POOL_GETSET $TEST_POOL_GETSET --yes-i-really-really-mean-it
+  ceph osd pool set $TEST_POOL_GETSET nodelete 0
+  ceph osd pool delete $TEST_POOL_GETSET $TEST_POOL_GETSET --yes-i-really-really-mean-it
+
+}
+
+function test_mon_osd_tiered_pool_set()
+{
+  # this is really a tier pool
+  ceph osd pool create real-tier 2
+  ceph osd tier add rbd real-tier
+
+  # expect us to be unable to set negative values for hit_set_*
+  for o in hit_set_period hit_set_count hit_set_fpp; do
+    expect_false ceph osd pool set real_tier $o -1
+  done
+
+  # and hit_set_fpp should be in range 0..1
+  expect_false ceph osd pool set real_tier hit_set_fpp 2
+
+  ceph osd pool set real-tier hit_set_type explicit_hash
+  ceph osd pool get real-tier hit_set_type | grep "hit_set_type: explicit_hash"
+  ceph osd pool set real-tier hit_set_type explicit_object
+  ceph osd pool get real-tier hit_set_type | grep "hit_set_type: explicit_object"
+  ceph osd pool set real-tier hit_set_type bloom
+  ceph osd pool get real-tier hit_set_type | grep "hit_set_type: bloom"
+  expect_false ceph osd pool set real-tier hit_set_type i_dont_exist
+  ceph osd pool set real-tier hit_set_period 123
+  ceph osd pool get real-tier hit_set_period | grep "hit_set_period: 123"
+  ceph osd pool set real-tier hit_set_count 12
+  ceph osd pool get real-tier hit_set_count | grep "hit_set_count: 12"
+  ceph osd pool set real-tier hit_set_fpp .01
+  ceph osd pool get real-tier hit_set_fpp | grep "hit_set_fpp: 0.01"
+
+  ceph osd pool set real-tier target_max_objects 123
+  ceph osd pool get real-tier target_max_objects | \
+    grep 'target_max_objects:[ \t]\+123'
+  ceph osd pool set real-tier target_max_bytes 123456
+  ceph osd pool get real-tier target_max_bytes | \
+    grep 'target_max_bytes:[ \t]\+123456'
+  ceph osd pool set real-tier cache_target_dirty_ratio .123
+  ceph osd pool get real-tier cache_target_dirty_ratio | \
+    grep 'cache_target_dirty_ratio:[ \t]\+0.123'
+  expect_false ceph osd pool set real-tier cache_target_dirty_ratio -.2
+  expect_false ceph osd pool set real-tier cache_target_dirty_ratio 1.1
+  ceph osd pool set real-tier cache_target_dirty_high_ratio .123
+  ceph osd pool get real-tier cache_target_dirty_high_ratio | \
+    grep 'cache_target_dirty_high_ratio:[ \t]\+0.123'
+  expect_false ceph osd pool set real-tier cache_target_dirty_high_ratio -.2
+  expect_false ceph osd pool set real-tier cache_target_dirty_high_ratio 1.1
+  ceph osd pool set real-tier cache_target_full_ratio .123
+  ceph osd pool get real-tier cache_target_full_ratio | \
+    grep 'cache_target_full_ratio:[ \t]\+0.123'
+  ceph osd dump -f json-pretty | grep '"cache_target_full_ratio_micro": 123000'
+  ceph osd pool set real-tier cache_target_full_ratio 1.0
+  ceph osd pool set real-tier cache_target_full_ratio 0
+  expect_false ceph osd pool set real-tier cache_target_full_ratio 1.1
+  ceph osd pool set real-tier cache_min_flush_age 123
+  ceph osd pool get real-tier cache_min_flush_age | \
+    grep 'cache_min_flush_age:[ \t]\+123'
+  ceph osd pool set real-tier cache_min_evict_age 234
+  ceph osd pool get real-tier cache_min_evict_age | \
+    grep 'cache_min_evict_age:[ \t]\+234'
+
+  # this is not a tier pool
+  ceph osd pool create fake-tier 2
+  ceph osd pool application enable fake-tier rados
+  wait_for_clean
+
+  expect_false ceph osd pool set fake-tier hit_set_type explicit_hash
+  expect_false ceph osd pool get fake-tier hit_set_type
+  expect_false ceph osd pool set fake-tier hit_set_type explicit_object
+  expect_false ceph osd pool get fake-tier hit_set_type
+  expect_false ceph osd pool set fake-tier hit_set_type bloom
+  expect_false ceph osd pool get fake-tier hit_set_type
+  expect_false ceph osd pool set fake-tier hit_set_type i_dont_exist
+  expect_false ceph osd pool set fake-tier hit_set_period 123
+  expect_false ceph osd pool get fake-tier hit_set_period
+  expect_false ceph osd pool set fake-tier hit_set_count 12
+  expect_false ceph osd pool get fake-tier hit_set_count
+  expect_false ceph osd pool set fake-tier hit_set_fpp .01
+  expect_false ceph osd pool get fake-tier hit_set_fpp
+
+  expect_false ceph osd pool set fake-tier target_max_objects 123
+  expect_false ceph osd pool get fake-tier target_max_objects
+  expect_false ceph osd pool set fake-tier target_max_bytes 123456
+  expect_false ceph osd pool get fake-tier target_max_bytes
+  expect_false ceph osd pool set fake-tier cache_target_dirty_ratio .123
+  expect_false ceph osd pool get fake-tier cache_target_dirty_ratio
+  expect_false ceph osd pool set fake-tier cache_target_dirty_ratio -.2
+  expect_false ceph osd pool set fake-tier cache_target_dirty_ratio 1.1
+  expect_false ceph osd pool set fake-tier cache_target_dirty_high_ratio .123
+  expect_false ceph osd pool get fake-tier cache_target_dirty_high_ratio
+  expect_false ceph osd pool set fake-tier cache_target_dirty_high_ratio -.2
+  expect_false ceph osd pool set fake-tier cache_target_dirty_high_ratio 1.1
+  expect_false ceph osd pool set fake-tier cache_target_full_ratio .123
+  expect_false ceph osd pool get fake-tier cache_target_full_ratio
+  expect_false ceph osd pool set fake-tier cache_target_full_ratio 1.0
+  expect_false ceph osd pool set fake-tier cache_target_full_ratio 0
+  expect_false ceph osd pool set fake-tier cache_target_full_ratio 1.1
+  expect_false ceph osd pool set fake-tier cache_min_flush_age 123
+  expect_false ceph osd pool get fake-tier cache_min_flush_age
+  expect_false ceph osd pool set fake-tier cache_min_evict_age 234
+  expect_false ceph osd pool get fake-tier cache_min_evict_age
+
+  ceph osd tier remove rbd real-tier
+  ceph osd pool delete real-tier real-tier --yes-i-really-really-mean-it
+  ceph osd pool delete fake-tier fake-tier --yes-i-really-really-mean-it
+}
+
+function test_mon_osd_erasure_code()
+{
+
+  ceph osd erasure-code-profile set fooprofile a=b c=d
+  ceph osd erasure-code-profile set fooprofile a=b c=d
+  expect_false ceph osd erasure-code-profile set fooprofile a=b c=d e=f
+  ceph osd erasure-code-profile set fooprofile a=b c=d e=f --force
+  ceph osd erasure-code-profile set fooprofile a=b c=d e=f
+  expect_false ceph osd erasure-code-profile set fooprofile a=b c=d e=f g=h
+  # make sure ruleset-foo doesn't work anymore
+  expect_false ceph osd erasure-code-profile set barprofile ruleset-failure-domain=host
+  ceph osd erasure-code-profile set barprofile crush-failure-domain=host
+  # clean up
+  ceph osd erasure-code-profile rm fooprofile
+  ceph osd erasure-code-profile rm barprofile
+
+  # try weird k and m values
+  expect_false ceph osd erasure-code-profile set badk k=1 m=1
+  expect_false ceph osd erasure-code-profile set badk k=1 m=2
+  expect_false ceph osd erasure-code-profile set badk k=0 m=2
+  expect_false ceph osd erasure-code-profile set badk k=-1 m=2
+  expect_false ceph osd erasure-code-profile set badm k=2 m=0
+  expect_false ceph osd erasure-code-profile set badm k=2 m=-1
+  ceph osd erasure-code-profile set good k=2 m=1
+  ceph osd erasure-code-profile rm good
+}
+
+function test_mon_osd_misc()
+{
+  set +e
+
+  # expect error about missing 'pool' argument
+  ceph osd map 2>$TMPFILE; check_response 'pool' $? 22
+
+  # expect error about unused argument foo
+  ceph osd ls foo 2>$TMPFILE; check_response 'unused' $? 22 
+
+  # expect "not in range" for invalid overload percentage
+  ceph osd reweight-by-utilization 80 2>$TMPFILE; check_response 'higher than 100' $? 22
+
+  set -e
+
+  ceph osd reweight-by-utilization 110
+  ceph osd reweight-by-utilization 110 .5
+  expect_false ceph osd reweight-by-utilization 110 0
+  expect_false ceph osd reweight-by-utilization 110 -0.1
+  ceph osd test-reweight-by-utilization 110 .5 --no-increasing
+  ceph osd test-reweight-by-utilization 110 .5 4 --no-increasing
+  expect_false ceph osd test-reweight-by-utilization 110 .5 0 --no-increasing
+  expect_false ceph osd test-reweight-by-utilization 110 .5 -10 --no-increasing
+  ceph osd reweight-by-pg 110
+  ceph osd test-reweight-by-pg 110 .5
+  ceph osd reweight-by-pg 110 rbd
+  ceph osd reweight-by-pg 110 .5 rbd
+  expect_false ceph osd reweight-by-pg 110 boguspoolasdfasdfasdf
+}
+
+function test_mon_heap_profiler()
+{
+  do_test=1
+  set +e
+  # expect 'heap' commands to be correctly parsed
+  ceph heap stats 2>$TMPFILE
+  if [[ $? -eq 22 && `grep 'tcmalloc not enabled' $TMPFILE` ]]; then
+    echo "tcmalloc not enabled; skip heap profiler test"
+    do_test=0
+  fi
+  set -e
+
+  [[ $do_test -eq 0 ]] && return 0
+
+  ceph heap start_profiler
+  ceph heap dump
+  ceph heap stop_profiler
+  ceph heap release
+}
+
+function test_admin_heap_profiler()
+{
+  do_test=1
+  set +e
+  # expect 'heap' commands to be correctly parsed
+  ceph heap stats 2>$TMPFILE
+  if [[ $? -eq 22 && `grep 'tcmalloc not enabled' $TMPFILE` ]]; then
+    echo "tcmalloc not enabled; skip heap profiler test"
+    do_test=0
+  fi
+  set -e
+
+  [[ $do_test -eq 0 ]] && return 0
+
+  local admin_socket=$(get_admin_socket osd.0)
+
+  $SUDO ceph --admin-daemon $admin_socket heap start_profiler
+  $SUDO ceph --admin-daemon $admin_socket heap dump
+  $SUDO ceph --admin-daemon $admin_socket heap stop_profiler
+  $SUDO ceph --admin-daemon $admin_socket heap release
+}
+
+function test_osd_bench()
+{
+  # test osd bench limits
+  # As we should not rely on defaults (as they may change over time),
+  # lets inject some values and perform some simple tests
+  # max iops: 10              # 100 IOPS
+  # max throughput: 10485760  # 10MB/s
+  # max block size: 2097152   # 2MB
+  # duration: 10              # 10 seconds
+
+  local args="\
+    --osd-bench-duration 10 \
+    --osd-bench-max-block-size 2097152 \
+    --osd-bench-large-size-max-throughput 10485760 \
+    --osd-bench-small-size-max-iops 10"
+  ceph tell osd.0 injectargs ${args## }
+
+  # anything with a bs larger than 2097152  must fail
+  expect_false ceph tell osd.0 bench 1 2097153
+  # but using 'osd_bench_max_bs' must succeed
+  ceph tell osd.0 bench 1 2097152
+
+  # we assume 1MB as a large bs; anything lower is a small bs
+  # for a 4096 bytes bs, for 10 seconds, we are limited by IOPS
+  # max count: 409600 (bytes)
+
+  # more than max count must not be allowed
+  expect_false ceph tell osd.0 bench 409601 4096
+  # but 409600 must be succeed
+  ceph tell osd.0 bench 409600 4096
+
+  # for a large bs, we are limited by throughput.
+  # for a 2MB block size for 10 seconds, assuming 10MB/s throughput,
+  # the max count will be (10MB * 10s) = 100MB
+  # max count: 104857600 (bytes)
+
+  # more than max count must not be allowed
+  expect_false ceph tell osd.0 bench 104857601 2097152
+  # up to max count must be allowed
+  ceph tell osd.0 bench 104857600 2097152
+}
+
+function test_osd_negative_filestore_merge_threshold()
+{
+  $SUDO ceph daemon osd.0 config set filestore_merge_threshold -1
+  expect_config_value "osd.0" "filestore_merge_threshold" -1
+}
+
+function test_mon_tell()
+{
+  ceph tell mon.a version
+  ceph tell mon.b version
+  expect_false ceph tell mon.foo version
+
+  sleep 1
+
+  ceph_watch_start debug audit
+  ceph tell mon.a version
+  ceph_watch_wait 'mon.a \[DBG\] from.*cmd=\[{"prefix": "version"}\]: dispatch'
+
+  ceph_watch_start debug audit
+  ceph tell mon.b version
+  ceph_watch_wait 'mon.b \[DBG\] from.*cmd=\[{"prefix": "version"}\]: dispatch'
+}
+
+function test_mon_ping()
+{
+  ceph ping mon.a
+  ceph ping mon.b
+  expect_false ceph ping mon.foo
+
+  ceph ping mon.\*
+}
+
+function test_mon_deprecated_commands()
+{
+  # current DEPRECATED commands are:
+  #  ceph compact
+  #  ceph scrub
+  #  ceph sync force
+  #
+  # Testing should be accomplished by setting
+  # 'mon_debug_deprecated_as_obsolete = true' and expecting ENOTSUP for
+  # each one of these commands.
+
+  ceph tell mon.a injectargs '--mon-debug-deprecated-as-obsolete'
+  expect_false ceph tell mon.a compact 2> $TMPFILE
+  check_response "\(EOPNOTSUPP\|ENOTSUP\): command is obsolete"
+
+  expect_false ceph tell mon.a scrub 2> $TMPFILE
+  check_response "\(EOPNOTSUPP\|ENOTSUP\): command is obsolete"
+
+  expect_false ceph tell mon.a sync force 2> $TMPFILE
+  check_response "\(EOPNOTSUPP\|ENOTSUP\): command is obsolete"
+
+  ceph tell mon.a injectargs '--no-mon-debug-deprecated-as-obsolete'
+}
+
+function test_mon_cephdf_commands()
+{
+  # ceph df detail:
+  # pool section:
+  # RAW USED The near raw used per pool in raw total
+
+  ceph osd pool create cephdf_for_test 1 1 replicated
+  ceph osd pool application enable cephdf_for_test rados
+  ceph osd pool set cephdf_for_test size 2
+
+  dd if=/dev/zero of=./cephdf_for_test bs=4k count=1
+  rados put cephdf_for_test cephdf_for_test -p cephdf_for_test
+
+  #wait for update
+  for i in `seq 1 10`; do
+    rados -p cephdf_for_test ls - | grep -q cephdf_for_test && break
+    sleep 1
+  done
+  # "rados ls" goes straight to osd, but "ceph df" is served by mon. so we need
+  # to sync mon with osd
+  flush_pg_stats
+  local jq_filter='.pools | .[] | select(.name == "cephdf_for_test") | .stats'
+  stored=`ceph df detail --format=json | jq "$jq_filter.stored * 2"`
+  stored_raw=`ceph df detail --format=json | jq "$jq_filter.stored_raw"`
+
+  ceph osd pool delete cephdf_for_test cephdf_for_test --yes-i-really-really-mean-it
+  rm ./cephdf_for_test
+
+  expect_false test $stored != $stored_raw
+}
+
+function test_mon_pool_application()
+{
+  ceph osd pool create app_for_test 10
+
+  ceph osd pool application enable app_for_test rbd
+  expect_false ceph osd pool application enable app_for_test rgw
+  ceph osd pool application enable app_for_test rgw --yes-i-really-mean-it
+  ceph osd pool ls detail | grep "application rbd,rgw"
+  ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{},"rgw":{}}'
+
+  expect_false ceph osd pool application set app_for_test cephfs key value
+  ceph osd pool application set app_for_test rbd key1 value1
+  ceph osd pool application set app_for_test rbd key2 value2
+  ceph osd pool application set app_for_test rgw key1 value1
+  ceph osd pool application get app_for_test rbd key1 | grep 'value1'
+  ceph osd pool application get app_for_test rbd key2 | grep 'value2'
+  ceph osd pool application get app_for_test rgw key1 | grep 'value1'
+
+  ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{"key1":"value1","key2":"value2"},"rgw":{"key1":"value1"}}'
+
+  ceph osd pool application rm app_for_test rgw key1
+  ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{"key1":"value1","key2":"value2"},"rgw":{}}'
+  ceph osd pool application rm app_for_test rbd key2
+  ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{"key1":"value1"},"rgw":{}}'
+  ceph osd pool application rm app_for_test rbd key1
+  ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{},"rgw":{}}'
+  ceph osd pool application rm app_for_test rbd key1 # should be idempotent
+
+  expect_false ceph osd pool application disable app_for_test rgw
+  ceph osd pool application disable app_for_test rgw --yes-i-really-mean-it
+  ceph osd pool application disable app_for_test rgw --yes-i-really-mean-it # should be idempotent
+  ceph osd pool ls detail | grep "application rbd"
+  ceph osd pool ls detail --format=json | grep '"application_metadata":{"rbd":{}}'
+
+  ceph osd pool application disable app_for_test rgw --yes-i-really-mean-it
+  ceph osd pool ls detail | grep -v "application "
+  ceph osd pool ls detail --format=json | grep '"application_metadata":{}'
+
+  ceph osd pool rm app_for_test app_for_test --yes-i-really-really-mean-it
+}
+
+function test_mon_tell_help_command()
+{
+  ceph tell mon.a help
+
+  # wrong target
+  expect_false ceph tell mon.zzz help
+}
+
+function test_mon_stdin_stdout()
+{
+  echo foo | ceph config-key set test_key -i -
+  ceph config-key get test_key -o - | grep -c foo | grep -q 1
+}
+
+function test_osd_tell_help_command()
+{
+  ceph tell osd.1 help
+  expect_false ceph tell osd.100 help
+}
+
+function test_osd_compact()
+{
+  ceph tell osd.1 compact
+  $SUDO ceph daemon osd.1 compact
+}
+
+function test_mds_tell_help_command()
+{
+  local FS_NAME=cephfs
+  if ! mds_exists ; then
+      echo "Skipping test, no MDS found"
+      return
+  fi
+
+  remove_all_fs
+  ceph osd pool create fs_data 10
+  ceph osd pool create fs_metadata 10
+  ceph fs new $FS_NAME fs_metadata fs_data
+  wait_mds_active $FS_NAME
+
+
+  ceph tell mds.a help
+  expect_false ceph tell mds.z help
+
+  remove_all_fs
+  ceph osd pool delete fs_data fs_data --yes-i-really-really-mean-it
+  ceph osd pool delete fs_metadata fs_metadata --yes-i-really-really-mean-it
+}
+
+function test_mgr_tell()
+{
+  ceph tell mgr help
+  #ceph tell mgr fs status   # see http://tracker.ceph.com/issues/20761
+  ceph tell mgr osd status
+}
+
+function test_mgr_devices()
+{
+  ceph device ls
+  expect_false ceph device info doesnotexist
+  expect_false ceph device get-health-metrics doesnotexist
+}
+
+function test_per_pool_scrub_status()
+{
+  ceph osd pool create noscrub_pool 12
+  ceph osd pool create noscrub_pool2 12
+  ceph -s | expect_false grep -q "Some pool(s) have the.*scrub.* flag(s) set"
+  ceph -s --format json | \
+    jq .health.checks.POOL_SCRUB_FLAGS.summary.message | \
+    expect_false grep -q "Some pool(s) have the.*scrub.* flag(s) set"
+  ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail |
+    expect_false grep -q "Pool .* has .*scrub.* flag"
+  ceph health detail | jq .health.checks.POOL_SCRUB_FLAGS.detail | \
+    expect_false grep -q "Pool .* has .*scrub.* flag"
+
+  ceph osd pool set noscrub_pool noscrub 1
+  ceph -s | expect_true grep -q "Some pool(s) have the noscrub flag(s) set"
+  ceph -s --format json | \
+    jq .health.checks.POOL_SCRUB_FLAGS.summary.message | \
+    expect_true grep -q "Some pool(s) have the noscrub flag(s) set"
+  ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail | \
+    expect_true grep -q "Pool noscrub_pool has noscrub flag"
+  ceph health detail | expect_true grep -q "Pool noscrub_pool has noscrub flag"
+
+  ceph osd pool set noscrub_pool nodeep-scrub 1
+  ceph osd pool set noscrub_pool2 nodeep-scrub 1
+  ceph -s | expect_true grep -q "Some pool(s) have the noscrub, nodeep-scrub flag(s) set"
+  ceph -s --format json | \
+    jq .health.checks.POOL_SCRUB_FLAGS.summary.message | \
+    expect_true grep -q "Some pool(s) have the noscrub, nodeep-scrub flag(s) set"
+  ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail | \
+    expect_true grep -q "Pool noscrub_pool has noscrub flag"
+  ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail | \
+    expect_true grep -q "Pool noscrub_pool has nodeep-scrub flag"
+  ceph report | jq .health.checks.POOL_SCRUB_FLAGS.detail | \
+    expect_true grep -q "Pool noscrub_pool2 has nodeep-scrub flag"
+  ceph health detail | expect_true grep -q "Pool noscrub_pool has noscrub flag"
+  ceph health detail | expect_true grep -q "Pool noscrub_pool has nodeep-scrub flag"
+  ceph health detail | expect_true grep -q "Pool noscrub_pool2 has nodeep-scrub flag"
+
+  ceph osd pool rm noscrub_pool noscrub_pool --yes-i-really-really-mean-it
+  ceph osd pool rm noscrub_pool2 noscrub_pool2 --yes-i-really-really-mean-it
+}
+
+#
+# New tests should be added to the TESTS array below
+#
+# Individual tests may be run using the '-t <testname>' argument
+# The user can specify '-t <testname>' as many times as she wants
+#
+# Tests will be run in order presented in the TESTS array, or in
+# the order specified by the '-t <testname>' options.
+#
+# '-l' will list all the available test names
+# '-h' will show usage
+#
+# The test maintains backward compatibility: not specifying arguments
+# will run all tests following the order they appear in the TESTS array.
+#
+
+set +x
+MON_TESTS+=" mon_injectargs"
+MON_TESTS+=" mon_injectargs_SI"
+for i in `seq 9`; do
+    MON_TESTS+=" tiering_$i";
+done
+MON_TESTS+=" auth"
+MON_TESTS+=" auth_profiles"
+MON_TESTS+=" mon_misc"
+MON_TESTS+=" mon_mon"
+MON_TESTS+=" mon_osd"
+MON_TESTS+=" mon_config_key"
+MON_TESTS+=" mon_crush"
+MON_TESTS+=" mon_osd_create_destroy"
+MON_TESTS+=" mon_osd_pool"
+MON_TESTS+=" mon_osd_pool_quota"
+MON_TESTS+=" mon_pg"
+MON_TESTS+=" mon_osd_pool_set"
+MON_TESTS+=" mon_osd_tiered_pool_set"
+MON_TESTS+=" mon_osd_erasure_code"
+MON_TESTS+=" mon_osd_misc"
+MON_TESTS+=" mon_heap_profiler"
+MON_TESTS+=" mon_tell"
+MON_TESTS+=" mon_ping"
+MON_TESTS+=" mon_deprecated_commands"
+MON_TESTS+=" mon_caps"
+MON_TESTS+=" mon_cephdf_commands"
+MON_TESTS+=" mon_tell_help_command"
+MON_TESTS+=" mon_stdin_stdout"
+
+OSD_TESTS+=" osd_bench"
+OSD_TESTS+=" osd_negative_filestore_merge_threshold"
+OSD_TESTS+=" tiering_agent"
+OSD_TESTS+=" admin_heap_profiler"
+OSD_TESTS+=" osd_tell_help_command"
+OSD_TESTS+=" osd_compact"
+OSD_TESTS+=" per_pool_scrub_status"
+
+MDS_TESTS+=" mds_tell"
+MDS_TESTS+=" mon_mds"
+MDS_TESTS+=" mon_mds_metadata"
+MDS_TESTS+=" mds_tell_help_command"
+
+MGR_TESTS+=" mgr_tell"
+MGR_TESTS+=" mgr_devices"
+
+TESTS+=$MON_TESTS
+TESTS+=$OSD_TESTS
+TESTS+=$MDS_TESTS
+TESTS+=$MGR_TESTS
+
+#
+# "main" follows
+#
+
+function list_tests()
+{
+  echo "AVAILABLE TESTS"
+  for i in $TESTS; do
+    echo "  $i"
+  done
+}
+
+function usage()
+{
+  echo "usage: $0 [-h|-l|-t <testname> [-t <testname>...]]"
+}
+
+tests_to_run=()
+
+sanity_check=true
+
+while [[ $# -gt 0 ]]; do
+  opt=$1
+
+  case "$opt" in
+    "-l" )
+      do_list=1
+      ;;
+    "--asok-does-not-need-root" )
+      SUDO=""
+      ;;
+    "--no-sanity-check" )
+      sanity_check=false
+      ;;
+    "--test-mon" )
+      tests_to_run+="$MON_TESTS"
+      ;;
+    "--test-osd" )
+      tests_to_run+="$OSD_TESTS"
+      ;;
+    "--test-mds" )
+      tests_to_run+="$MDS_TESTS"
+      ;;
+    "--test-mgr" )
+      tests_to_run+="$MGR_TESTS"
+      ;;
+    "-t" )
+      shift
+      if [[ -z "$1" ]]; then
+        echo "missing argument to '-t'"
+        usage ;
+        exit 1
+      fi
+      tests_to_run+=" $1"
+      ;;
+    "-h" )
+      usage ;
+      exit 0
+      ;;
+  esac
+  shift
+done
+
+if [[ $do_list -eq 1 ]]; then
+  list_tests ;
+  exit 0
+fi
+
+ceph osd pool create rbd 10
+
+if test -z "$tests_to_run" ; then
+  tests_to_run="$TESTS"
+fi
+
+if $sanity_check ; then
+    wait_no_osd_down
+fi
+for i in $tests_to_run; do
+  if $sanity_check ; then
+      check_no_osd_down
+  fi
+  set -x
+  test_${i}
+  set +x
+done
+if $sanity_check ; then
+    check_no_osd_down
+fi
+
+set -x
+
+echo OK
diff --git a/qa/workunits/cephtool/test_daemon.sh b/qa/workunits/cephtool/test_daemon.sh
new file mode 100755
index 00000000..08ae937c
--- /dev/null
+++ b/qa/workunits/cephtool/test_daemon.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -ex
+
+expect_false()
+{
+	set -x
+	if "$@"; then return 1; else return 0; fi
+}
+
+echo note: assuming mon.a is on the current host
+
+# can set to 'sudo ./ceph' to execute tests from current dir for development
+CEPH=${CEPH:-'sudo ceph'}
+
+${CEPH} daemon mon.a version | grep version
+
+# get debug_ms setting and strip it, painfully for reuse
+old_ms=$(${CEPH} daemon mon.a config get debug_ms | \
+	grep debug_ms | sed -e 's/.*: //' -e 's/["\}\\]//g')
+${CEPH} daemon mon.a config set debug_ms 13
+new_ms=$(${CEPH} daemon mon.a config get debug_ms | \
+	grep debug_ms | sed -e 's/.*: //' -e 's/["\}\\]//g')
+[ "$new_ms" = "13/13" ]
+${CEPH} daemon mon.a config set debug_ms $old_ms
+new_ms=$(${CEPH} daemon mon.a config get debug_ms | \
+	grep debug_ms | sed -e 's/.*: //' -e 's/["\}\\]//g')
+[ "$new_ms" = "$old_ms" ]
+
+# unregistered/non-existent command
+expect_false ${CEPH} daemon mon.a bogus_command_blah foo
+
+set +e
+OUTPUT=$(${CEPH} -c /not/a/ceph.conf daemon mon.a help 2>&1)
+# look for EINVAL
+if [ $? != 22 ] ; then exit 1; fi
+if ! echo "$OUTPUT" | grep -q '.*open.*/not/a/ceph.conf'; then 
+	echo "didn't find expected error in bad conf search"
+	exit 1
+fi
+set -e
+
+echo OK
diff --git a/qa/workunits/cephtool/test_kvstore_tool.sh b/qa/workunits/cephtool/test_kvstore_tool.sh
new file mode 100755
index 00000000..b7953dd2
--- /dev/null
+++ b/qa/workunits/cephtool/test_kvstore_tool.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -x
+
+source $(dirname $0)/../../standalone/ceph-helpers.sh
+
+set -e
+set -o functrace
+PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
+SUDO=${SUDO:-sudo}
+export CEPH_DEV=1
+
+echo note: test ceph_kvstore_tool with bluestore
+
+expect_false()
+{
+    set -x
+    if "$@"; then return 1; else return 0; fi
+}
+
+TEMP_DIR=$(mktemp -d ./cephtool.XXX)
+trap "rm -fr $TEMP_DIR" 0
+
+TEMP_FILE=$(mktemp $TEMP_DIR/test_invalid.XXX)
+
+function test_ceph_kvstore_tool()
+{
+  # create a data directory
+  ceph-objectstore-tool --data-path ${TEMP_DIR} --op mkfs --no-mon-config
+
+  # list
+  origin_kv_nums=`ceph-kvstore-tool  bluestore-kv ${TEMP_DIR} list 2>/dev/null | wc -l`
+  
+  # exists
+  prefix=`ceph-kvstore-tool bluestore-kv ${TEMP_DIR} list 2>/dev/null | head -n 1 | awk '{print $1}'`
+  ceph-kvstore-tool bluestore-kv ${TEMP_DIR} exists ${prefix}
+  expect_false ceph-kvstore-tool bluestore-kv ${TEMP_DIR} exists ${prefix}notexist
+
+  # list-crc
+  ceph-kvstore-tool  bluestore-kv ${TEMP_DIR} list-crc
+  ceph-kvstore-tool  bluestore-kv ${TEMP_DIR} list-crc ${prefix}
+
+  # list with prefix
+  ceph-kvstore-tool  bluestore-kv ${TEMP_DIR} list ${prefix}
+
+  # set
+  echo "helloworld" >> ${TEMP_FILE}
+  ceph-kvstore-tool bluestore-kv ${TEMP_DIR} set TESTPREFIX TESTKEY in ${TEMP_FILE}
+  ceph-kvstore-tool bluestore-kv ${TEMP_DIR} exists TESTPREFIX TESTKEY
+
+  # get 
+  ceph-kvstore-tool bluestore-kv ${TEMP_DIR} get TESTPREFIX TESTKEY out ${TEMP_FILE}.bak
+  diff ${TEMP_FILE} ${TEMP_FILE}.bak
+
+  # rm 
+  ceph-kvstore-tool bluestore-kv ${TEMP_DIR} rm TESTPREFIX TESTKEY
+  expect_false ceph-kvstore-tool bluestore-kv ${TEMP_DIR} exists TESTPREFIX TESTKEY
+
+  # compact
+  ceph-kvstore-tool bluestore-kv ${TEMP_DIR} compact
+
+  # destructive-repair 
+  ceph-kvstore-tool bluestore-kv ${TEMP_DIR} destructive-repair 
+
+  current_kv_nums=`ceph-kvstore-tool  bluestore-kv ${TEMP_DIR} list 2>/dev/null | wc -l`
+  test ${origin_kv_nums} -eq ${current_kv_nums}
+} 
+
+test_ceph_kvstore_tool
+
+echo OK
diff --git a/qa/workunits/cls/test_cls_hello.sh b/qa/workunits/cls/test_cls_hello.sh
new file mode 100755
index 00000000..0a2e0962
--- /dev/null
+++ b/qa/workunits/cls/test_cls_hello.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -e
+
+ceph_test_cls_hello
+
+exit 0
diff --git a/qa/workunits/cls/test_cls_journal.sh b/qa/workunits/cls/test_cls_journal.sh
new file mode 100755
index 00000000..9aa7450a
--- /dev/null
+++ b/qa/workunits/cls/test_cls_journal.sh
@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+GTEST_FILTER=${CLS_JOURNAL_GTEST_FILTER:-*}
+ceph_test_cls_journal --gtest_filter=${GTEST_FILTER}
+
+exit 0
diff --git a/qa/workunits/cls/test_cls_lock.sh b/qa/workunits/cls/test_cls_lock.sh
new file mode 100755
index 00000000..c1452705
--- /dev/null
+++ b/qa/workunits/cls/test_cls_lock.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -e
+
+ceph_test_cls_lock
+
+exit 0
diff --git a/qa/workunits/cls/test_cls_log.sh b/qa/workunits/cls/test_cls_log.sh
new file mode 100755
index 00000000..523f985e
--- /dev/null
+++ b/qa/workunits/cls/test_cls_log.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -e
+
+ceph_test_cls_log
+
+exit 0
diff --git a/qa/workunits/cls/test_cls_numops.sh b/qa/workunits/cls/test_cls_numops.sh
new file mode 100755
index 00000000..dcbafcab
--- /dev/null
+++ b/qa/workunits/cls/test_cls_numops.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -e
+
+ceph_test_cls_numops
+
+exit 0
diff --git a/qa/workunits/cls/test_cls_rbd.sh b/qa/workunits/cls/test_cls_rbd.sh
new file mode 100755
index 00000000..fd4bec0f
--- /dev/null
+++ b/qa/workunits/cls/test_cls_rbd.sh
@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+GTEST_FILTER=${CLS_RBD_GTEST_FILTER:-*}
+ceph_test_cls_rbd --gtest_filter=${GTEST_FILTER}
+
+exit 0
diff --git a/qa/workunits/cls/test_cls_refcount.sh b/qa/workunits/cls/test_cls_refcount.sh
new file mode 100755
index 00000000..d722f5ad
--- /dev/null
+++ b/qa/workunits/cls/test_cls_refcount.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -e
+
+ceph_test_cls_refcount
+
+exit 0
diff --git a/qa/workunits/cls/test_cls_rgw.sh b/qa/workunits/cls/test_cls_rgw.sh
new file mode 100755
index 00000000..257338a0
--- /dev/null
+++ b/qa/workunits/cls/test_cls_rgw.sh
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+ceph_test_cls_rgw
+#ceph_test_cls_rgw_meta
+#ceph_test_cls_rgw_log
+#ceph_test_cls_rgw_opstate
+
+exit 0
diff --git a/qa/workunits/cls/test_cls_sdk.sh b/qa/workunits/cls/test_cls_sdk.sh
new file mode 100755
index 00000000..f1ccdc3b
--- /dev/null
+++ b/qa/workunits/cls/test_cls_sdk.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -e
+
+ceph_test_cls_sdk
+
+exit 0
diff --git a/qa/workunits/direct_io/.gitignore b/qa/workunits/direct_io/.gitignore
new file mode 100644
index 00000000..80f1fd1a
--- /dev/null
+++ b/qa/workunits/direct_io/.gitignore
@@ -0,0 +1,3 @@
+/direct_io_test
+/test_sync_io
+/test_short_dio_read
diff --git a/qa/workunits/direct_io/Makefile b/qa/workunits/direct_io/Makefile
new file mode 100644
index 00000000..20fec0be
--- /dev/null
+++ b/qa/workunits/direct_io/Makefile
@@ -0,0 +1,11 @@
+CFLAGS = -Wall -Wextra -D_GNU_SOURCE
+
+TARGETS = direct_io_test test_sync_io test_short_dio_read
+
+.c:
+	$(CC) $(CFLAGS) $@.c -o $@
+
+all:	$(TARGETS)
+
+clean:
+	rm $(TARGETS)
diff --git a/qa/workunits/direct_io/big.sh b/qa/workunits/direct_io/big.sh
new file mode 100755
index 00000000..43bd6d72
--- /dev/null
+++ b/qa/workunits/direct_io/big.sh
@@ -0,0 +1,6 @@
+#!/bin/sh -ex
+
+echo "test large (16MB) dio write"
+dd if=/dev/zero of=foo.big bs=16M count=1 oflag=direct
+
+echo OK
diff --git a/qa/workunits/direct_io/direct_io_test.c b/qa/workunits/direct_io/direct_io_test.c
new file mode 100644
index 00000000..ccfbbb86
--- /dev/null
+++ b/qa/workunits/direct_io/direct_io_test.c
@@ -0,0 +1,312 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <errno.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+/*
+ * direct_io_test
+ *
+ * This test does some I/O using O_DIRECT.
+ *
+ * Semantics of O_DIRECT can be found at http://lwn.net/Articles/348739/
+ *
+ */
+
+static int g_num_pages = 100;
+
+static int g_duration = 10;
+
+struct chunk {
+        uint64_t offset;
+        uint64_t pad0;
+        uint64_t pad1;
+        uint64_t pad2;
+        uint64_t pad3;
+        uint64_t pad4;
+        uint64_t pad5;
+        uint64_t not_offset;
+} __attribute__((packed));
+
+static int page_size;
+
+static char temp_file[] = "direct_io_temp_file_XXXXXX";
+
+static int safe_write(int fd, const void *buf, signed int len)
+{
+        const char *b = (const char*)buf;
+        /* Handle EINTR and short writes */
+        while (1) {
+                int res = write(fd, b, len);
+                if (res < 0) {
+                        int err = errno;
+                        if (err != EINTR) {
+                                return err;
+                        }
+                }
+                len -= res;
+                b += res;
+                if (len <= 0)
+                        return 0;
+        }
+}
+
+static int do_read(int fd, char *buf, int buf_sz)
+{
+        /* We assume no short reads or EINTR. It's not really clear how
+         * those things interact with O_DIRECT. */
+        int ret = read(fd, buf, buf_sz);
+        if (ret < 0) {
+                int err = errno;
+                printf("do_read: error: %d (%s)\n", err, strerror(err));
+                return err;
+        }
+        if (ret != buf_sz) {
+                printf("do_read: short read\n");
+                return -EIO;
+        }
+        return 0;
+}
+
+static int setup_temp_file(void)
+{
+        int fd;
+        int64_t num_chunks, i;
+
+        if (page_size % sizeof(struct chunk)) {
+                printf("setup_big_file: page_size doesn't divide evenly "
+                        "into data blocks.\n");
+                return -EINVAL;
+        }
+
+        fd = mkstemp(temp_file);
+        if (fd < 0) {
+                int err = errno;
+                printf("setup_big_file: mkostemps failed with error %d\n", err);
+                return err;
+        }
+
+        num_chunks = g_num_pages * (page_size / sizeof(struct chunk));
+        for (i = 0; i < num_chunks; ++i) {
+                int ret;
+                struct chunk c;
+                memset(&c, 0, sizeof(c));
+                c.offset = i * sizeof(struct chunk);
+                c.pad0 = 0;
+                c.pad1 = 1;
+                c.pad2 = 2;
+                c.pad3 = 3;
+                c.pad4 = 4;
+                c.pad5 = 5;
+                c.not_offset = ~c.offset;
+                ret = safe_write(fd, &c, sizeof(struct chunk));
+                if (ret) {
+                        printf("setup_big_file: safe_write failed with "
+                               "error: %d\n", ret);
+                        TEMP_FAILURE_RETRY(close(fd));
+                        unlink(temp_file);
+                        return ret;
+                }
+        }
+        TEMP_FAILURE_RETRY(close(fd));
+        return 0;
+}
+
+static int verify_chunk(const struct chunk *c, uint64_t offset)
+{
+        if (c->offset != offset) {
+                printf("verify_chunk(%" PRId64 "): bad offset value (got: %"
+                       PRId64 ", expected: %" PRId64 "\n", offset, c->offset, offset);
+                return EIO;
+        }
+        if (c->pad0 != 0) {
+                printf("verify_chunk(%" PRId64 "): bad pad0 value\n", offset);
+                return EIO;
+        }
+        if (c->pad1 != 1) {
+                printf("verify_chunk(%" PRId64 "): bad pad1 value\n", offset);
+                return EIO;
+        }
+        if (c->pad2 != 2) {
+                printf("verify_chunk(%" PRId64 "): bad pad2 value\n", offset);
+                return EIO;
+        }
+        if (c->pad3 != 3) {
+                printf("verify_chunk(%" PRId64 "): bad pad3 value\n", offset);
+                return EIO;
+        }
+        if (c->pad4 != 4) {
+                printf("verify_chunk(%" PRId64 "): bad pad4 value\n", offset);
+                return EIO;
+        }
+        if (c->pad5 != 5) {
+                printf("verify_chunk(%" PRId64 "): bad pad5 value\n", offset);
+                return EIO;
+        }
+        if (c->not_offset != ~offset) {
+                printf("verify_chunk(%" PRId64 "): bad not_offset value\n",
+                       offset);
+                return EIO;
+        }
+        return 0;
+}
+
+static int do_o_direct_reads(void)
+{
+        int fd, ret;
+        unsigned int i;
+        void *buf = 0;
+        time_t cur_time, end_time;
+        ret = posix_memalign(&buf, page_size, page_size);
+        if (ret) {
+                printf("do_o_direct_reads: posix_memalign returned %d\n", ret);
+                goto done;
+        }
+
+        fd = open(temp_file, O_RDONLY | O_DIRECT);
+        if (fd < 0) {
+                ret = errno;
+                printf("do_o_direct_reads: error opening fd: %d\n", ret);
+                goto free_buf;
+        }
+
+        // read the first chunk and see if it looks OK
+        ret = do_read(fd, buf, page_size);
+        if (ret)
+                goto close_fd;
+        ret = verify_chunk((struct chunk*)buf, 0);
+        if (ret)
+                goto close_fd;
+
+        // read some random chunks and see how they look
+        cur_time = time(NULL);
+        end_time = cur_time + g_duration;
+        i = 0;
+        do {
+                time_t next_time;
+                uint64_t offset;
+                int page;
+                unsigned int seed;
+
+                seed = i++;
+                page = rand_r(&seed) % g_num_pages;
+                offset = page;
+                offset *= page_size;
+                if (lseek64(fd, offset, SEEK_SET) == -1) {
+                        int err = errno;
+                        printf("lseek64(%" PRId64 ") failed: error %d (%s)\n",
+                               offset, err, strerror(err));
+                        goto close_fd;
+                }
+                ret = do_read(fd, buf, page_size);
+                if (ret)
+                        goto close_fd;
+                ret = verify_chunk((struct chunk*)buf, offset);
+                if (ret)
+                        goto close_fd;
+                next_time = time(NULL);
+                if (next_time > cur_time) {
+                        printf(".");
+                }
+                cur_time = next_time;
+        } while (time(NULL) < end_time);
+
+        printf("\ndo_o_direct_reads: SUCCESS\n");
+close_fd:
+        TEMP_FAILURE_RETRY(close(fd));
+free_buf:
+        free(buf);
+done:
+        return ret;
+}
+
+static void usage(char *argv0)
+{
+        printf("%s: tests direct I/O\n", argv0);
+        printf("-d <seconds>:          sets duration to <seconds>\n");
+        printf("-h:                    this help\n");
+        printf("-p <pages>:            sets number of pages to allocate\n");
+}
+
+static void parse_args(int argc, char *argv[])
+{
+        int c;
+        while ((c = getopt (argc, argv, "d:hp:")) != -1) {
+                switch (c) {
+                case 'd':
+                        g_duration = atoi(optarg);
+                        if (g_duration <= 0) {
+                                printf("tried to set invalid value of "
+                                       "g_duration: %d\n", g_num_pages);
+                                exit(1);
+                        }
+                        break;
+                case 'h':
+                        usage(argv[0]);
+                        exit(0);
+                        break;
+                case 'p':
+                        g_num_pages = atoi(optarg);
+                        if (g_num_pages <= 0) {
+                                printf("tried to set invalid value of "
+                                       "g_num_pages: %d\n", g_num_pages);
+                                exit(1);
+                        }
+                        break;
+                case '?':
+                        usage(argv[0]);
+                        exit(1);
+                        break;
+                default:
+                        usage(argv[0]);
+                        exit(1);
+                        break;
+                }
+        }
+}
+
+int main(int argc, char *argv[])
+{
+        int ret;
+
+        parse_args(argc, argv);
+
+        setvbuf(stdout, NULL, _IONBF, 0);
+
+        page_size = getpagesize();
+
+        ret = setup_temp_file();
+        if (ret) {
+                printf("setup_temp_file failed with error %d\n", ret);
+                goto done;
+        }
+
+        ret = do_o_direct_reads();
+        if (ret) {
+                printf("do_o_direct_reads failed with error %d\n", ret);
+                goto unlink_temp_file;
+        }
+
+unlink_temp_file:
+        unlink(temp_file);
+done:
+        return ret;
+}
diff --git a/qa/workunits/direct_io/misc.sh b/qa/workunits/direct_io/misc.sh
new file mode 100755
index 00000000..6de080d2
--- /dev/null
+++ b/qa/workunits/direct_io/misc.sh
@@ -0,0 +1,16 @@
+#!/bin/sh -ex
+
+# a few test cases from henry
+echo "test read from hole"
+dd if=/dev/zero of=dd3 bs=1 seek=1048576 count=0
+dd if=dd3 of=/tmp/ddout1 skip=8 bs=512 count=2 iflag=direct
+dd if=/dev/zero of=/tmp/dd3 bs=512 count=2
+cmp /tmp/dd3 /tmp/ddout1
+
+echo "other thing"
+dd if=/dev/urandom of=/tmp/dd10 bs=500 count=1
+dd if=/tmp/dd10 of=dd10 bs=512 seek=8388 count=1
+dd if=dd10 of=/tmp/dd10out bs=512 skip=8388 count=1 iflag=direct
+cmp /tmp/dd10 /tmp/dd10out
+
+echo OK
diff --git a/qa/workunits/direct_io/test_short_dio_read.c b/qa/workunits/direct_io/test_short_dio_read.c
new file mode 100644
index 00000000..50248555
--- /dev/null
+++ b/qa/workunits/direct_io/test_short_dio_read.c
@@ -0,0 +1,57 @@
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main()
+{
+        char buf[409600];
+        ssize_t r;
+	int err;
+	int fd = open("shortfile", O_WRONLY|O_CREAT, 0644);
+
+	if (fd < 0) {
+		err = errno;
+		printf("error: open() failed with: %d (%s)\n", err, strerror(err));
+		exit(err);
+	}
+
+	printf("writing first 3 bytes of 10k file\n");
+        r = write(fd, "foo", 3);
+	if (r == -1) {
+		err = errno;
+		printf("error: write() failed with: %d (%s)\n", err, strerror(err));
+		close(fd);
+		exit(err);
+	}
+        r = ftruncate(fd, 10000);
+	if (r == -1) {
+		err = errno;
+		printf("error: ftruncate() failed with: %d (%s)\n", err, strerror(err));
+		close(fd);
+		exit(err);
+	}
+	
+        fsync(fd);
+        close(fd);
+
+	printf("reading O_DIRECT\n");
+        fd = open("shortfile", O_RDONLY|O_DIRECT);
+	if (fd < 0) {
+		err = errno;
+		printf("error: open() failed with: %d (%s)\n", err, strerror(err));
+		exit(err);
+	}
+
+        r = read(fd, buf, sizeof(buf));
+        close(fd);
+
+        printf("got %d\n", (int)r);
+	if (r != 10000)
+		return 1;
+        return 0;
+}
diff --git a/qa/workunits/direct_io/test_sync_io.c b/qa/workunits/direct_io/test_sync_io.c
new file mode 100644
index 00000000..f393fa6e
--- /dev/null
+++ b/qa/workunits/direct_io/test_sync_io.c
@@ -0,0 +1,250 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <linux/types.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <errno.h>
+
+//#include "../client/ioctl.h"
+
+#include <linux/ioctl.h>
+#define CEPH_IOCTL_MAGIC 0x97
+#define CEPH_IOC_SYNCIO _IO(CEPH_IOCTL_MAGIC, 5)
+
+void write_pattern()
+{
+	printf("writing pattern\n");
+
+	uint64_t i;
+	int r;
+
+	int fd = open("foo", O_CREAT|O_WRONLY, 0644);
+	if (fd < 0) {
+	   r = errno;
+	   printf("write_pattern: error: open() failed with: %d (%s)\n", r, strerror(r));
+	   exit(r);
+	}
+	for (i=0; i<1048576 * sizeof(i); i += sizeof(i)) {
+		r = write(fd, &i, sizeof(i));
+		if (r == -1) {
+			r = errno;
+			printf("write_pattern: error: write() failed with: %d (%s)\n", r, strerror(r));
+			break;
+		}
+	}
+
+	close(fd);
+}
+
+int verify_pattern(char *buf, size_t len, uint64_t off)
+{
+	size_t i;
+
+	for (i = 0; i < len; i += sizeof(uint64_t)) {
+		uint64_t expected = i + off;
+		uint64_t actual = *(uint64_t*)(buf + i);
+		if (expected != actual) {
+			printf("error: offset %llu had %llu\n", (unsigned long long)expected,
+			       (unsigned long long)actual);
+			exit(1);
+		}
+	}
+	return 0;
+}
+
+void generate_pattern(void *buf, size_t len, uint64_t offset)
+{
+	uint64_t *v = buf;
+	size_t i;
+
+	for (i=0; i<len / sizeof(v); i++)
+		v[i] = i * sizeof(v) + offset;
+	verify_pattern(buf, len, offset);
+}
+
+int read_file(int buf_align, uint64_t offset, int len, int direct) {
+
+	printf("read_file buf_align %d offset %llu len %d\n", buf_align,
+	       (unsigned long long)offset, len);
+	void *rawbuf;
+	int r;
+        int flags;
+	int err = 0;
+
+	if(direct)
+	   flags = O_RDONLY|O_DIRECT;
+	else
+	   flags = O_RDONLY;
+
+	int fd = open("foo", flags);
+	if (fd < 0) {
+	   err = errno;
+	   printf("read_file: error: open() failed with: %d (%s)\n", err, strerror(err));
+	   exit(err);
+	}
+
+	if (!direct)
+	   ioctl(fd, CEPH_IOC_SYNCIO);
+
+	if ((r = posix_memalign(&rawbuf, 4096, len + buf_align)) != 0) {
+	   printf("read_file: error: posix_memalign failed with %d", r);
+	   close(fd);
+	   exit (r);
+	}
+
+	void *buf = (char *)rawbuf + buf_align;
+	memset(buf, 0, len);
+	r = pread(fd, buf, len, offset);
+	if (r == -1) {
+	   err = errno;
+	   printf("read_file: error: pread() failed with: %d (%s)\n", err, strerror(err));
+	   goto out;
+	}
+	r = verify_pattern(buf, len, offset);
+
+out:
+	close(fd);
+	free(rawbuf);
+	return r;
+}
+
+int read_direct(int buf_align, uint64_t offset, int len)
+{
+	printf("read_direct buf_align %d offset %llu len %d\n", buf_align,
+	       (unsigned long long)offset, len);
+	return read_file(buf_align, offset, len, 1);
+}
+
+int read_sync(int buf_align, uint64_t offset, int len)
+{
+	printf("read_sync buf_align %d offset %llu len %d\n", buf_align,
+	       (unsigned long long)offset, len);
+	return read_file(buf_align, offset, len, 0);
+}
+
+int write_file(int buf_align, uint64_t offset, int len, int direct)
+{
+	printf("write_file buf_align %d offset %llu len %d\n", buf_align,
+	       (unsigned long long)offset, len);
+	void *rawbuf;
+	int r;
+        int err = 0;
+	int flags;
+	if (direct)
+	   flags = O_WRONLY|O_DIRECT|O_CREAT;
+        else
+	   flags = O_WRONLY|O_CREAT;
+
+	int fd = open("foo", flags, 0644);
+	if (fd < 0) {
+	   int err = errno;
+	   printf("write_file: error: open() failed with: %d (%s)\n", err, strerror(err));
+	   exit(err);
+	}
+
+	if ((r = posix_memalign(&rawbuf, 4096, len + buf_align)) != 0) {
+	   printf("write_file: error: posix_memalign failed with %d", r);
+	   err = r;
+	   goto out_close;
+	}
+
+	if (!direct)
+	   ioctl(fd, CEPH_IOC_SYNCIO);
+
+	void *buf = (char *)rawbuf + buf_align;
+
+	generate_pattern(buf, len, offset);
+
+	r = pwrite(fd, buf, len, offset);
+	close(fd);
+
+	fd = open("foo", O_RDONLY);
+	if (fd < 0) {
+	   err = errno;
+	   printf("write_file: error: open() failed with: %d (%s)\n", err, strerror(err));
+	   free(rawbuf);
+	   goto out_unlink;
+	}
+	void *buf2 = malloc(len);
+	if (!buf2) {
+	   err = -ENOMEM;
+	   printf("write_file: error: malloc failed\n");
+	   goto out_free;
+	}
+
+	memset(buf2, 0, len);
+	r = pread(fd, buf2, len, offset);
+	if (r == -1) {
+	   err = errno;
+	   printf("write_file: error: pread() failed with: %d (%s)\n", err, strerror(err));
+	   goto out_free_buf;
+	}
+	r = verify_pattern(buf2, len, offset);
+
+out_free_buf:
+	free(buf2);
+out_free:
+	free(rawbuf);
+out_close:
+	close(fd);
+out_unlink:
+	unlink("foo");
+	if (err)
+	   exit(err);
+	return r;
+}
+
+int write_direct(int buf_align, uint64_t offset, int len)
+{
+	printf("write_direct buf_align %d offset %llu len %d\n", buf_align,
+	       (unsigned long long)offset, len);
+	return write_file (buf_align, offset, len, 1);
+}
+
+int write_sync(int buf_align, uint64_t offset, int len)
+{
+	printf("write_sync buf_align %d offset %llu len %d\n", buf_align,
+	       (unsigned long long)offset, len);
+	return write_file (buf_align, offset, len, 0);
+}
+
+int main(int argc, char **argv)
+{
+	uint64_t i, j, k;
+	int read = 1;
+	int write = 1;
+
+	if (argc >= 2 && strcmp(argv[1], "read") == 0)
+		write = 0;
+	if (argc >= 2 && strcmp(argv[1], "write") == 0)
+		read = 0;
+
+	if (read) {
+		write_pattern();
+		
+		for (i = 0; i < 4096; i += 512)
+			for (j = 4*1024*1024 - 4096; j < 4*1024*1024 + 4096; j += 512)
+				for (k = 1024; k <= 16384; k *= 2) {
+					read_direct(i, j, k);
+					read_sync(i, j, k);
+				}
+		
+	}
+	unlink("foo");
+	if (write) {
+		for (i = 0; i < 4096; i += 512)
+			for (j = 4*1024*1024 - 4096 + 512; j < 4*1024*1024 + 4096; j += 512)
+				for (k = 1024; k <= 16384; k *= 2) {
+					write_direct(i, j, k);
+					write_sync(i, j, k);
+				}
+	}
+	
+
+	return 0;
+}
diff --git a/qa/workunits/erasure-code/.gitignore b/qa/workunits/erasure-code/.gitignore
new file mode 100644
index 00000000..7e563b8b
--- /dev/null
+++ b/qa/workunits/erasure-code/.gitignore
@@ -0,0 +1,2 @@
+*.log
+*.trs
diff --git a/qa/workunits/erasure-code/bench.html b/qa/workunits/erasure-code/bench.html
new file mode 100644
index 00000000..3b4b6c74
--- /dev/null
+++ b/qa/workunits/erasure-code/bench.html
@@ -0,0 +1,34 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd" >
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <title>Erasure Code Plugins Benchmarks</title>
+    <link href="examples.css" rel="stylesheet" type="text/css">
+    <script language="javascript" type="text/javascript" src="jquery.js"></script>
+    <script language="javascript" type="text/javascript" src="jquery.flot.js"></script>
+    <script language="javascript" type="text/javascript" src="jquery.flot.categories.js"></script>
+    <script language="javascript" type="text/javascript" src="bench.js"></script>
+    <script language="javascript" type="text/javascript" src="plot.js"></script>
+  </head>
+  <body>
+
+    <div id="header">
+      <h2>Erasure Code Plugins Benchmarks</h2>
+    </div>
+
+    <div id="content">
+
+      <div class="demo-container">
+	<div id="encode" class="demo-placeholder"></div>
+      </div>
+      <p>encode: Y = GB/s, X = K/M</p>
+
+      <div class="demo-container">
+	<div id="decode" class="demo-placeholder"></div>
+      </div>
+      <p>decode: Y = GB/s, X = K/M/erasures</p>
+
+    </div>
+
+  </body>
+</html>
diff --git a/qa/workunits/erasure-code/bench.sh b/qa/workunits/erasure-code/bench.sh
new file mode 100755
index 00000000..23914ef8
--- /dev/null
+++ b/qa/workunits/erasure-code/bench.sh
@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2015 Red Hat <contact@redhat.com>
+# Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+#
+# Author: Loic Dachary <loic@dachary.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library Public License for more details.
+#
+# Test that it works from sources with:
+#
+#  CEPH_ERASURE_CODE_BENCHMARK=src/ceph_erasure_code_benchmark  \
+#  PLUGIN_DIRECTORY=build/lib \
+#      qa/workunits/erasure-code/bench.sh fplot jerasure |
+#      tee qa/workunits/erasure-code/bench.js
+#
+# This should start immediately and display:
+#
+# ...
+# [ '2/1',  .48035538612887358583  ],
+# [ '3/2',  .21648470405675016626  ],
+# etc.
+#
+# and complete within a few seconds. The result can then be displayed with:
+#
+#  firefox qa/workunits/erasure-code/bench.html
+#
+# Once it is confirmed to work, it can be run with a more significant
+# volume of data so that the measures are more reliable:
+#
+#  TOTAL_SIZE=$((4 * 1024 * 1024 * 1024)) \
+#  CEPH_ERASURE_CODE_BENCHMARK=src/ceph_erasure_code_benchmark  \
+#  PLUGIN_DIRECTORY=build/lib \
+#      qa/workunits/erasure-code/bench.sh fplot jerasure |
+#      tee qa/workunits/erasure-code/bench.js
+#
+set -e
+
+export PATH=/sbin:$PATH
+
+: ${VERBOSE:=false}
+: ${CEPH_ERASURE_CODE_BENCHMARK:=ceph_erasure_code_benchmark}
+: ${PLUGIN_DIRECTORY:=/usr/lib/ceph/erasure-code}
+: ${PLUGINS:=isa jerasure}
+: ${TECHNIQUES:=vandermonde cauchy}
+: ${TOTAL_SIZE:=$((1024 * 1024))}
+: ${SIZE:=4096}
+: ${PARAMETERS:=--parameter jerasure-per-chunk-alignment=true}
+
+function bench_header() {
+    echo -e "seconds\tKB\tplugin\tk\tm\twork.\titer.\tsize\teras.\tcommand."
+}
+
+function bench() {
+    local plugin=$1
+    shift
+    local k=$1
+    shift
+    local m=$1
+    shift
+    local workload=$1
+    shift
+    local iterations=$1
+    shift
+    local size=$1
+    shift
+    local erasures=$1
+    shift
+    command=$(echo $CEPH_ERASURE_CODE_BENCHMARK \
+        --plugin $plugin \
+        --workload $workload \
+        --iterations $iterations \
+        --size $size \
+        --erasures $erasures \
+        --parameter k=$k \
+        --parameter m=$m \
+        --erasure-code-dir $PLUGIN_DIRECTORY)
+    result=$($command "$@")
+    echo -e "$result\t$plugin\t$k\t$m\t$workload\t$iterations\t$size\t$erasures\t$command ""$@"
+}
+
+function packetsize() {
+    local k=$1
+    local w=$2
+    local vector_wordsize=$3
+    local size=$4
+
+    local p=$(( ($size / $k / $w / $vector_wordsize ) * $vector_wordsize))
+    if [ $p -gt 3100 ] ; then
+        p=3100
+    fi
+    echo $p
+}
+
+function bench_run() {
+    local plugin=jerasure
+    local w=8
+    local VECTOR_WORDSIZE=16
+    local ks="2 3 4 6 10"
+    declare -A k2ms
+    k2ms[2]="1"
+    k2ms[3]="2"
+    k2ms[4]="2 3"
+    k2ms[6]="2 3 4"
+    k2ms[10]="3 4"
+    for technique in ${TECHNIQUES} ; do
+        for plugin in ${PLUGINS} ; do
+            eval technique_parameter=\$${plugin}2technique_${technique}
+            echo "serie encode_${technique}_${plugin}"
+            for k in $ks ; do
+                for m in ${k2ms[$k]} ; do
+                    bench $plugin $k $m encode $(($TOTAL_SIZE / $SIZE)) $SIZE 0 \
+                        --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE $SIZE) \
+                        ${PARAMETERS} \
+                        --parameter technique=$technique_parameter
+
+                done
+            done
+        done
+    done
+    for technique in ${TECHNIQUES} ; do
+        for plugin in ${PLUGINS} ; do
+            eval technique_parameter=\$${plugin}2technique_${technique}
+            echo "serie decode_${technique}_${plugin}"
+            for k in $ks ; do
+                for m in ${k2ms[$k]} ; do
+                    echo
+                    for erasures in $(seq 1 $m) ; do
+                        bench $plugin $k $m decode $(($TOTAL_SIZE / $SIZE)) $SIZE $erasures \
+                            --parameter packetsize=$(packetsize $k $w $VECTOR_WORDSIZE  $SIZE) \
+                            ${PARAMETERS} \
+                            --parameter technique=$technique_parameter
+                    done
+                done
+            done
+        done
+    done
+}
+
+function fplot() {
+    local serie
+    bench_run | while read seconds total plugin k m workload iteration size erasures rest ; do 
+        if [ -z $seconds ] ; then
+            echo null,
+        elif [ $seconds = serie ] ; then
+            if [ "$serie" ] ; then
+                echo '];'
+            fi
+            local serie=`echo $total | sed 's/cauchy_\([0-9]\)/cauchy_good_\1/g'`
+            echo "var $serie = ["
+        else
+            local x
+            if [ $workload = encode ] ; then
+                x=$k/$m
+            else
+                x=$k/$m/$erasures
+            fi
+            echo "[ '$x', " $(echo "( $total / 1024 / 1024 ) / $seconds" | bc -ql) " ], "
+        fi
+    done
+    echo '];'
+}
+
+function main() {
+    bench_header
+    bench_run
+}
+
+if [ "$1" = fplot ] ; then
+    "$@"
+else
+    main
+fi
+# Local Variables:
+# compile-command: "\
+#   CEPH_ERASURE_CODE_BENCHMARK=../../../src/ceph_erasure_code_benchmark \
+#   PLUGIN_DIRECTORY=../../../build/lib \
+#   ./bench.sh
+# "
+# End:
diff --git a/qa/workunits/erasure-code/encode-decode-non-regression.sh b/qa/workunits/erasure-code/encode-decode-non-regression.sh
new file mode 100755
index 00000000..7f36c91c
--- /dev/null
+++ b/qa/workunits/erasure-code/encode-decode-non-regression.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2014 Red Hat <contact@redhat.com>
+#
+# Author: Loic Dachary <loic@dachary.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library Public License for more details.
+#
+set -ex
+
+: ${CORPUS:=https://github.com/ceph/ceph-erasure-code-corpus.git}
+: ${DIRECTORY:=$CEPH_ROOT/ceph-erasure-code-corpus}
+
+# when running from sources, the current directory must have precedence
+export PATH=:$PATH
+
+if ! test -d $DIRECTORY ; then
+    git clone $CORPUS $DIRECTORY
+fi
+
+my_version=v$(ceph --version | cut -f3 -d ' ')
+
+all_versions=$((ls -d $DIRECTORY/v* ; echo $DIRECTORY/$my_version ) | sort)
+
+for version in $all_versions ; do
+    if test -d $version ; then
+        $version/non-regression.sh
+    fi
+    if test $version = $DIRECTORY/$my_version ; then
+        break
+    fi
+done
diff --git a/qa/workunits/erasure-code/examples.css b/qa/workunits/erasure-code/examples.css
new file mode 100644
index 00000000..ee472477
--- /dev/null
+++ b/qa/workunits/erasure-code/examples.css
@@ -0,0 +1,97 @@
+* {	padding: 0; margin: 0; vertical-align: top; }
+
+body {
+	background: url(background.png) repeat-x;
+	font: 18px/1.5em "proxima-nova", Helvetica, Arial, sans-serif;
+}
+
+a {	color: #069; }
+a:hover { color: #28b; }
+
+h2 {
+	margin-top: 15px;
+	font: normal 32px "omnes-pro", Helvetica, Arial, sans-serif;
+}
+
+h3 {
+	margin-left: 30px;
+	font: normal 26px "omnes-pro", Helvetica, Arial, sans-serif;
+	color: #666;
+}
+
+p {
+	margin-top: 10px;
+}
+
+button {
+	font-size: 18px;
+	padding: 1px 7px;
+}
+
+input {
+	font-size: 18px;
+}
+
+input[type=checkbox] {
+	margin: 7px;
+}
+
+#header {
+	position: relative;
+	width: 900px;
+	margin: auto;
+}
+
+#header h2 {
+	margin-left: 10px;
+	vertical-align: middle;
+	font-size: 42px;
+	font-weight: bold;
+	text-decoration: none;
+	color: #000;
+}
+
+#content {
+	width: 880px;
+	margin: 0 auto;
+	padding: 10px;
+}
+
+#footer {
+	margin-top: 25px;
+	margin-bottom: 10px;
+	text-align: center;
+	font-size: 12px;
+	color: #999;
+}
+
+.demo-container {
+	box-sizing: border-box;
+	width: 850px;
+	height: 450px;
+	padding: 20px 15px 15px 15px;
+	margin: 15px auto 30px auto;
+	border: 1px solid #ddd;
+	background: #fff;
+	background: linear-gradient(#f6f6f6 0, #fff 50px);
+	background: -o-linear-gradient(#f6f6f6 0, #fff 50px);
+	background: -ms-linear-gradient(#f6f6f6 0, #fff 50px);
+	background: -moz-linear-gradient(#f6f6f6 0, #fff 50px);
+	background: -webkit-linear-gradient(#f6f6f6 0, #fff 50px);
+	box-shadow: 0 3px 10px rgba(0,0,0,0.15);
+	-o-box-shadow: 0 3px 10px rgba(0,0,0,0.1);
+	-ms-box-shadow: 0 3px 10px rgba(0,0,0,0.1);
+	-moz-box-shadow: 0 3px 10px rgba(0,0,0,0.1);
+	-webkit-box-shadow: 0 3px 10px rgba(0,0,0,0.1);
+}
+
+.demo-placeholder {
+	width: 100%;
+	height: 100%;
+	font-size: 14px;
+	line-height: 1.2em;
+}
+
+.legend table {
+	border-spacing: 5px;
+}
\ No newline at end of file
diff --git a/qa/workunits/erasure-code/jquery.flot.categories.js b/qa/workunits/erasure-code/jquery.flot.categories.js
new file mode 100644
index 00000000..2f9b2579
--- /dev/null
+++ b/qa/workunits/erasure-code/jquery.flot.categories.js
@@ -0,0 +1,190 @@
+/* Flot plugin for plotting textual data or categories.
+
+Copyright (c) 2007-2014 IOLA and Ole Laursen.
+Licensed under the MIT license.
+
+Consider a dataset like [["February", 34], ["March", 20], ...]. This plugin
+allows you to plot such a dataset directly.
+
+To enable it, you must specify mode: "categories" on the axis with the textual
+labels, e.g.
+
+	$.plot("#placeholder", data, { xaxis: { mode: "categories" } });
+
+By default, the labels are ordered as they are met in the data series. If you
+need a different ordering, you can specify "categories" on the axis options
+and list the categories there:
+
+	xaxis: {
+		mode: "categories",
+		categories: ["February", "March", "April"]
+	}
+
+If you need to customize the distances between the categories, you can specify
+"categories" as an object mapping labels to values
+
+	xaxis: {
+		mode: "categories",
+		categories: { "February": 1, "March": 3, "April": 4 }
+	}
+
+If you don't specify all categories, the remaining categories will be numbered
+from the max value plus 1 (with a spacing of 1 between each).
+
+Internally, the plugin works by transforming the input data through an auto-
+generated mapping where the first category becomes 0, the second 1, etc.
+Hence, a point like ["February", 34] becomes [0, 34] internally in Flot (this
+is visible in hover and click events that return numbers rather than the
+category labels). The plugin also overrides the tick generator to spit out the
+categories as ticks instead of the values.
+
+If you need to map a value back to its label, the mapping is always accessible
+as "categories" on the axis object, e.g. plot.getAxes().xaxis.categories.
+
+*/
+
+(function ($) {
+    var options = {
+        xaxis: {
+            categories: null
+        },
+        yaxis: {
+            categories: null
+        }
+    };
+    
+    function processRawData(plot, series, data, datapoints) {
+        // if categories are enabled, we need to disable
+        // auto-transformation to numbers so the strings are intact
+        // for later processing
+
+        var xCategories = series.xaxis.options.mode == "categories",
+            yCategories = series.yaxis.options.mode == "categories";
+        
+        if (!(xCategories || yCategories))
+            return;
+
+        var format = datapoints.format;
+
+        if (!format) {
+            // FIXME: auto-detection should really not be defined here
+            var s = series;
+            format = [];
+            format.push({ x: true, number: true, required: true });
+            format.push({ y: true, number: true, required: true });
+
+            if (s.bars.show || (s.lines.show && s.lines.fill)) {
+                var autoscale = !!((s.bars.show && s.bars.zero) || (s.lines.show && s.lines.zero));
+                format.push({ y: true, number: true, required: false, defaultValue: 0, autoscale: autoscale });
+                if (s.bars.horizontal) {
+                    delete format[format.length - 1].y;
+                    format[format.length - 1].x = true;
+                }
+            }
+            
+            datapoints.format = format;
+        }
+
+        for (var m = 0; m < format.length; ++m) {
+            if (format[m].x && xCategories)
+                format[m].number = false;
+            
+            if (format[m].y && yCategories)
+                format[m].number = false;
+        }
+    }
+
+    function getNextIndex(categories) {
+        var index = -1;
+        
+        for (var v in categories)
+            if (categories[v] > index)
+                index = categories[v];
+
+        return index + 1;
+    }
+
+    function categoriesTickGenerator(axis) {
+        var res = [];
+        for (var label in axis.categories) {
+            var v = axis.categories[label];
+            if (v >= axis.min && v <= axis.max)
+                res.push([v, label]);
+        }
+
+        res.sort(function (a, b) { return a[0] - b[0]; });
+
+        return res;
+    }
+    
+    function setupCategoriesForAxis(series, axis, datapoints) {
+        if (series[axis].options.mode != "categories")
+            return;
+        
+        if (!series[axis].categories) {
+            // parse options
+            var c = {}, o = series[axis].options.categories || {};
+            if ($.isArray(o)) {
+                for (var i = 0; i < o.length; ++i)
+                    c[o[i]] = i;
+            }
+            else {
+                for (var v in o)
+                    c[v] = o[v];
+            }
+            
+            series[axis].categories = c;
+        }
+
+        // fix ticks
+        if (!series[axis].options.ticks)
+            series[axis].options.ticks = categoriesTickGenerator;
+
+        transformPointsOnAxis(datapoints, axis, series[axis].categories);
+    }
+    
+    function transformPointsOnAxis(datapoints, axis, categories) {
+        // go through the points, transforming them
+        var points = datapoints.points,
+            ps = datapoints.pointsize,
+            format = datapoints.format,
+            formatColumn = axis.charAt(0),
+            index = getNextIndex(categories);
+
+        for (var i = 0; i < points.length; i += ps) {
+            if (points[i] == null)
+                continue;
+            
+            for (var m = 0; m < ps; ++m) {
+                var val = points[i + m];
+
+                if (val == null || !format[m][formatColumn])
+                    continue;
+
+                if (!(val in categories)) {
+                    categories[val] = index;
+                    ++index;
+                }
+                
+                points[i + m] = categories[val];
+            }
+        }
+    }
+
+    function processDatapoints(plot, series, datapoints) {
+        setupCategoriesForAxis(series, "xaxis", datapoints);
+        setupCategoriesForAxis(series, "yaxis", datapoints);
+    }
+
+    function init(plot) {
+        plot.hooks.processRawData.push(processRawData);
+        plot.hooks.processDatapoints.push(processDatapoints);
+    }
+    
+    $.plot.plugins.push({
+        init: init,
+        options: options,
+        name: 'categories',
+        version: '1.0'
+    });
+})(jQuery);
diff --git a/qa/workunits/erasure-code/jquery.flot.js b/qa/workunits/erasure-code/jquery.flot.js
new file mode 100644
index 00000000..39f3e4cf
--- /dev/null
+++ b/qa/workunits/erasure-code/jquery.flot.js
@@ -0,0 +1,3168 @@
+/* Javascript plotting library for jQuery, version 0.8.3.
+
+Copyright (c) 2007-2014 IOLA and Ole Laursen.
+Licensed under the MIT license.
+
+*/
+
+// first an inline dependency, jquery.colorhelpers.js, we inline it here
+// for convenience
+
+/* Plugin for jQuery for working with colors.
+ *
+ * Version 1.1.
+ *
+ * Inspiration from jQuery color animation plugin by John Resig.
+ *
+ * Released under the MIT license by Ole Laursen, October 2009.
+ *
+ * Examples:
+ *
+ *   $.color.parse("#fff").scale('rgb', 0.25).add('a', -0.5).toString()
+ *   var c = $.color.extract($("#mydiv"), 'background-color');
+ *   console.log(c.r, c.g, c.b, c.a);
+ *   $.color.make(100, 50, 25, 0.4).toString() // returns "rgba(100,50,25,0.4)"
+ *
+ * Note that .scale() and .add() return the same modified object
+ * instead of making a new one.
+ *
+ * V. 1.1: Fix error handling so e.g. parsing an empty string does
+ * produce a color rather than just crashing.
+ */
+(function($){$.color={};$.color.make=function(r,g,b,a){var o={};o.r=r||0;o.g=g||0;o.b=b||0;o.a=a!=null?a:1;o.add=function(c,d){for(var i=0;i<c.length;++i)o[c.charAt(i)]+=d;return o.normalize()};o.scale=function(c,f){for(var i=0;i<c.length;++i)o[c.charAt(i)]*=f;return o.normalize()};o.toString=function(){if(o.a>=1){return"rgb("+[o.r,o.g,o.b].join(",")+")"}else{return"rgba("+[o.r,o.g,o.b,o.a].join(",")+")"}};o.normalize=function(){function clamp(min,value,max){return value<min?min:value>max?max:value}o.r=clamp(0,parseInt(o.r),255);o.g=clamp(0,parseInt(o.g),255);o.b=clamp(0,parseInt(o.b),255);o.a=clamp(0,o.a,1);return o};o.clone=function(){return $.color.make(o.r,o.b,o.g,o.a)};return o.normalize()};$.color.extract=function(elem,css){var c;do{c=elem.css(css).toLowerCase();if(c!=""&&c!="transparent")break;elem=elem.parent()}while(elem.length&&!$.nodeName(elem.get(0),"body"));if(c=="rgba(0, 0, 0, 0)")c="transparent";return $.color.parse(c)};$.color.parse=function(str){var res,m=$.color.make;if(res=/rgb\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*\)/.exec(str))return m(parseInt(res[1],10),parseInt(res[2],10),parseInt(res[3],10));if(res=/rgba\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(str))return m(parseInt(res[1],10),parseInt(res[2],10),parseInt(res[3],10),parseFloat(res[4]));if(res=/rgb\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*\)/.exec(str))return m(parseFloat(res[1])*2.55,parseFloat(res[2])*2.55,parseFloat(res[3])*2.55);if(res=/rgba\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(str))return m(parseFloat(res[1])*2.55,parseFloat(res[2])*2.55,parseFloat(res[3])*2.55,parseFloat(res[4]));if(res=/#([a-fA-F0-9]{2})([a-fA-F0-9]{2})([a-fA-F0-9]{2})/.exec(str))return m(parseInt(res[1],16),parseInt(res[2],16),parseInt(res[3],16));if(res=/#([a-fA-F0-9])([a-fA-F0-9])([a-fA-F0-9])/.exec(str))return m(parseInt(res[1]+res[1],16),parseInt(res[2]+res[2],16),parseInt(res[3]+res[3],16));var name=$.trim(str).toLowerCase();if(name=="transparent")return m(255,255,255,0);else{res=lookupColors[name]||[0,0,0];return m(res[0],res[1],res[2])}};var lookupColors={aqua:[0,255,255],azure:[240,255,255],beige:[245,245,220],black:[0,0,0],blue:[0,0,255],brown:[165,42,42],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgrey:[169,169,169],darkgreen:[0,100,0],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkviolet:[148,0,211],fuchsia:[255,0,255],gold:[255,215,0],green:[0,128,0],indigo:[75,0,130],khaki:[240,230,140],lightblue:[173,216,230],lightcyan:[224,255,255],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightyellow:[255,255,224],lime:[0,255,0],magenta:[255,0,255],maroon:[128,0,0],navy:[0,0,128],olive:[128,128,0],orange:[255,165,0],pink:[255,192,203],purple:[128,0,128],violet:[128,0,128],red:[255,0,0],silver:[192,192,192],white:[255,255,255],yellow:[255,255,0]}})(jQuery);
+
+// the actual Flot code
+(function($) {
+
+	// Cache the prototype hasOwnProperty for faster access
+
+	var hasOwnProperty = Object.prototype.hasOwnProperty;
+
+    // A shim to provide 'detach' to jQuery versions prior to 1.4.  Using a DOM
+    // operation produces the same effect as detach, i.e. removing the element
+    // without touching its jQuery data.
+
+    // Do not merge this into Flot 0.9, since it requires jQuery 1.4.4+.
+
+    if (!$.fn.detach) {
+        $.fn.detach = function() {
+            return this.each(function() {
+                if (this.parentNode) {
+                    this.parentNode.removeChild( this );
+                }
+            });
+        };
+    }
+
+	///////////////////////////////////////////////////////////////////////////
+	// The Canvas object is a wrapper around an HTML5 <canvas> tag.
+	//
+	// @constructor
+	// @param {string} cls List of classes to apply to the canvas.
+	// @param {element} container Element onto which to append the canvas.
+	//
+	// Requiring a container is a little iffy, but unfortunately canvas
+	// operations don't work unless the canvas is attached to the DOM.
+
+	function Canvas(cls, container) {
+
+		var element = container.children("." + cls)[0];
+
+		if (element == null) {
+
+			element = document.createElement("canvas");
+			element.className = cls;
+
+			$(element).css({ direction: "ltr", position: "absolute", left: 0, top: 0 })
+				.appendTo(container);
+
+			// If HTML5 Canvas isn't available, fall back to [Ex|Flash]canvas
+
+			if (!element.getContext) {
+				if (window.G_vmlCanvasManager) {
+					element = window.G_vmlCanvasManager.initElement(element);
+				} else {
+					throw new Error("Canvas is not available. If you're using IE with a fall-back such as Excanvas, then there's either a mistake in your conditional include, or the page has no DOCTYPE and is rendering in Quirks Mode.");
+				}
+			}
+		}
+
+		this.element = element;
+
+		var context = this.context = element.getContext("2d");
+
+		// Determine the screen's ratio of physical to device-independent
+		// pixels.  This is the ratio between the canvas width that the browser
+		// advertises and the number of pixels actually present in that space.
+
+		// The iPhone 4, for example, has a device-independent width of 320px,
+		// but its screen is actually 640px wide.  It therefore has a pixel
+		// ratio of 2, while most normal devices have a ratio of 1.
+
+		var devicePixelRatio = window.devicePixelRatio || 1,
+			backingStoreRatio =
+				context.webkitBackingStorePixelRatio ||
+				context.mozBackingStorePixelRatio ||
+				context.msBackingStorePixelRatio ||
+				context.oBackingStorePixelRatio ||
+				context.backingStorePixelRatio || 1;
+
+		this.pixelRatio = devicePixelRatio / backingStoreRatio;
+
+		// Size the canvas to match the internal dimensions of its container
+
+		this.resize(container.width(), container.height());
+
+		// Collection of HTML div layers for text overlaid onto the canvas
+
+		this.textContainer = null;
+		this.text = {};
+
+		// Cache of text fragments and metrics, so we can avoid expensively
+		// re-calculating them when the plot is re-rendered in a loop.
+
+		this._textCache = {};
+	}
+
+	// Resizes the canvas to the given dimensions.
+	//
+	// @param {number} width New width of the canvas, in pixels.
+	// @param {number} width New height of the canvas, in pixels.
+
+	Canvas.prototype.resize = function(width, height) {
+
+		if (width <= 0 || height <= 0) {
+			throw new Error("Invalid dimensions for plot, width = " + width + ", height = " + height);
+		}
+
+		var element = this.element,
+			context = this.context,
+			pixelRatio = this.pixelRatio;
+
+		// Resize the canvas, increasing its density based on the display's
+		// pixel ratio; basically giving it more pixels without increasing the
+		// size of its element, to take advantage of the fact that retina
+		// displays have that many more pixels in the same advertised space.
+
+		// Resizing should reset the state (excanvas seems to be buggy though)
+
+		if (this.width != width) {
+			element.width = width * pixelRatio;
+			element.style.width = width + "px";
+			this.width = width;
+		}
+
+		if (this.height != height) {
+			element.height = height * pixelRatio;
+			element.style.height = height + "px";
+			this.height = height;
+		}
+
+		// Save the context, so we can reset in case we get replotted.  The
+		// restore ensure that we're really back at the initial state, and
+		// should be safe even if we haven't saved the initial state yet.
+
+		context.restore();
+		context.save();
+
+		// Scale the coordinate space to match the display density; so even though we
+		// may have twice as many pixels, we still want lines and other drawing to
+		// appear at the same size; the extra pixels will just make them crisper.
+
+		context.scale(pixelRatio, pixelRatio);
+	};
+
+	// Clears the entire canvas area, not including any overlaid HTML text
+
+	Canvas.prototype.clear = function() {
+		this.context.clearRect(0, 0, this.width, this.height);
+	};
+
+	// Finishes rendering the canvas, including managing the text overlay.
+
+	Canvas.prototype.render = function() {
+
+		var cache = this._textCache;
+
+		// For each text layer, add elements marked as active that haven't
+		// already been rendered, and remove those that are no longer active.
+
+		for (var layerKey in cache) {
+			if (hasOwnProperty.call(cache, layerKey)) {
+
+				var layer = this.getTextLayer(layerKey),
+					layerCache = cache[layerKey];
+
+				layer.hide();
+
+				for (var styleKey in layerCache) {
+					if (hasOwnProperty.call(layerCache, styleKey)) {
+						var styleCache = layerCache[styleKey];
+						for (var key in styleCache) {
+							if (hasOwnProperty.call(styleCache, key)) {
+
+								var positions = styleCache[key].positions;
+
+								for (var i = 0, position; position = positions[i]; i++) {
+									if (position.active) {
+										if (!position.rendered) {
+											layer.append(position.element);
+											position.rendered = true;
+										}
+									} else {
+										positions.splice(i--, 1);
+										if (position.rendered) {
+											position.element.detach();
+										}
+									}
+								}
+
+								if (positions.length == 0) {
+									delete styleCache[key];
+								}
+							}
+						}
+					}
+				}
+
+				layer.show();
+			}
+		}
+	};
+
+	// Creates (if necessary) and returns the text overlay container.
+	//
+	// @param {string} classes String of space-separated CSS classes used to
+	//     uniquely identify the text layer.
+	// @return {object} The jQuery-wrapped text-layer div.
+
+	Canvas.prototype.getTextLayer = function(classes) {
+
+		var layer = this.text[classes];
+
+		// Create the text layer if it doesn't exist
+
+		if (layer == null) {
+
+			// Create the text layer container, if it doesn't exist
+
+			if (this.textContainer == null) {
+				this.textContainer = $("<div class='flot-text'></div>")
+					.css({
+						position: "absolute",
+						top: 0,
+						left: 0,
+						bottom: 0,
+						right: 0,
+						'font-size': "smaller",
+						color: "#545454"
+					})
+					.insertAfter(this.element);
+			}
+
+			layer = this.text[classes] = $("<div></div>")
+				.addClass(classes)
+				.css({
+					position: "absolute",
+					top: 0,
+					left: 0,
+					bottom: 0,
+					right: 0
+				})
+				.appendTo(this.textContainer);
+		}
+
+		return layer;
+	};
+
+	// Creates (if necessary) and returns a text info object.
+	//
+	// The object looks like this:
+	//
+	// {
+	//     width: Width of the text's wrapper div.
+	//     height: Height of the text's wrapper div.
+	//     element: The jQuery-wrapped HTML div containing the text.
+	//     positions: Array of positions at which this text is drawn.
+	// }
+	//
+	// The positions array contains objects that look like this:
+	//
+	// {
+	//     active: Flag indicating whether the text should be visible.
+	//     rendered: Flag indicating whether the text is currently visible.
+	//     element: The jQuery-wrapped HTML div containing the text.
+	//     x: X coordinate at which to draw the text.
+	//     y: Y coordinate at which to draw the text.
+	// }
+	//
+	// Each position after the first receives a clone of the original element.
+	//
+	// The idea is that that the width, height, and general 'identity' of the
+	// text is constant no matter where it is placed; the placements are a
+	// secondary property.
+	//
+	// Canvas maintains a cache of recently-used text info objects; getTextInfo
+	// either returns the cached element or creates a new entry.
+	//
+	// @param {string} layer A string of space-separated CSS classes uniquely
+	//     identifying the layer containing this text.
+	// @param {string} text Text string to retrieve info for.
+	// @param {(string|object)=} font Either a string of space-separated CSS
+	//     classes or a font-spec object, defining the text's font and style.
+	// @param {number=} angle Angle at which to rotate the text, in degrees.
+	//     Angle is currently unused, it will be implemented in the future.
+	// @param {number=} width Maximum width of the text before it wraps.
+	// @return {object} a text info object.
+
+	Canvas.prototype.getTextInfo = function(layer, text, font, angle, width) {
+
+		var textStyle, layerCache, styleCache, info;
+
+		// Cast the value to a string, in case we were given a number or such
+
+		text = "" + text;
+
+		// If the font is a font-spec object, generate a CSS font definition
+
+		if (typeof font === "object") {
+			textStyle = font.style + " " + font.variant + " " + font.weight + " " + font.size + "px/" + font.lineHeight + "px " + font.family;
+		} else {
+			textStyle = font;
+		}
+
+		// Retrieve (or create) the cache for the text's layer and styles
+
+		layerCache = this._textCache[layer];
+
+		if (layerCache == null) {
+			layerCache = this._textCache[layer] = {};
+		}
+
+		styleCache = layerCache[textStyle];
+
+		if (styleCache == null) {
+			styleCache = layerCache[textStyle] = {};
+		}
+
+		info = styleCache[text];
+
+		// If we can't find a matching element in our cache, create a new one
+
+		if (info == null) {
+
+			var element = $("<div></div>").html(text)
+				.css({
+					position: "absolute",
+					'max-width': width,
+					top: -9999
+				})
+				.appendTo(this.getTextLayer(layer));
+
+			if (typeof font === "object") {
+				element.css({
+					font: textStyle,
+					color: font.color
+				});
+			} else if (typeof font === "string") {
+				element.addClass(font);
+			}
+
+			info = styleCache[text] = {
+				width: element.outerWidth(true),
+				height: element.outerHeight(true),
+				element: element,
+				positions: []
+			};
+
+			element.detach();
+		}
+
+		return info;
+	};
+
+	// Adds a text string to the canvas text overlay.
+	//
+	// The text isn't drawn immediately; it is marked as rendering, which will
+	// result in its addition to the canvas on the next render pass.
+	//
+	// @param {string} layer A string of space-separated CSS classes uniquely
+	//     identifying the layer containing this text.
+	// @param {number} x X coordinate at which to draw the text.
+	// @param {number} y Y coordinate at which to draw the text.
+	// @param {string} text Text string to draw.
+	// @param {(string|object)=} font Either a string of space-separated CSS
+	//     classes or a font-spec object, defining the text's font and style.
+	// @param {number=} angle Angle at which to rotate the text, in degrees.
+	//     Angle is currently unused, it will be implemented in the future.
+	// @param {number=} width Maximum width of the text before it wraps.
+	// @param {string=} halign Horizontal alignment of the text; either "left",
+	//     "center" or "right".
+	// @param {string=} valign Vertical alignment of the text; either "top",
+	//     "middle" or "bottom".
+
+	Canvas.prototype.addText = function(layer, x, y, text, font, angle, width, halign, valign) {
+
+		var info = this.getTextInfo(layer, text, font, angle, width),
+			positions = info.positions;
+
+		// Tweak the div's position to match the text's alignment
+
+		if (halign == "center") {
+			x -= info.width / 2;
+		} else if (halign == "right") {
+			x -= info.width;
+		}
+
+		if (valign == "middle") {
+			y -= info.height / 2;
+		} else if (valign == "bottom") {
+			y -= info.height;
+		}
+
+		// Determine whether this text already exists at this position.
+		// If so, mark it for inclusion in the next render pass.
+
+		for (var i = 0, position; position = positions[i]; i++) {
+			if (position.x == x && position.y == y) {
+				position.active = true;
+				return;
+			}
+		}
+
+		// If the text doesn't exist at this position, create a new entry
+
+		// For the very first position we'll re-use the original element,
+		// while for subsequent ones we'll clone it.
+
+		position = {
+			active: true,
+			rendered: false,
+			element: positions.length ? info.element.clone() : info.element,
+			x: x,
+			y: y
+		};
+
+		positions.push(position);
+
+		// Move the element to its final position within the container
+
+		position.element.css({
+			top: Math.round(y),
+			left: Math.round(x),
+			'text-align': halign	// In case the text wraps
+		});
+	};
+
+	// Removes one or more text strings from the canvas text overlay.
+	//
+	// If no parameters are given, all text within the layer is removed.
+	//
+	// Note that the text is not immediately removed; it is simply marked as
+	// inactive, which will result in its removal on the next render pass.
+	// This avoids the performance penalty for 'clear and redraw' behavior,
+	// where we potentially get rid of all text on a layer, but will likely
+	// add back most or all of it later, as when redrawing axes, for example.
+	//
+	// @param {string} layer A string of space-separated CSS classes uniquely
+	//     identifying the layer containing this text.
+	// @param {number=} x X coordinate of the text.
+	// @param {number=} y Y coordinate of the text.
+	// @param {string=} text Text string to remove.
+	// @param {(string|object)=} font Either a string of space-separated CSS
+	//     classes or a font-spec object, defining the text's font and style.
+	// @param {number=} angle Angle at which the text is rotated, in degrees.
+	//     Angle is currently unused, it will be implemented in the future.
+
+	Canvas.prototype.removeText = function(layer, x, y, text, font, angle) {
+		if (text == null) {
+			var layerCache = this._textCache[layer];
+			if (layerCache != null) {
+				for (var styleKey in layerCache) {
+					if (hasOwnProperty.call(layerCache, styleKey)) {
+						var styleCache = layerCache[styleKey];
+						for (var key in styleCache) {
+							if (hasOwnProperty.call(styleCache, key)) {
+								var positions = styleCache[key].positions;
+								for (var i = 0, position; position = positions[i]; i++) {
+									position.active = false;
+								}
+							}
+						}
+					}
+				}
+			}
+		} else {
+			var positions = this.getTextInfo(layer, text, font, angle).positions;
+			for (var i = 0, position; position = positions[i]; i++) {
+				if (position.x == x && position.y == y) {
+					position.active = false;
+				}
+			}
+		}
+	};
+
+	///////////////////////////////////////////////////////////////////////////
+	// The top-level container for the entire plot.
+
+    function Plot(placeholder, data_, options_, plugins) {
+        // data is on the form:
+        //   [ series1, series2 ... ]
+        // where series is either just the data as [ [x1, y1], [x2, y2], ... ]
+        // or { data: [ [x1, y1], [x2, y2], ... ], label: "some label", ... }
+
+        var series = [],
+            options = {
+                // the color theme used for graphs
+                colors: ["#edc240", "#afd8f8", "#cb4b4b", "#4da74d", "#9440ed"],
+                legend: {
+                    show: true,
+                    noColumns: 1, // number of colums in legend table
+                    labelFormatter: null, // fn: string -> string
+                    labelBoxBorderColor: "#ccc", // border color for the little label boxes
+                    container: null, // container (as jQuery object) to put legend in, null means default on top of graph
+                    position: "ne", // position of default legend container within plot
+                    margin: 5, // distance from grid edge to default legend container within plot
+                    backgroundColor: null, // null means auto-detect
+                    backgroundOpacity: 0.85, // set to 0 to avoid background
+                    sorted: null    // default to no legend sorting
+                },
+                xaxis: {
+                    show: null, // null = auto-detect, true = always, false = never
+                    position: "bottom", // or "top"
+                    mode: null, // null or "time"
+                    font: null, // null (derived from CSS in placeholder) or object like { size: 11, lineHeight: 13, style: "italic", weight: "bold", family: "sans-serif", variant: "small-caps" }
+                    color: null, // base color, labels, ticks
+                    tickColor: null, // possibly different color of ticks, e.g. "rgba(0,0,0,0.15)"
+                    transform: null, // null or f: number -> number to transform axis
+                    inverseTransform: null, // if transform is set, this should be the inverse function
+                    min: null, // min. value to show, null means set automatically
+                    max: null, // max. value to show, null means set automatically
+                    autoscaleMargin: null, // margin in % to add if auto-setting min/max
+                    ticks: null, // either [1, 3] or [[1, "a"], 3] or (fn: axis info -> ticks) or app. number of ticks for auto-ticks
+                    tickFormatter: null, // fn: number -> string
+                    labelWidth: null, // size of tick labels in pixels
+                    labelHeight: null,
+                    reserveSpace: null, // whether to reserve space even if axis isn't shown
+                    tickLength: null, // size in pixels of ticks, or "full" for whole line
+                    alignTicksWithAxis: null, // axis number or null for no sync
+                    tickDecimals: null, // no. of decimals, null means auto
+                    tickSize: null, // number or [number, "unit"]
+                    minTickSize: null // number or [number, "unit"]
+                },
+                yaxis: {
+                    autoscaleMargin: 0.02,
+                    position: "left" // or "right"
+                },
+                xaxes: [],
+                yaxes: [],
+                series: {
+                    points: {
+                        show: false,
+                        radius: 3,
+                        lineWidth: 2, // in pixels
+                        fill: true,
+                        fillColor: "#ffffff",
+                        symbol: "circle" // or callback
+                    },
+                    lines: {
+                        // we don't put in show: false so we can see
+                        // whether lines were actively disabled
+                        lineWidth: 2, // in pixels
+                        fill: false,
+                        fillColor: null,
+                        steps: false
+                        // Omit 'zero', so we can later default its value to
+                        // match that of the 'fill' option.
+                    },
+                    bars: {
+                        show: false,
+                        lineWidth: 2, // in pixels
+                        barWidth: 1, // in units of the x axis
+                        fill: true,
+                        fillColor: null,
+                        align: "left", // "left", "right", or "center"
+                        horizontal: false,
+                        zero: true
+                    },
+                    shadowSize: 3,
+                    highlightColor: null
+                },
+                grid: {
+                    show: true,
+                    aboveData: false,
+                    color: "#545454", // primary color used for outline and labels
+                    backgroundColor: null, // null for transparent, else color
+                    borderColor: null, // set if different from the grid color
+                    tickColor: null, // color for the ticks, e.g. "rgba(0,0,0,0.15)"
+                    margin: 0, // distance from the canvas edge to the grid
+                    labelMargin: 5, // in pixels
+                    axisMargin: 8, // in pixels
+                    borderWidth: 2, // in pixels
+                    minBorderMargin: null, // in pixels, null means taken from points radius
+                    markings: null, // array of ranges or fn: axes -> array of ranges
+                    markingsColor: "#f4f4f4",
+                    markingsLineWidth: 2,
+                    // interactive stuff
+                    clickable: false,
+                    hoverable: false,
+                    autoHighlight: true, // highlight in case mouse is near
+                    mouseActiveRadius: 10 // how far the mouse can be away to activate an item
+                },
+                interaction: {
+                    redrawOverlayInterval: 1000/60 // time between updates, -1 means in same flow
+                },
+                hooks: {}
+            },
+        surface = null,     // the canvas for the plot itself
+        overlay = null,     // canvas for interactive stuff on top of plot
+        eventHolder = null, // jQuery object that events should be bound to
+        ctx = null, octx = null,
+        xaxes = [], yaxes = [],
+        plotOffset = { left: 0, right: 0, top: 0, bottom: 0},
+        plotWidth = 0, plotHeight = 0,
+        hooks = {
+            processOptions: [],
+            processRawData: [],
+            processDatapoints: [],
+            processOffset: [],
+            drawBackground: [],
+            drawSeries: [],
+            draw: [],
+            bindEvents: [],
+            drawOverlay: [],
+            shutdown: []
+        },
+        plot = this;
+
+        // public functions
+        plot.setData = setData;
+        plot.setupGrid = setupGrid;
+        plot.draw = draw;
+        plot.getPlaceholder = function() { return placeholder; };
+        plot.getCanvas = function() { return surface.element; };
+        plot.getPlotOffset = function() { return plotOffset; };
+        plot.width = function () { return plotWidth; };
+        plot.height = function () { return plotHeight; };
+        plot.offset = function () {
+            var o = eventHolder.offset();
+            o.left += plotOffset.left;
+            o.top += plotOffset.top;
+            return o;
+        };
+        plot.getData = function () { return series; };
+        plot.getAxes = function () {
+            var res = {}, i;
+            $.each(xaxes.concat(yaxes), function (_, axis) {
+                if (axis)
+                    res[axis.direction + (axis.n != 1 ? axis.n : "") + "axis"] = axis;
+            });
+            return res;
+        };
+        plot.getXAxes = function () { return xaxes; };
+        plot.getYAxes = function () { return yaxes; };
+        plot.c2p = canvasToAxisCoords;
+        plot.p2c = axisToCanvasCoords;
+        plot.getOptions = function () { return options; };
+        plot.highlight = highlight;
+        plot.unhighlight = unhighlight;
+        plot.triggerRedrawOverlay = triggerRedrawOverlay;
+        plot.pointOffset = function(point) {
+            return {
+                left: parseInt(xaxes[axisNumber(point, "x") - 1].p2c(+point.x) + plotOffset.left, 10),
+                top: parseInt(yaxes[axisNumber(point, "y") - 1].p2c(+point.y) + plotOffset.top, 10)
+            };
+        };
+        plot.shutdown = shutdown;
+        plot.destroy = function () {
+            shutdown();
+            placeholder.removeData("plot").empty();
+
+            series = [];
+            options = null;
+            surface = null;
+            overlay = null;
+            eventHolder = null;
+            ctx = null;
+            octx = null;
+            xaxes = [];
+            yaxes = [];
+            hooks = null;
+            highlights = [];
+            plot = null;
+        };
+        plot.resize = function () {
+        	var width = placeholder.width(),
+        		height = placeholder.height();
+            surface.resize(width, height);
+            overlay.resize(width, height);
+        };
+
+        // public attributes
+        plot.hooks = hooks;
+
+        // initialize
+        initPlugins(plot);
+        parseOptions(options_);
+        setupCanvases();
+        setData(data_);
+        setupGrid();
+        draw();
+        bindEvents();
+
+
+        function executeHooks(hook, args) {
+            args = [plot].concat(args);
+            for (var i = 0; i < hook.length; ++i)
+                hook[i].apply(this, args);
+        }
+
+        function initPlugins() {
+
+            // References to key classes, allowing plugins to modify them
+
+            var classes = {
+                Canvas: Canvas
+            };
+
+            for (var i = 0; i < plugins.length; ++i) {
+                var p = plugins[i];
+                p.init(plot, classes);
+                if (p.options)
+                    $.extend(true, options, p.options);
+            }
+        }
+
+        function parseOptions(opts) {
+
+            $.extend(true, options, opts);
+
+            // $.extend merges arrays, rather than replacing them.  When less
+            // colors are provided than the size of the default palette, we
+            // end up with those colors plus the remaining defaults, which is
+            // not expected behavior; avoid it by replacing them here.
+
+            if (opts && opts.colors) {
+            	options.colors = opts.colors;
+            }
+
+            if (options.xaxis.color == null)
+                options.xaxis.color = $.color.parse(options.grid.color).scale('a', 0.22).toString();
+            if (options.yaxis.color == null)
+                options.yaxis.color = $.color.parse(options.grid.color).scale('a', 0.22).toString();
+
+            if (options.xaxis.tickColor == null) // grid.tickColor for back-compatibility
+                options.xaxis.tickColor = options.grid.tickColor || options.xaxis.color;
+            if (options.yaxis.tickColor == null) // grid.tickColor for back-compatibility
+                options.yaxis.tickColor = options.grid.tickColor || options.yaxis.color;
+
+            if (options.grid.borderColor == null)
+                options.grid.borderColor = options.grid.color;
+            if (options.grid.tickColor == null)
+                options.grid.tickColor = $.color.parse(options.grid.color).scale('a', 0.22).toString();
+
+            // Fill in defaults for axis options, including any unspecified
+            // font-spec fields, if a font-spec was provided.
+
+            // If no x/y axis options were provided, create one of each anyway,
+            // since the rest of the code assumes that they exist.
+
+            var i, axisOptions, axisCount,
+                fontSize = placeholder.css("font-size"),
+                fontSizeDefault = fontSize ? +fontSize.replace("px", "") : 13,
+                fontDefaults = {
+                    style: placeholder.css("font-style"),
+                    size: Math.round(0.8 * fontSizeDefault),
+                    variant: placeholder.css("font-variant"),
+                    weight: placeholder.css("font-weight"),
+                    family: placeholder.css("font-family")
+                };
+
+            axisCount = options.xaxes.length || 1;
+            for (i = 0; i < axisCount; ++i) {
+
+                axisOptions = options.xaxes[i];
+                if (axisOptions && !axisOptions.tickColor) {
+                    axisOptions.tickColor = axisOptions.color;
+                }
+
+                axisOptions = $.extend(true, {}, options.xaxis, axisOptions);
+                options.xaxes[i] = axisOptions;
+
+                if (axisOptions.font) {
+                    axisOptions.font = $.extend({}, fontDefaults, axisOptions.font);
+                    if (!axisOptions.font.color) {
+                        axisOptions.font.color = axisOptions.color;
+                    }
+                    if (!axisOptions.font.lineHeight) {
+                        axisOptions.font.lineHeight = Math.round(axisOptions.font.size * 1.15);
+                    }
+                }
+            }
+
+            axisCount = options.yaxes.length || 1;
+            for (i = 0; i < axisCount; ++i) {
+
+                axisOptions = options.yaxes[i];
+                if (axisOptions && !axisOptions.tickColor) {
+                    axisOptions.tickColor = axisOptions.color;
+                }
+
+                axisOptions = $.extend(true, {}, options.yaxis, axisOptions);
+                options.yaxes[i] = axisOptions;
+
+                if (axisOptions.font) {
+                    axisOptions.font = $.extend({}, fontDefaults, axisOptions.font);
+                    if (!axisOptions.font.color) {
+                        axisOptions.font.color = axisOptions.color;
+                    }
+                    if (!axisOptions.font.lineHeight) {
+                        axisOptions.font.lineHeight = Math.round(axisOptions.font.size * 1.15);
+                    }
+                }
+            }
+
+            // backwards compatibility, to be removed in future
+            if (options.xaxis.noTicks && options.xaxis.ticks == null)
+                options.xaxis.ticks = options.xaxis.noTicks;
+            if (options.yaxis.noTicks && options.yaxis.ticks == null)
+                options.yaxis.ticks = options.yaxis.noTicks;
+            if (options.x2axis) {
+                options.xaxes[1] = $.extend(true, {}, options.xaxis, options.x2axis);
+                options.xaxes[1].position = "top";
+                // Override the inherit to allow the axis to auto-scale
+                if (options.x2axis.min == null) {
+                    options.xaxes[1].min = null;
+                }
+                if (options.x2axis.max == null) {
+                    options.xaxes[1].max = null;
+                }
+            }
+            if (options.y2axis) {
+                options.yaxes[1] = $.extend(true, {}, options.yaxis, options.y2axis);
+                options.yaxes[1].position = "right";
+                // Override the inherit to allow the axis to auto-scale
+                if (options.y2axis.min == null) {
+                    options.yaxes[1].min = null;
+                }
+                if (options.y2axis.max == null) {
+                    options.yaxes[1].max = null;
+                }
+            }
+            if (options.grid.coloredAreas)
+                options.grid.markings = options.grid.coloredAreas;
+            if (options.grid.coloredAreasColor)
+                options.grid.markingsColor = options.grid.coloredAreasColor;
+            if (options.lines)
+                $.extend(true, options.series.lines, options.lines);
+            if (options.points)
+                $.extend(true, options.series.points, options.points);
+            if (options.bars)
+                $.extend(true, options.series.bars, options.bars);
+            if (options.shadowSize != null)
+                options.series.shadowSize = options.shadowSize;
+            if (options.highlightColor != null)
+                options.series.highlightColor = options.highlightColor;
+
+            // save options on axes for future reference
+            for (i = 0; i < options.xaxes.length; ++i)
+                getOrCreateAxis(xaxes, i + 1).options = options.xaxes[i];
+            for (i = 0; i < options.yaxes.length; ++i)
+                getOrCreateAxis(yaxes, i + 1).options = options.yaxes[i];
+
+            // add hooks from options
+            for (var n in hooks)
+                if (options.hooks[n] && options.hooks[n].length)
+                    hooks[n] = hooks[n].concat(options.hooks[n]);
+
+            executeHooks(hooks.processOptions, [options]);
+        }
+
+        function setData(d) {
+            series = parseData(d);
+            fillInSeriesOptions();
+            processData();
+        }
+
+        function parseData(d) {
+            var res = [];
+            for (var i = 0; i < d.length; ++i) {
+                var s = $.extend(true, {}, options.series);
+
+                if (d[i].data != null) {
+                    s.data = d[i].data; // move the data instead of deep-copy
+                    delete d[i].data;
+
+                    $.extend(true, s, d[i]);
+
+                    d[i].data = s.data;
+                }
+                else
+                    s.data = d[i];
+                res.push(s);
+            }
+
+            return res;
+        }
+
+        function axisNumber(obj, coord) {
+            var a = obj[coord + "axis"];
+            if (typeof a == "object") // if we got a real axis, extract number
+                a = a.n;
+            if (typeof a != "number")
+                a = 1; // default to first axis
+            return a;
+        }
+
+        function allAxes() {
+            // return flat array without annoying null entries
+            return $.grep(xaxes.concat(yaxes), function (a) { return a; });
+        }
+
+        function canvasToAxisCoords(pos) {
+            // return an object with x/y corresponding to all used axes
+            var res = {}, i, axis;
+            for (i = 0; i < xaxes.length; ++i) {
+                axis = xaxes[i];
+                if (axis && axis.used)
+                    res["x" + axis.n] = axis.c2p(pos.left);
+            }
+
+            for (i = 0; i < yaxes.length; ++i) {
+                axis = yaxes[i];
+                if (axis && axis.used)
+                    res["y" + axis.n] = axis.c2p(pos.top);
+            }
+
+            if (res.x1 !== undefined)
+                res.x = res.x1;
+            if (res.y1 !== undefined)
+                res.y = res.y1;
+
+            return res;
+        }
+
+        function axisToCanvasCoords(pos) {
+            // get canvas coords from the first pair of x/y found in pos
+            var res = {}, i, axis, key;
+
+            for (i = 0; i < xaxes.length; ++i) {
+                axis = xaxes[i];
+                if (axis && axis.used) {
+                    key = "x" + axis.n;
+                    if (pos[key] == null && axis.n == 1)
+                        key = "x";
+
+                    if (pos[key] != null) {
+                        res.left = axis.p2c(pos[key]);
+                        break;
+                    }
+                }
+            }
+
+            for (i = 0; i < yaxes.length; ++i) {
+                axis = yaxes[i];
+                if (axis && axis.used) {
+                    key = "y" + axis.n;
+                    if (pos[key] == null && axis.n == 1)
+                        key = "y";
+
+                    if (pos[key] != null) {
+                        res.top = axis.p2c(pos[key]);
+                        break;
+                    }
+                }
+            }
+
+            return res;
+        }
+
+        function getOrCreateAxis(axes, number) {
+            if (!axes[number - 1])
+                axes[number - 1] = {
+                    n: number, // save the number for future reference
+                    direction: axes == xaxes ? "x" : "y",
+                    options: $.extend(true, {}, axes == xaxes ? options.xaxis : options.yaxis)
+                };
+
+            return axes[number - 1];
+        }
+
+        function fillInSeriesOptions() {
+
+            var neededColors = series.length, maxIndex = -1, i;
+
+            // Subtract the number of series that already have fixed colors or
+            // color indexes from the number that we still need to generate.
+
+            for (i = 0; i < series.length; ++i) {
+                var sc = series[i].color;
+                if (sc != null) {
+                    neededColors--;
+                    if (typeof sc == "number" && sc > maxIndex) {
+                        maxIndex = sc;
+                    }
+                }
+            }
+
+            // If any of the series have fixed color indexes, then we need to
+            // generate at least as many colors as the highest index.
+
+            if (neededColors <= maxIndex) {
+                neededColors = maxIndex + 1;
+            }
+
+            // Generate all the colors, using first the option colors and then
+            // variations on those colors once they're exhausted.
+
+            var c, colors = [], colorPool = options.colors,
+                colorPoolSize = colorPool.length, variation = 0;
+
+            for (i = 0; i < neededColors; i++) {
+
+                c = $.color.parse(colorPool[i % colorPoolSize] || "#666");
+
+                // Each time we exhaust the colors in the pool we adjust
+                // a scaling factor used to produce more variations on
+                // those colors. The factor alternates negative/positive
+                // to produce lighter/darker colors.
+
+                // Reset the variation after every few cycles, or else
+                // it will end up producing only white or black colors.
+
+                if (i % colorPoolSize == 0 && i) {
+                    if (variation >= 0) {
+                        if (variation < 0.5) {
+                            variation = -variation - 0.2;
+                        } else variation = 0;
+                    } else variation = -variation;
+                }
+
+                colors[i] = c.scale('rgb', 1 + variation);
+            }
+
+            // Finalize the series options, filling in their colors
+
+            var colori = 0, s;
+            for (i = 0; i < series.length; ++i) {
+                s = series[i];
+
+                // assign colors
+                if (s.color == null) {
+                    s.color = colors[colori].toString();
+                    ++colori;
+                }
+                else if (typeof s.color == "number")
+                    s.color = colors[s.color].toString();
+
+                // turn on lines automatically in case nothing is set
+                if (s.lines.show == null) {
+                    var v, show = true;
+                    for (v in s)
+                        if (s[v] && s[v].show) {
+                            show = false;
+                            break;
+                        }
+                    if (show)
+                        s.lines.show = true;
+                }
+
+                // If nothing was provided for lines.zero, default it to match
+                // lines.fill, since areas by default should extend to zero.
+
+                if (s.lines.zero == null) {
+                    s.lines.zero = !!s.lines.fill;
+                }
+
+                // setup axes
+                s.xaxis = getOrCreateAxis(xaxes, axisNumber(s, "x"));
+                s.yaxis = getOrCreateAxis(yaxes, axisNumber(s, "y"));
+            }
+        }
+
+        function processData() {
+            var topSentry = Number.POSITIVE_INFINITY,
+                bottomSentry = Number.NEGATIVE_INFINITY,
+                fakeInfinity = Number.MAX_VALUE,
+                i, j, k, m, length,
+                s, points, ps, x, y, axis, val, f, p,
+                data, format;
+
+            function updateAxis(axis, min, max) {
+                if (min < axis.datamin && min != -fakeInfinity)
+                    axis.datamin = min;
+                if (max > axis.datamax && max != fakeInfinity)
+                    axis.datamax = max;
+            }
+
+            $.each(allAxes(), function (_, axis) {
+                // init axis
+                axis.datamin = topSentry;
+                axis.datamax = bottomSentry;
+                axis.used = false;
+            });
+
+            for (i = 0; i < series.length; ++i) {
+                s = series[i];
+                s.datapoints = { points: [] };
+
+                executeHooks(hooks.processRawData, [ s, s.data, s.datapoints ]);
+            }
+
+            // first pass: clean and copy data
+            for (i = 0; i < series.length; ++i) {
+                s = series[i];
+
+                data = s.data;
+                format = s.datapoints.format;
+
+                if (!format) {
+                    format = [];
+                    // find out how to copy
+                    format.push({ x: true, number: true, required: true });
+                    format.push({ y: true, number: true, required: true });
+
+                    if (s.bars.show || (s.lines.show && s.lines.fill)) {
+                        var autoscale = !!((s.bars.show && s.bars.zero) || (s.lines.show && s.lines.zero));
+                        format.push({ y: true, number: true, required: false, defaultValue: 0, autoscale: autoscale });
+                        if (s.bars.horizontal) {
+                            delete format[format.length - 1].y;
+                            format[format.length - 1].x = true;
+                        }
+                    }
+
+                    s.datapoints.format = format;
+                }
+
+                if (s.datapoints.pointsize != null)
+                    continue; // already filled in
+
+                s.datapoints.pointsize = format.length;
+
+                ps = s.datapoints.pointsize;
+                points = s.datapoints.points;
+
+                var insertSteps = s.lines.show && s.lines.steps;
+                s.xaxis.used = s.yaxis.used = true;
+
+                for (j = k = 0; j < data.length; ++j, k += ps) {
+                    p = data[j];
+
+                    var nullify = p == null;
+                    if (!nullify) {
+                        for (m = 0; m < ps; ++m) {
+                            val = p[m];
+                            f = format[m];
+
+                            if (f) {
+                                if (f.number && val != null) {
+                                    val = +val; // convert to number
+                                    if (isNaN(val))
+                                        val = null;
+                                    else if (val == Infinity)
+                                        val = fakeInfinity;
+                                    else if (val == -Infinity)
+                                        val = -fakeInfinity;
+                                }
+
+                                if (val == null) {
+                                    if (f.required)
+                                        nullify = true;
+
+                                    if (f.defaultValue != null)
+                                        val = f.defaultValue;
+                                }
+                            }
+
+                            points[k + m] = val;
+                        }
+                    }
+
+                    if (nullify) {
+                        for (m = 0; m < ps; ++m) {
+                            val = points[k + m];
+                            if (val != null) {
+                                f = format[m];
+                                // extract min/max info
+                                if (f.autoscale !== false) {
+                                    if (f.x) {
+                                        updateAxis(s.xaxis, val, val);
+                                    }
+                                    if (f.y) {
+                                        updateAxis(s.yaxis, val, val);
+                                    }
+                                }
+                            }
+                            points[k + m] = null;
+                        }
+                    }
+                    else {
+                        // a little bit of line specific stuff that
+                        // perhaps shouldn't be here, but lacking
+                        // better means...
+                        if (insertSteps && k > 0
+                            && points[k - ps] != null
+                            && points[k - ps] != points[k]
+                            && points[k - ps + 1] != points[k + 1]) {
+                            // copy the point to make room for a middle point
+                            for (m = 0; m < ps; ++m)
+                                points[k + ps + m] = points[k + m];
+
+                            // middle point has same y
+                            points[k + 1] = points[k - ps + 1];
+
+                            // we've added a point, better reflect that
+                            k += ps;
+                        }
+                    }
+                }
+            }
+
+            // give the hooks a chance to run
+            for (i = 0; i < series.length; ++i) {
+                s = series[i];
+
+                executeHooks(hooks.processDatapoints, [ s, s.datapoints]);
+            }
+
+            // second pass: find datamax/datamin for auto-scaling
+            for (i = 0; i < series.length; ++i) {
+                s = series[i];
+                points = s.datapoints.points;
+                ps = s.datapoints.pointsize;
+                format = s.datapoints.format;
+
+                var xmin = topSentry, ymin = topSentry,
+                    xmax = bottomSentry, ymax = bottomSentry;
+
+                for (j = 0; j < points.length; j += ps) {
+                    if (points[j] == null)
+                        continue;
+
+                    for (m = 0; m < ps; ++m) {
+                        val = points[j + m];
+                        f = format[m];
+                        if (!f || f.autoscale === false || val == fakeInfinity || val == -fakeInfinity)
+                            continue;
+
+                        if (f.x) {
+                            if (val < xmin)
+                                xmin = val;
+                            if (val > xmax)
+                                xmax = val;
+                        }
+                        if (f.y) {
+                            if (val < ymin)
+                                ymin = val;
+                            if (val > ymax)
+                                ymax = val;
+                        }
+                    }
+                }
+
+                if (s.bars.show) {
+                    // make sure we got room for the bar on the dancing floor
+                    var delta;
+
+                    switch (s.bars.align) {
+                        case "left":
+                            delta = 0;
+                            break;
+                        case "right":
+                            delta = -s.bars.barWidth;
+                            break;
+                        default:
+                            delta = -s.bars.barWidth / 2;
+                    }
+
+                    if (s.bars.horizontal) {
+                        ymin += delta;
+                        ymax += delta + s.bars.barWidth;
+                    }
+                    else {
+                        xmin += delta;
+                        xmax += delta + s.bars.barWidth;
+                    }
+                }
+
+                updateAxis(s.xaxis, xmin, xmax);
+                updateAxis(s.yaxis, ymin, ymax);
+            }
+
+            $.each(allAxes(), function (_, axis) {
+                if (axis.datamin == topSentry)
+                    axis.datamin = null;
+                if (axis.datamax == bottomSentry)
+                    axis.datamax = null;
+            });
+        }
+
+        function setupCanvases() {
+
+            // Make sure the placeholder is clear of everything except canvases
+            // from a previous plot in this container that we'll try to re-use.
+
+            placeholder.css("padding", 0) // padding messes up the positioning
+                .children().filter(function(){
+                    return !$(this).hasClass("flot-overlay") && !$(this).hasClass('flot-base');
+                }).remove();
+
+            if (placeholder.css("position") == 'static')
+                placeholder.css("position", "relative"); // for positioning labels and overlay
+
+            surface = new Canvas("flot-base", placeholder);
+            overlay = new Canvas("flot-overlay", placeholder); // overlay canvas for interactive features
+
+            ctx = surface.context;
+            octx = overlay.context;
+
+            // define which element we're listening for events on
+            eventHolder = $(overlay.element).unbind();
+
+            // If we're re-using a plot object, shut down the old one
+
+            var existing = placeholder.data("plot");
+
+            if (existing) {
+                existing.shutdown();
+                overlay.clear();
+            }
+
+            // save in case we get replotted
+            placeholder.data("plot", plot);
+        }
+
+        function bindEvents() {
+            // bind events
+            if (options.grid.hoverable) {
+                eventHolder.mousemove(onMouseMove);
+
+                // Use bind, rather than .mouseleave, because we officially
+                // still support jQuery 1.2.6, which doesn't define a shortcut
+                // for mouseenter or mouseleave.  This was a bug/oversight that
+                // was fixed somewhere around 1.3.x.  We can return to using
+                // .mouseleave when we drop support for 1.2.6.
+
+                eventHolder.bind("mouseleave", onMouseLeave);
+            }
+
+            if (options.grid.clickable)
+                eventHolder.click(onClick);
+
+            executeHooks(hooks.bindEvents, [eventHolder]);
+        }
+
+        function shutdown() {
+            if (redrawTimeout)
+                clearTimeout(redrawTimeout);
+
+            eventHolder.unbind("mousemove", onMouseMove);
+            eventHolder.unbind("mouseleave", onMouseLeave);
+            eventHolder.unbind("click", onClick);
+
+            executeHooks(hooks.shutdown, [eventHolder]);
+        }
+
+        function setTransformationHelpers(axis) {
+            // set helper functions on the axis, assumes plot area
+            // has been computed already
+
+            function identity(x) { return x; }
+
+            var s, m, t = axis.options.transform || identity,
+                it = axis.options.inverseTransform;
+
+            // precompute how much the axis is scaling a point
+            // in canvas space
+            if (axis.direction == "x") {
+                s = axis.scale = plotWidth / Math.abs(t(axis.max) - t(axis.min));
+                m = Math.min(t(axis.max), t(axis.min));
+            }
+            else {
+                s = axis.scale = plotHeight / Math.abs(t(axis.max) - t(axis.min));
+                s = -s;
+                m = Math.max(t(axis.max), t(axis.min));
+            }
+
+            // data point to canvas coordinate
+            if (t == identity) // slight optimization
+                axis.p2c = function (p) { return (p - m) * s; };
+            else
+                axis.p2c = function (p) { return (t(p) - m) * s; };
+            // canvas coordinate to data point
+            if (!it)
+                axis.c2p = function (c) { return m + c / s; };
+            else
+                axis.c2p = function (c) { return it(m + c / s); };
+        }
+
+        function measureTickLabels(axis) {
+
+            var opts = axis.options,
+                ticks = axis.ticks || [],
+                labelWidth = opts.labelWidth || 0,
+                labelHeight = opts.labelHeight || 0,
+                maxWidth = labelWidth || (axis.direction == "x" ? Math.floor(surface.width / (ticks.length || 1)) : null),
+                legacyStyles = axis.direction + "Axis " + axis.direction + axis.n + "Axis",
+                layer = "flot-" + axis.direction + "-axis flot-" + axis.direction + axis.n + "-axis " + legacyStyles,
+                font = opts.font || "flot-tick-label tickLabel";
+
+            for (var i = 0; i < ticks.length; ++i) {
+
+                var t = ticks[i];
+
+                if (!t.label)
+                    continue;
+
+                var info = surface.getTextInfo(layer, t.label, font, null, maxWidth);
+
+                labelWidth = Math.max(labelWidth, info.width);
+                labelHeight = Math.max(labelHeight, info.height);
+            }
+
+            axis.labelWidth = opts.labelWidth || labelWidth;
+            axis.labelHeight = opts.labelHeight || labelHeight;
+        }
+
+        function allocateAxisBoxFirstPhase(axis) {
+            // find the bounding box of the axis by looking at label
+            // widths/heights and ticks, make room by diminishing the
+            // plotOffset; this first phase only looks at one
+            // dimension per axis, the other dimension depends on the
+            // other axes so will have to wait
+
+            var lw = axis.labelWidth,
+                lh = axis.labelHeight,
+                pos = axis.options.position,
+                isXAxis = axis.direction === "x",
+                tickLength = axis.options.tickLength,
+                axisMargin = options.grid.axisMargin,
+                padding = options.grid.labelMargin,
+                innermost = true,
+                outermost = true,
+                first = true,
+                found = false;
+
+            // Determine the axis's position in its direction and on its side
+
+            $.each(isXAxis ? xaxes : yaxes, function(i, a) {
+                if (a && (a.show || a.reserveSpace)) {
+                    if (a === axis) {
+                        found = true;
+                    } else if (a.options.position === pos) {
+                        if (found) {
+                            outermost = false;
+                        } else {
+                            innermost = false;
+                        }
+                    }
+                    if (!found) {
+                        first = false;
+                    }
+                }
+            });
+
+            // The outermost axis on each side has no margin
+
+            if (outermost) {
+                axisMargin = 0;
+            }
+
+            // The ticks for the first axis in each direction stretch across
+
+            if (tickLength == null) {
+                tickLength = first ? "full" : 5;
+            }
+
+            if (!isNaN(+tickLength))
+                padding += +tickLength;
+
+            if (isXAxis) {
+                lh += padding;
+
+                if (pos == "bottom") {
+                    plotOffset.bottom += lh + axisMargin;
+                    axis.box = { top: surface.height - plotOffset.bottom, height: lh };
+                }
+                else {
+                    axis.box = { top: plotOffset.top + axisMargin, height: lh };
+                    plotOffset.top += lh + axisMargin;
+                }
+            }
+            else {
+                lw += padding;
+
+                if (pos == "left") {
+                    axis.box = { left: plotOffset.left + axisMargin, width: lw };
+                    plotOffset.left += lw + axisMargin;
+                }
+                else {
+                    plotOffset.right += lw + axisMargin;
+                    axis.box = { left: surface.width - plotOffset.right, width: lw };
+                }
+            }
+
+             // save for future reference
+            axis.position = pos;
+            axis.tickLength = tickLength;
+            axis.box.padding = padding;
+            axis.innermost = innermost;
+        }
+
+        function allocateAxisBoxSecondPhase(axis) {
+            // now that all axis boxes have been placed in one
+            // dimension, we can set the remaining dimension coordinates
+            if (axis.direction == "x") {
+                axis.box.left = plotOffset.left - axis.labelWidth / 2;
+                axis.box.width = surface.width - plotOffset.left - plotOffset.right + axis.labelWidth;
+            }
+            else {
+                axis.box.top = plotOffset.top - axis.labelHeight / 2;
+                axis.box.height = surface.height - plotOffset.bottom - plotOffset.top + axis.labelHeight;
+            }
+        }
+
+        function adjustLayoutForThingsStickingOut() {
+            // possibly adjust plot offset to ensure everything stays
+            // inside the canvas and isn't clipped off
+
+            var minMargin = options.grid.minBorderMargin,
+                axis, i;
+
+            // check stuff from the plot (FIXME: this should just read
+            // a value from the series, otherwise it's impossible to
+            // customize)
+            if (minMargin == null) {
+                minMargin = 0;
+                for (i = 0; i < series.length; ++i)
+                    minMargin = Math.max(minMargin, 2 * (series[i].points.radius + series[i].points.lineWidth/2));
+            }
+
+            var margins = {
+                left: minMargin,
+                right: minMargin,
+                top: minMargin,
+                bottom: minMargin
+            };
+
+            // check axis labels, note we don't check the actual
+            // labels but instead use the overall width/height to not
+            // jump as much around with replots
+            $.each(allAxes(), function (_, axis) {
+                if (axis.reserveSpace && axis.ticks && axis.ticks.length) {
+                    if (axis.direction === "x") {
+                        margins.left = Math.max(margins.left, axis.labelWidth / 2);
+                        margins.right = Math.max(margins.right, axis.labelWidth / 2);
+                    } else {
+                        margins.bottom = Math.max(margins.bottom, axis.labelHeight / 2);
+                        margins.top = Math.max(margins.top, axis.labelHeight / 2);
+                    }
+                }
+            });
+
+            plotOffset.left = Math.ceil(Math.max(margins.left, plotOffset.left));
+            plotOffset.right = Math.ceil(Math.max(margins.right, plotOffset.right));
+            plotOffset.top = Math.ceil(Math.max(margins.top, plotOffset.top));
+            plotOffset.bottom = Math.ceil(Math.max(margins.bottom, plotOffset.bottom));
+        }
+
+        function setupGrid() {
+            var i, axes = allAxes(), showGrid = options.grid.show;
+
+            // Initialize the plot's offset from the edge of the canvas
+
+            for (var a in plotOffset) {
+                var margin = options.grid.margin || 0;
+                plotOffset[a] = typeof margin == "number" ? margin : margin[a] || 0;
+            }
+
+            executeHooks(hooks.processOffset, [plotOffset]);
+
+            // If the grid is visible, add its border width to the offset
+
+            for (var a in plotOffset) {
+                if(typeof(options.grid.borderWidth) == "object") {
+                    plotOffset[a] += showGrid ? options.grid.borderWidth[a] : 0;
+                }
+                else {
+                    plotOffset[a] += showGrid ? options.grid.borderWidth : 0;
+                }
+            }
+
+            $.each(axes, function (_, axis) {
+                var axisOpts = axis.options;
+                axis.show = axisOpts.show == null ? axis.used : axisOpts.show;
+                axis.reserveSpace = axisOpts.reserveSpace == null ? axis.show : axisOpts.reserveSpace;
+                setRange(axis);
+            });
+
+            if (showGrid) {
+
+                var allocatedAxes = $.grep(axes, function (axis) {
+                    return axis.show || axis.reserveSpace;
+                });
+
+                $.each(allocatedAxes, function (_, axis) {
+                    // make the ticks
+                    setupTickGeneration(axis);
+                    setTicks(axis);
+                    snapRangeToTicks(axis, axis.ticks);
+                    // find labelWidth/Height for axis
+                    measureTickLabels(axis);
+                });
+
+                // with all dimensions calculated, we can compute the
+                // axis bounding boxes, start from the outside
+                // (reverse order)
+                for (i = allocatedAxes.length - 1; i >= 0; --i)
+                    allocateAxisBoxFirstPhase(allocatedAxes[i]);
+
+                // make sure we've got enough space for things that
+                // might stick out
+                adjustLayoutForThingsStickingOut();
+
+                $.each(allocatedAxes, function (_, axis) {
+                    allocateAxisBoxSecondPhase(axis);
+                });
+            }
+
+            plotWidth = surface.width - plotOffset.left - plotOffset.right;
+            plotHeight = surface.height - plotOffset.bottom - plotOffset.top;
+
+            // now we got the proper plot dimensions, we can compute the scaling
+            $.each(axes, function (_, axis) {
+                setTransformationHelpers(axis);
+            });
+
+            if (showGrid) {
+                drawAxisLabels();
+            }
+
+            insertLegend();
+        }
+
+        function setRange(axis) {
+            var opts = axis.options,
+                min = +(opts.min != null ? opts.min : axis.datamin),
+                max = +(opts.max != null ? opts.max : axis.datamax),
+                delta = max - min;
+
+            if (delta == 0.0) {
+                // degenerate case
+                var widen = max == 0 ? 1 : 0.01;
+
+                if (opts.min == null)
+                    min -= widen;
+                // always widen max if we couldn't widen min to ensure we
+                // don't fall into min == max which doesn't work
+                if (opts.max == null || opts.min != null)
+                    max += widen;
+            }
+            else {
+                // consider autoscaling
+                var margin = opts.autoscaleMargin;
+                if (margin != null) {
+                    if (opts.min == null) {
+                        min -= delta * margin;
+                        // make sure we don't go below zero if all values
+                        // are positive
+                        if (min < 0 && axis.datamin != null && axis.datamin >= 0)
+                            min = 0;
+                    }
+                    if (opts.max == null) {
+                        max += delta * margin;
+                        if (max > 0 && axis.datamax != null && axis.datamax <= 0)
+                            max = 0;
+                    }
+                }
+            }
+            axis.min = min;
+            axis.max = max;
+        }
+
+        function setupTickGeneration(axis) {
+            var opts = axis.options;
+
+            // estimate number of ticks
+            var noTicks;
+            if (typeof opts.ticks == "number" && opts.ticks > 0)
+                noTicks = opts.ticks;
+            else
+                // heuristic based on the model a*sqrt(x) fitted to
+                // some data points that seemed reasonable
+                noTicks = 0.3 * Math.sqrt(axis.direction == "x" ? surface.width : surface.height);
+
+            var delta = (axis.max - axis.min) / noTicks,
+                dec = -Math.floor(Math.log(delta) / Math.LN10),
+                maxDec = opts.tickDecimals;
+
+            if (maxDec != null && dec > maxDec) {
+                dec = maxDec;
+            }
+
+            var magn = Math.pow(10, -dec),
+                norm = delta / magn, // norm is between 1.0 and 10.0
+                size;
+
+            if (norm < 1.5) {
+                size = 1;
+            } else if (norm < 3) {
+                size = 2;
+                // special case for 2.5, requires an extra decimal
+                if (norm > 2.25 && (maxDec == null || dec + 1 <= maxDec)) {
+                    size = 2.5;
+                    ++dec;
+                }
+            } else if (norm < 7.5) {
+                size = 5;
+            } else {
+                size = 10;
+            }
+
+            size *= magn;
+
+            if (opts.minTickSize != null && size < opts.minTickSize) {
+                size = opts.minTickSize;
+            }
+
+            axis.delta = delta;
+            axis.tickDecimals = Math.max(0, maxDec != null ? maxDec : dec);
+            axis.tickSize = opts.tickSize || size;
+
+            // Time mode was moved to a plug-in in 0.8, and since so many people use it
+            // we'll add an especially friendly reminder to make sure they included it.
+
+            if (opts.mode == "time" && !axis.tickGenerator) {
+                throw new Error("Time mode requires the flot.time plugin.");
+            }
+
+            // Flot supports base-10 axes; any other mode else is handled by a plug-in,
+            // like flot.time.js.
+
+            if (!axis.tickGenerator) {
+
+                axis.tickGenerator = function (axis) {
+
+                    var ticks = [],
+                        start = floorInBase(axis.min, axis.tickSize),
+                        i = 0,
+                        v = Number.NaN,
+                        prev;
+
+                    do {
+                        prev = v;
+                        v = start + i * axis.tickSize;
+                        ticks.push(v);
+                        ++i;
+                    } while (v < axis.max && v != prev);
+                    return ticks;
+                };
+
+				axis.tickFormatter = function (value, axis) {
+
+					var factor = axis.tickDecimals ? Math.pow(10, axis.tickDecimals) : 1;
+					var formatted = "" + Math.round(value * factor) / factor;
+
+					// If tickDecimals was specified, ensure that we have exactly that
+					// much precision; otherwise default to the value's own precision.
+
+					if (axis.tickDecimals != null) {
+						var decimal = formatted.indexOf(".");
+						var precision = decimal == -1 ? 0 : formatted.length - decimal - 1;
+						if (precision < axis.tickDecimals) {
+							return (precision ? formatted : formatted + ".") + ("" + factor).substr(1, axis.tickDecimals - precision);
+						}
+					}
+
+                    return formatted;
+                };
+            }
+
+            if ($.isFunction(opts.tickFormatter))
+                axis.tickFormatter = function (v, axis) { return "" + opts.tickFormatter(v, axis); };
+
+            if (opts.alignTicksWithAxis != null) {
+                var otherAxis = (axis.direction == "x" ? xaxes : yaxes)[opts.alignTicksWithAxis - 1];
+                if (otherAxis && otherAxis.used && otherAxis != axis) {
+                    // consider snapping min/max to outermost nice ticks
+                    var niceTicks = axis.tickGenerator(axis);
+                    if (niceTicks.length > 0) {
+                        if (opts.min == null)
+                            axis.min = Math.min(axis.min, niceTicks[0]);
+                        if (opts.max == null && niceTicks.length > 1)
+                            axis.max = Math.max(axis.max, niceTicks[niceTicks.length - 1]);
+                    }
+
+                    axis.tickGenerator = function (axis) {
+                        // copy ticks, scaled to this axis
+                        var ticks = [], v, i;
+                        for (i = 0; i < otherAxis.ticks.length; ++i) {
+                            v = (otherAxis.ticks[i].v - otherAxis.min) / (otherAxis.max - otherAxis.min);
+                            v = axis.min + v * (axis.max - axis.min);
+                            ticks.push(v);
+                        }
+                        return ticks;
+                    };
+
+                    // we might need an extra decimal since forced
+                    // ticks don't necessarily fit naturally
+                    if (!axis.mode && opts.tickDecimals == null) {
+                        var extraDec = Math.max(0, -Math.floor(Math.log(axis.delta) / Math.LN10) + 1),
+                            ts = axis.tickGenerator(axis);
+
+                        // only proceed if the tick interval rounded
+                        // with an extra decimal doesn't give us a
+                        // zero at end
+                        if (!(ts.length > 1 && /\..*0$/.test((ts[1] - ts[0]).toFixed(extraDec))))
+                            axis.tickDecimals = extraDec;
+                    }
+                }
+            }
+        }
+
+        function setTicks(axis) {
+            var oticks = axis.options.ticks, ticks = [];
+            if (oticks == null || (typeof oticks == "number" && oticks > 0))
+                ticks = axis.tickGenerator(axis);
+            else if (oticks) {
+                if ($.isFunction(oticks))
+                    // generate the ticks
+                    ticks = oticks(axis);
+                else
+                    ticks = oticks;
+            }
+
+            // clean up/labelify the supplied ticks, copy them over
+            var i, v;
+            axis.ticks = [];
+            for (i = 0; i < ticks.length; ++i) {
+                var label = null;
+                var t = ticks[i];
+                if (typeof t == "object") {
+                    v = +t[0];
+                    if (t.length > 1)
+                        label = t[1];
+                }
+                else
+                    v = +t;
+                if (label == null)
+                    label = axis.tickFormatter(v, axis);
+                if (!isNaN(v))
+                    axis.ticks.push({ v: v, label: label });
+            }
+        }
+
+        function snapRangeToTicks(axis, ticks) {
+            if (axis.options.autoscaleMargin && ticks.length > 0) {
+                // snap to ticks
+                if (axis.options.min == null)
+                    axis.min = Math.min(axis.min, ticks[0].v);
+                if (axis.options.max == null && ticks.length > 1)
+                    axis.max = Math.max(axis.max, ticks[ticks.length - 1].v);
+            }
+        }
+
+        function draw() {
+
+            surface.clear();
+
+            executeHooks(hooks.drawBackground, [ctx]);
+
+            var grid = options.grid;
+
+            // draw background, if any
+            if (grid.show && grid.backgroundColor)
+                drawBackground();
+
+            if (grid.show && !grid.aboveData) {
+                drawGrid();
+            }
+
+            for (var i = 0; i < series.length; ++i) {
+                executeHooks(hooks.drawSeries, [ctx, series[i]]);
+                drawSeries(series[i]);
+            }
+
+            executeHooks(hooks.draw, [ctx]);
+
+            if (grid.show && grid.aboveData) {
+                drawGrid();
+            }
+
+            surface.render();
+
+            // A draw implies that either the axes or data have changed, so we
+            // should probably update the overlay highlights as well.
+
+            triggerRedrawOverlay();
+        }
+
+        function extractRange(ranges, coord) {
+            var axis, from, to, key, axes = allAxes();
+
+            for (var i = 0; i < axes.length; ++i) {
+                axis = axes[i];
+                if (axis.direction == coord) {
+                    key = coord + axis.n + "axis";
+                    if (!ranges[key] && axis.n == 1)
+                        key = coord + "axis"; // support x1axis as xaxis
+                    if (ranges[key]) {
+                        from = ranges[key].from;
+                        to = ranges[key].to;
+                        break;
+                    }
+                }
+            }
+
+            // backwards-compat stuff - to be removed in future
+            if (!ranges[key]) {
+                axis = coord == "x" ? xaxes[0] : yaxes[0];
+                from = ranges[coord + "1"];
+                to = ranges[coord + "2"];
+            }
+
+            // auto-reverse as an added bonus
+            if (from != null && to != null && from > to) {
+                var tmp = from;
+                from = to;
+                to = tmp;
+            }
+
+            return { from: from, to: to, axis: axis };
+        }
+
+        function drawBackground() {
+            ctx.save();
+            ctx.translate(plotOffset.left, plotOffset.top);
+
+            ctx.fillStyle = getColorOrGradient(options.grid.backgroundColor, plotHeight, 0, "rgba(255, 255, 255, 0)");
+            ctx.fillRect(0, 0, plotWidth, plotHeight);
+            ctx.restore();
+        }
+
+        function drawGrid() {
+            var i, axes, bw, bc;
+
+            ctx.save();
+            ctx.translate(plotOffset.left, plotOffset.top);
+
+            // draw markings
+            var markings = options.grid.markings;
+            if (markings) {
+                if ($.isFunction(markings)) {
+                    axes = plot.getAxes();
+                    // xmin etc. is backwards compatibility, to be
+                    // removed in the future
+                    axes.xmin = axes.xaxis.min;
+                    axes.xmax = axes.xaxis.max;
+                    axes.ymin = axes.yaxis.min;
+                    axes.ymax = axes.yaxis.max;
+
+                    markings = markings(axes);
+                }
+
+                for (i = 0; i < markings.length; ++i) {
+                    var m = markings[i],
+                        xrange = extractRange(m, "x"),
+                        yrange = extractRange(m, "y");
+
+                    // fill in missing
+                    if (xrange.from == null)
+                        xrange.from = xrange.axis.min;
+                    if (xrange.to == null)
+                        xrange.to = xrange.axis.max;
+                    if (yrange.from == null)
+                        yrange.from = yrange.axis.min;
+                    if (yrange.to == null)
+                        yrange.to = yrange.axis.max;
+
+                    // clip
+                    if (xrange.to < xrange.axis.min || xrange.from > xrange.axis.max ||
+                        yrange.to < yrange.axis.min || yrange.from > yrange.axis.max)
+                        continue;
+
+                    xrange.from = Math.max(xrange.from, xrange.axis.min);
+                    xrange.to = Math.min(xrange.to, xrange.axis.max);
+                    yrange.from = Math.max(yrange.from, yrange.axis.min);
+                    yrange.to = Math.min(yrange.to, yrange.axis.max);
+
+                    var xequal = xrange.from === xrange.to,
+                        yequal = yrange.from === yrange.to;
+
+                    if (xequal && yequal) {
+                        continue;
+                    }
+
+                    // then draw
+                    xrange.from = Math.floor(xrange.axis.p2c(xrange.from));
+                    xrange.to = Math.floor(xrange.axis.p2c(xrange.to));
+                    yrange.from = Math.floor(yrange.axis.p2c(yrange.from));
+                    yrange.to = Math.floor(yrange.axis.p2c(yrange.to));
+
+                    if (xequal || yequal) {
+                        var lineWidth = m.lineWidth || options.grid.markingsLineWidth,
+                            subPixel = lineWidth % 2 ? 0.5 : 0;
+                        ctx.beginPath();
+                        ctx.strokeStyle = m.color || options.grid.markingsColor;
+                        ctx.lineWidth = lineWidth;
+                        if (xequal) {
+                            ctx.moveTo(xrange.to + subPixel, yrange.from);
+                            ctx.lineTo(xrange.to + subPixel, yrange.to);
+                        } else {
+                            ctx.moveTo(xrange.from, yrange.to + subPixel);
+                            ctx.lineTo(xrange.to, yrange.to + subPixel);                            
+                        }
+                        ctx.stroke();
+                    } else {
+                        ctx.fillStyle = m.color || options.grid.markingsColor;
+                        ctx.fillRect(xrange.from, yrange.to,
+                                     xrange.to - xrange.from,
+                                     yrange.from - yrange.to);
+                    }
+                }
+            }
+
+            // draw the ticks
+            axes = allAxes();
+            bw = options.grid.borderWidth;
+
+            for (var j = 0; j < axes.length; ++j) {
+                var axis = axes[j], box = axis.box,
+                    t = axis.tickLength, x, y, xoff, yoff;
+                if (!axis.show || axis.ticks.length == 0)
+                    continue;
+
+                ctx.lineWidth = 1;
+
+                // find the edges
+                if (axis.direction == "x") {
+                    x = 0;
+                    if (t == "full")
+                        y = (axis.position == "top" ? 0 : plotHeight);
+                    else
+                        y = box.top - plotOffset.top + (axis.position == "top" ? box.height : 0);
+                }
+                else {
+                    y = 0;
+                    if (t == "full")
+                        x = (axis.position == "left" ? 0 : plotWidth);
+                    else
+                        x = box.left - plotOffset.left + (axis.position == "left" ? box.width : 0);
+                }
+
+                // draw tick bar
+                if (!axis.innermost) {
+                    ctx.strokeStyle = axis.options.color;
+                    ctx.beginPath();
+                    xoff = yoff = 0;
+                    if (axis.direction == "x")
+                        xoff = plotWidth + 1;
+                    else
+                        yoff = plotHeight + 1;
+
+                    if (ctx.lineWidth == 1) {
+                        if (axis.direction == "x") {
+                            y = Math.floor(y) + 0.5;
+                        } else {
+                            x = Math.floor(x) + 0.5;
+                        }
+                    }
+
+                    ctx.moveTo(x, y);
+                    ctx.lineTo(x + xoff, y + yoff);
+                    ctx.stroke();
+                }
+
+                // draw ticks
+
+                ctx.strokeStyle = axis.options.tickColor;
+
+                ctx.beginPath();
+                for (i = 0; i < axis.ticks.length; ++i) {
+                    var v = axis.ticks[i].v;
+
+                    xoff = yoff = 0;
+
+                    if (isNaN(v) || v < axis.min || v > axis.max
+                        // skip those lying on the axes if we got a border
+                        || (t == "full"
+                            && ((typeof bw == "object" && bw[axis.position] > 0) || bw > 0)
+                            && (v == axis.min || v == axis.max)))
+                        continue;
+
+                    if (axis.direction == "x") {
+                        x = axis.p2c(v);
+                        yoff = t == "full" ? -plotHeight : t;
+
+                        if (axis.position == "top")
+                            yoff = -yoff;
+                    }
+                    else {
+                        y = axis.p2c(v);
+                        xoff = t == "full" ? -plotWidth : t;
+
+                        if (axis.position == "left")
+                            xoff = -xoff;
+                    }
+
+                    if (ctx.lineWidth == 1) {
+                        if (axis.direction == "x")
+                            x = Math.floor(x) + 0.5;
+                        else
+                            y = Math.floor(y) + 0.5;
+                    }
+
+                    ctx.moveTo(x, y);
+                    ctx.lineTo(x + xoff, y + yoff);
+                }
+
+                ctx.stroke();
+            }
+
+
+            // draw border
+            if (bw) {
+                // If either borderWidth or borderColor is an object, then draw the border
+                // line by line instead of as one rectangle
+                bc = options.grid.borderColor;
+                if(typeof bw == "object" || typeof bc == "object") {
+                    if (typeof bw !== "object") {
+                        bw = {top: bw, right: bw, bottom: bw, left: bw};
+                    }
+                    if (typeof bc !== "object") {
+                        bc = {top: bc, right: bc, bottom: bc, left: bc};
+                    }
+
+                    if (bw.top > 0) {
+                        ctx.strokeStyle = bc.top;
+                        ctx.lineWidth = bw.top;
+                        ctx.beginPath();
+                        ctx.moveTo(0 - bw.left, 0 - bw.top/2);
+                        ctx.lineTo(plotWidth, 0 - bw.top/2);
+                        ctx.stroke();
+                    }
+
+                    if (bw.right > 0) {
+                        ctx.strokeStyle = bc.right;
+                        ctx.lineWidth = bw.right;
+                        ctx.beginPath();
+                        ctx.moveTo(plotWidth + bw.right / 2, 0 - bw.top);
+                        ctx.lineTo(plotWidth + bw.right / 2, plotHeight);
+                        ctx.stroke();
+                    }
+
+                    if (bw.bottom > 0) {
+                        ctx.strokeStyle = bc.bottom;
+                        ctx.lineWidth = bw.bottom;
+                        ctx.beginPath();
+                        ctx.moveTo(plotWidth + bw.right, plotHeight + bw.bottom / 2);
+                        ctx.lineTo(0, plotHeight + bw.bottom / 2);
+                        ctx.stroke();
+                    }
+
+                    if (bw.left > 0) {
+                        ctx.strokeStyle = bc.left;
+                        ctx.lineWidth = bw.left;
+                        ctx.beginPath();
+                        ctx.moveTo(0 - bw.left/2, plotHeight + bw.bottom);
+                        ctx.lineTo(0- bw.left/2, 0);
+                        ctx.stroke();
+                    }
+                }
+                else {
+                    ctx.lineWidth = bw;
+                    ctx.strokeStyle = options.grid.borderColor;
+                    ctx.strokeRect(-bw/2, -bw/2, plotWidth + bw, plotHeight + bw);
+                }
+            }
+
+            ctx.restore();
+        }
+
+        function drawAxisLabels() {
+
+            $.each(allAxes(), function (_, axis) {
+                var box = axis.box,
+                    legacyStyles = axis.direction + "Axis " + axis.direction + axis.n + "Axis",
+                    layer = "flot-" + axis.direction + "-axis flot-" + axis.direction + axis.n + "-axis " + legacyStyles,
+                    font = axis.options.font || "flot-tick-label tickLabel",
+                    tick, x, y, halign, valign;
+
+                // Remove text before checking for axis.show and ticks.length;
+                // otherwise plugins, like flot-tickrotor, that draw their own
+                // tick labels will end up with both theirs and the defaults.
+
+                surface.removeText(layer);
+
+                if (!axis.show || axis.ticks.length == 0)
+                    return;
+
+                for (var i = 0; i < axis.ticks.length; ++i) {
+
+                    tick = axis.ticks[i];
+                    if (!tick.label || tick.v < axis.min || tick.v > axis.max)
+                        continue;
+
+                    if (axis.direction == "x") {
+                        halign = "center";
+                        x = plotOffset.left + axis.p2c(tick.v);
+                        if (axis.position == "bottom") {
+                            y = box.top + box.padding;
+                        } else {
+                            y = box.top + box.height - box.padding;
+                            valign = "bottom";
+                        }
+                    } else {
+                        valign = "middle";
+                        y = plotOffset.top + axis.p2c(tick.v);
+                        if (axis.position == "left") {
+                            x = box.left + box.width - box.padding;
+                            halign = "right";
+                        } else {
+                            x = box.left + box.padding;
+                        }
+                    }
+
+                    surface.addText(layer, x, y, tick.label, font, null, null, halign, valign);
+                }
+            });
+        }
+
+        function drawSeries(series) {
+            if (series.lines.show)
+                drawSeriesLines(series);
+            if (series.bars.show)
+                drawSeriesBars(series);
+            if (series.points.show)
+                drawSeriesPoints(series);
+        }
+
+        function drawSeriesLines(series) {
+            function plotLine(datapoints, xoffset, yoffset, axisx, axisy) {
+                var points = datapoints.points,
+                    ps = datapoints.pointsize,
+                    prevx = null, prevy = null;
+
+                ctx.beginPath();
+                for (var i = ps; i < points.length; i += ps) {
+                    var x1 = points[i - ps], y1 = points[i - ps + 1],
+                        x2 = points[i], y2 = points[i + 1];
+
+                    if (x1 == null || x2 == null)
+                        continue;
+
+                    // clip with ymin
+                    if (y1 <= y2 && y1 < axisy.min) {
+                        if (y2 < axisy.min)
+                            continue;   // line segment is outside
+                        // compute new intersection point
+                        x1 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1;
+                        y1 = axisy.min;
+                    }
+                    else if (y2 <= y1 && y2 < axisy.min) {
+                        if (y1 < axisy.min)
+                            continue;
+                        x2 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1;
+                        y2 = axisy.min;
+                    }
+
+                    // clip with ymax
+                    if (y1 >= y2 && y1 > axisy.max) {
+                        if (y2 > axisy.max)
+                            continue;
+                        x1 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1;
+                        y1 = axisy.max;
+                    }
+                    else if (y2 >= y1 && y2 > axisy.max) {
+                        if (y1 > axisy.max)
+                            continue;
+                        x2 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1;
+                        y2 = axisy.max;
+                    }
+
+                    // clip with xmin
+                    if (x1 <= x2 && x1 < axisx.min) {
+                        if (x2 < axisx.min)
+                            continue;
+                        y1 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1;
+                        x1 = axisx.min;
+                    }
+                    else if (x2 <= x1 && x2 < axisx.min) {
+                        if (x1 < axisx.min)
+                            continue;
+                        y2 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1;
+                        x2 = axisx.min;
+                    }
+
+                    // clip with xmax
+                    if (x1 >= x2 && x1 > axisx.max) {
+                        if (x2 > axisx.max)
+                            continue;
+                        y1 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1;
+                        x1 = axisx.max;
+                    }
+                    else if (x2 >= x1 && x2 > axisx.max) {
+                        if (x1 > axisx.max)
+                            continue;
+                        y2 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1;
+                        x2 = axisx.max;
+                    }
+
+                    if (x1 != prevx || y1 != prevy)
+                        ctx.moveTo(axisx.p2c(x1) + xoffset, axisy.p2c(y1) + yoffset);
+
+                    prevx = x2;
+                    prevy = y2;
+                    ctx.lineTo(axisx.p2c(x2) + xoffset, axisy.p2c(y2) + yoffset);
+                }
+                ctx.stroke();
+            }
+
+            function plotLineArea(datapoints, axisx, axisy) {
+                var points = datapoints.points,
+                    ps = datapoints.pointsize,
+                    bottom = Math.min(Math.max(0, axisy.min), axisy.max),
+                    i = 0, top, areaOpen = false,
+                    ypos = 1, segmentStart = 0, segmentEnd = 0;
+
+                // we process each segment in two turns, first forward
+                // direction to sketch out top, then once we hit the
+                // end we go backwards to sketch the bottom
+                while (true) {
+                    if (ps > 0 && i > points.length + ps)
+                        break;
+
+                    i += ps; // ps is negative if going backwards
+
+                    var x1 = points[i - ps],
+                        y1 = points[i - ps + ypos],
+                        x2 = points[i], y2 = points[i + ypos];
+
+                    if (areaOpen) {
+                        if (ps > 0 && x1 != null && x2 == null) {
+                            // at turning point
+                            segmentEnd = i;
+                            ps = -ps;
+                            ypos = 2;
+                            continue;
+                        }
+
+                        if (ps < 0 && i == segmentStart + ps) {
+                            // done with the reverse sweep
+                            ctx.fill();
+                            areaOpen = false;
+                            ps = -ps;
+                            ypos = 1;
+                            i = segmentStart = segmentEnd + ps;
+                            continue;
+                        }
+                    }
+
+                    if (x1 == null || x2 == null)
+                        continue;
+
+                    // clip x values
+
+                    // clip with xmin
+                    if (x1 <= x2 && x1 < axisx.min) {
+                        if (x2 < axisx.min)
+                            continue;
+                        y1 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1;
+                        x1 = axisx.min;
+                    }
+                    else if (x2 <= x1 && x2 < axisx.min) {
+                        if (x1 < axisx.min)
+                            continue;
+                        y2 = (axisx.min - x1) / (x2 - x1) * (y2 - y1) + y1;
+                        x2 = axisx.min;
+                    }
+
+                    // clip with xmax
+                    if (x1 >= x2 && x1 > axisx.max) {
+                        if (x2 > axisx.max)
+                            continue;
+                        y1 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1;
+                        x1 = axisx.max;
+                    }
+                    else if (x2 >= x1 && x2 > axisx.max) {
+                        if (x1 > axisx.max)
+                            continue;
+                        y2 = (axisx.max - x1) / (x2 - x1) * (y2 - y1) + y1;
+                        x2 = axisx.max;
+                    }
+
+                    if (!areaOpen) {
+                        // open area
+                        ctx.beginPath();
+                        ctx.moveTo(axisx.p2c(x1), axisy.p2c(bottom));
+                        areaOpen = true;
+                    }
+
+                    // now first check the case where both is outside
+                    if (y1 >= axisy.max && y2 >= axisy.max) {
+                        ctx.lineTo(axisx.p2c(x1), axisy.p2c(axisy.max));
+                        ctx.lineTo(axisx.p2c(x2), axisy.p2c(axisy.max));
+                        continue;
+                    }
+                    else if (y1 <= axisy.min && y2 <= axisy.min) {
+                        ctx.lineTo(axisx.p2c(x1), axisy.p2c(axisy.min));
+                        ctx.lineTo(axisx.p2c(x2), axisy.p2c(axisy.min));
+                        continue;
+                    }
+
+                    // else it's a bit more complicated, there might
+                    // be a flat maxed out rectangle first, then a
+                    // triangular cutout or reverse; to find these
+                    // keep track of the current x values
+                    var x1old = x1, x2old = x2;
+
+                    // clip the y values, without shortcutting, we
+                    // go through all cases in turn
+
+                    // clip with ymin
+                    if (y1 <= y2 && y1 < axisy.min && y2 >= axisy.min) {
+                        x1 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1;
+                        y1 = axisy.min;
+                    }
+                    else if (y2 <= y1 && y2 < axisy.min && y1 >= axisy.min) {
+                        x2 = (axisy.min - y1) / (y2 - y1) * (x2 - x1) + x1;
+                        y2 = axisy.min;
+                    }
+
+                    // clip with ymax
+                    if (y1 >= y2 && y1 > axisy.max && y2 <= axisy.max) {
+                        x1 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1;
+                        y1 = axisy.max;
+                    }
+                    else if (y2 >= y1 && y2 > axisy.max && y1 <= axisy.max) {
+                        x2 = (axisy.max - y1) / (y2 - y1) * (x2 - x1) + x1;
+                        y2 = axisy.max;
+                    }
+
+                    // if the x value was changed we got a rectangle
+                    // to fill
+                    if (x1 != x1old) {
+                        ctx.lineTo(axisx.p2c(x1old), axisy.p2c(y1));
+                        // it goes to (x1, y1), but we fill that below
+                    }
+
+                    // fill triangular section, this sometimes result
+                    // in redundant points if (x1, y1) hasn't changed
+                    // from previous line to, but we just ignore that
+                    ctx.lineTo(axisx.p2c(x1), axisy.p2c(y1));
+                    ctx.lineTo(axisx.p2c(x2), axisy.p2c(y2));
+
+                    // fill the other rectangle if it's there
+                    if (x2 != x2old) {
+                        ctx.lineTo(axisx.p2c(x2), axisy.p2c(y2));
+                        ctx.lineTo(axisx.p2c(x2old), axisy.p2c(y2));
+                    }
+                }
+            }
+
+            ctx.save();
+            ctx.translate(plotOffset.left, plotOffset.top);
+            ctx.lineJoin = "round";
+
+            var lw = series.lines.lineWidth,
+                sw = series.shadowSize;
+            // FIXME: consider another form of shadow when filling is turned on
+            if (lw > 0 && sw > 0) {
+                // draw shadow as a thick and thin line with transparency
+                ctx.lineWidth = sw;
+                ctx.strokeStyle = "rgba(0,0,0,0.1)";
+                // position shadow at angle from the mid of line
+                var angle = Math.PI/18;
+                plotLine(series.datapoints, Math.sin(angle) * (lw/2 + sw/2), Math.cos(angle) * (lw/2 + sw/2), series.xaxis, series.yaxis);
+                ctx.lineWidth = sw/2;
+                plotLine(series.datapoints, Math.sin(angle) * (lw/2 + sw/4), Math.cos(angle) * (lw/2 + sw/4), series.xaxis, series.yaxis);
+            }
+
+            ctx.lineWidth = lw;
+            ctx.strokeStyle = series.color;
+            var fillStyle = getFillStyle(series.lines, series.color, 0, plotHeight);
+            if (fillStyle) {
+                ctx.fillStyle = fillStyle;
+                plotLineArea(series.datapoints, series.xaxis, series.yaxis);
+            }
+
+            if (lw > 0)
+                plotLine(series.datapoints, 0, 0, series.xaxis, series.yaxis);
+            ctx.restore();
+        }
+
+        function drawSeriesPoints(series) {
+            function plotPoints(datapoints, radius, fillStyle, offset, shadow, axisx, axisy, symbol) {
+                var points = datapoints.points, ps = datapoints.pointsize;
+
+                for (var i = 0; i < points.length; i += ps) {
+                    var x = points[i], y = points[i + 1];
+                    if (x == null || x < axisx.min || x > axisx.max || y < axisy.min || y > axisy.max)
+                        continue;
+
+                    ctx.beginPath();
+                    x = axisx.p2c(x);
+                    y = axisy.p2c(y) + offset;
+                    if (symbol == "circle")
+                        ctx.arc(x, y, radius, 0, shadow ? Math.PI : Math.PI * 2, false);
+                    else
+                        symbol(ctx, x, y, radius, shadow);
+                    ctx.closePath();
+
+                    if (fillStyle) {
+                        ctx.fillStyle = fillStyle;
+                        ctx.fill();
+                    }
+                    ctx.stroke();
+                }
+            }
+
+            ctx.save();
+            ctx.translate(plotOffset.left, plotOffset.top);
+
+            var lw = series.points.lineWidth,
+                sw = series.shadowSize,
+                radius = series.points.radius,
+                symbol = series.points.symbol;
+
+            // If the user sets the line width to 0, we change it to a very 
+            // small value. A line width of 0 seems to force the default of 1.
+            // Doing the conditional here allows the shadow setting to still be 
+            // optional even with a lineWidth of 0.
+
+            if( lw == 0 )
+                lw = 0.0001;
+
+            if (lw > 0 && sw > 0) {
+                // draw shadow in two steps
+                var w = sw / 2;
+                ctx.lineWidth = w;
+                ctx.strokeStyle = "rgba(0,0,0,0.1)";
+                plotPoints(series.datapoints, radius, null, w + w/2, true,
+                           series.xaxis, series.yaxis, symbol);
+
+                ctx.strokeStyle = "rgba(0,0,0,0.2)";
+                plotPoints(series.datapoints, radius, null, w/2, true,
+                           series.xaxis, series.yaxis, symbol);
+            }
+
+            ctx.lineWidth = lw;
+            ctx.strokeStyle = series.color;
+            plotPoints(series.datapoints, radius,
+                       getFillStyle(series.points, series.color), 0, false,
+                       series.xaxis, series.yaxis, symbol);
+            ctx.restore();
+        }
+
+        function drawBar(x, y, b, barLeft, barRight, fillStyleCallback, axisx, axisy, c, horizontal, lineWidth) {
+            var left, right, bottom, top,
+                drawLeft, drawRight, drawTop, drawBottom,
+                tmp;
+
+            // in horizontal mode, we start the bar from the left
+            // instead of from the bottom so it appears to be
+            // horizontal rather than vertical
+            if (horizontal) {
+                drawBottom = drawRight = drawTop = true;
+                drawLeft = false;
+                left = b;
+                right = x;
+                top = y + barLeft;
+                bottom = y + barRight;
+
+                // account for negative bars
+                if (right < left) {
+                    tmp = right;
+                    right = left;
+                    left = tmp;
+                    drawLeft = true;
+                    drawRight = false;
+                }
+            }
+            else {
+                drawLeft = drawRight = drawTop = true;
+                drawBottom = false;
+                left = x + barLeft;
+                right = x + barRight;
+                bottom = b;
+                top = y;
+
+                // account for negative bars
+                if (top < bottom) {
+                    tmp = top;
+                    top = bottom;
+                    bottom = tmp;
+                    drawBottom = true;
+                    drawTop = false;
+                }
+            }
+
+            // clip
+            if (right < axisx.min || left > axisx.max ||
+                top < axisy.min || bottom > axisy.max)
+                return;
+
+            if (left < axisx.min) {
+                left = axisx.min;
+                drawLeft = false;
+            }
+
+            if (right > axisx.max) {
+                right = axisx.max;
+                drawRight = false;
+            }
+
+            if (bottom < axisy.min) {
+                bottom = axisy.min;
+                drawBottom = false;
+            }
+
+            if (top > axisy.max) {
+                top = axisy.max;
+                drawTop = false;
+            }
+
+            left = axisx.p2c(left);
+            bottom = axisy.p2c(bottom);
+            right = axisx.p2c(right);
+            top = axisy.p2c(top);
+
+            // fill the bar
+            if (fillStyleCallback) {
+                c.fillStyle = fillStyleCallback(bottom, top);
+                c.fillRect(left, top, right - left, bottom - top)
+            }
+
+            // draw outline
+            if (lineWidth > 0 && (drawLeft || drawRight || drawTop || drawBottom)) {
+                c.beginPath();
+
+                // FIXME: inline moveTo is buggy with excanvas
+                c.moveTo(left, bottom);
+                if (drawLeft)
+                    c.lineTo(left, top);
+                else
+                    c.moveTo(left, top);
+                if (drawTop)
+                    c.lineTo(right, top);
+                else
+                    c.moveTo(right, top);
+                if (drawRight)
+                    c.lineTo(right, bottom);
+                else
+                    c.moveTo(right, bottom);
+                if (drawBottom)
+                    c.lineTo(left, bottom);
+                else
+                    c.moveTo(left, bottom);
+                c.stroke();
+            }
+        }
+
+        function drawSeriesBars(series) {
+            function plotBars(datapoints, barLeft, barRight, fillStyleCallback, axisx, axisy) {
+                var points = datapoints.points, ps = datapoints.pointsize;
+
+                for (var i = 0; i < points.length; i += ps) {
+                    if (points[i] == null)
+                        continue;
+                    drawBar(points[i], points[i + 1], points[i + 2], barLeft, barRight, fillStyleCallback, axisx, axisy, ctx, series.bars.horizontal, series.bars.lineWidth);
+                }
+            }
+
+            ctx.save();
+            ctx.translate(plotOffset.left, plotOffset.top);
+
+            // FIXME: figure out a way to add shadows (for instance along the right edge)
+            ctx.lineWidth = series.bars.lineWidth;
+            ctx.strokeStyle = series.color;
+
+            var barLeft;
+
+            switch (series.bars.align) {
+                case "left":
+                    barLeft = 0;
+                    break;
+                case "right":
+                    barLeft = -series.bars.barWidth;
+                    break;
+                default:
+                    barLeft = -series.bars.barWidth / 2;
+            }
+
+            var fillStyleCallback = series.bars.fill ? function (bottom, top) { return getFillStyle(series.bars, series.color, bottom, top); } : null;
+            plotBars(series.datapoints, barLeft, barLeft + series.bars.barWidth, fillStyleCallback, series.xaxis, series.yaxis);
+            ctx.restore();
+        }
+
+        function getFillStyle(filloptions, seriesColor, bottom, top) {
+            var fill = filloptions.fill;
+            if (!fill)
+                return null;
+
+            if (filloptions.fillColor)
+                return getColorOrGradient(filloptions.fillColor, bottom, top, seriesColor);
+
+            var c = $.color.parse(seriesColor);
+            c.a = typeof fill == "number" ? fill : 0.4;
+            c.normalize();
+            return c.toString();
+        }
+
+        function insertLegend() {
+
+            if (options.legend.container != null) {
+                $(options.legend.container).html("");
+            } else {
+                placeholder.find(".legend").remove();
+            }
+
+            if (!options.legend.show) {
+                return;
+            }
+
+            var fragments = [], entries = [], rowStarted = false,
+                lf = options.legend.labelFormatter, s, label;
+
+            // Build a list of legend entries, with each having a label and a color
+
+            for (var i = 0; i < series.length; ++i) {
+                s = series[i];
+                if (s.label) {
+                    label = lf ? lf(s.label, s) : s.label;
+                    if (label) {
+                        entries.push({
+                            label: label,
+                            color: s.color
+                        });
+                    }
+                }
+            }
+
+            // Sort the legend using either the default or a custom comparator
+
+            if (options.legend.sorted) {
+                if ($.isFunction(options.legend.sorted)) {
+                    entries.sort(options.legend.sorted);
+                } else if (options.legend.sorted == "reverse") {
+                	entries.reverse();
+                } else {
+                    var ascending = options.legend.sorted != "descending";
+                    entries.sort(function(a, b) {
+                        return a.label == b.label ? 0 : (
+                            (a.label < b.label) != ascending ? 1 : -1   // Logical XOR
+                        );
+                    });
+                }
+            }
+
+            // Generate markup for the list of entries, in their final order
+
+            for (var i = 0; i < entries.length; ++i) {
+
+                var entry = entries[i];
+
+                if (i % options.legend.noColumns == 0) {
+                    if (rowStarted)
+                        fragments.push('</tr>');
+                    fragments.push('<tr>');
+                    rowStarted = true;
+                }
+
+                fragments.push(
+                    '<td class="legendColorBox"><div style="border:1px solid ' + options.legend.labelBoxBorderColor + ';padding:1px"><div style="width:4px;height:0;border:5px solid ' + entry.color + ';overflow:hidden"></div></div></td>' +
+                    '<td class="legendLabel">' + entry.label + '</td>'
+                );
+            }
+
+            if (rowStarted)
+                fragments.push('</tr>');
+
+            if (fragments.length == 0)
+                return;
+
+            var table = '<table style="font-size:smaller;color:' + options.grid.color + '">' + fragments.join("") + '</table>';
+            if (options.legend.container != null)
+                $(options.legend.container).html(table);
+            else {
+                var pos = "",
+                    p = options.legend.position,
+                    m = options.legend.margin;
+                if (m[0] == null)
+                    m = [m, m];
+                if (p.charAt(0) == "n")
+                    pos += 'top:' + (m[1] + plotOffset.top) + 'px;';
+                else if (p.charAt(0) == "s")
+                    pos += 'bottom:' + (m[1] + plotOffset.bottom) + 'px;';
+                if (p.charAt(1) == "e")
+                    pos += 'right:' + (m[0] + plotOffset.right) + 'px;';
+                else if (p.charAt(1) == "w")
+                    pos += 'left:' + (m[0] + plotOffset.left) + 'px;';
+                var legend = $('<div class="legend">' + table.replace('style="', 'style="position:absolute;' + pos +';') + '</div>').appendTo(placeholder);
+                if (options.legend.backgroundOpacity != 0.0) {
+                    // put in the transparent background
+                    // separately to avoid blended labels and
+                    // label boxes
+                    var c = options.legend.backgroundColor;
+                    if (c == null) {
+                        c = options.grid.backgroundColor;
+                        if (c && typeof c == "string")
+                            c = $.color.parse(c);
+                        else
+                            c = $.color.extract(legend, 'background-color');
+                        c.a = 1;
+                        c = c.toString();
+                    }
+                    var div = legend.children();
+                    $('<div style="position:absolute;width:' + div.width() + 'px;height:' + div.height() + 'px;' + pos +'background-color:' + c + ';"> </div>').prependTo(legend).css('opacity', options.legend.backgroundOpacity);
+                }
+            }
+        }
+
+
+        // interactive features
+
+        var highlights = [],
+            redrawTimeout = null;
+
+        // returns the data item the mouse is over, or null if none is found
+        function findNearbyItem(mouseX, mouseY, seriesFilter) {
+            var maxDistance = options.grid.mouseActiveRadius,
+                smallestDistance = maxDistance * maxDistance + 1,
+                item = null, foundPoint = false, i, j, ps;
+
+            for (i = series.length - 1; i >= 0; --i) {
+                if (!seriesFilter(series[i]))
+                    continue;
+
+                var s = series[i],
+                    axisx = s.xaxis,
+                    axisy = s.yaxis,
+                    points = s.datapoints.points,
+                    mx = axisx.c2p(mouseX), // precompute some stuff to make the loop faster
+                    my = axisy.c2p(mouseY),
+                    maxx = maxDistance / axisx.scale,
+                    maxy = maxDistance / axisy.scale;
+
+                ps = s.datapoints.pointsize;
+                // with inverse transforms, we can't use the maxx/maxy
+                // optimization, sadly
+                if (axisx.options.inverseTransform)
+                    maxx = Number.MAX_VALUE;
+                if (axisy.options.inverseTransform)
+                    maxy = Number.MAX_VALUE;
+
+                if (s.lines.show || s.points.show) {
+                    for (j = 0; j < points.length; j += ps) {
+                        var x = points[j], y = points[j + 1];
+                        if (x == null)
+                            continue;
+
+                        // For points and lines, the cursor must be within a
+                        // certain distance to the data point
+                        if (x - mx > maxx || x - mx < -maxx ||
+                            y - my > maxy || y - my < -maxy)
+                            continue;
+
+                        // We have to calculate distances in pixels, not in
+                        // data units, because the scales of the axes may be different
+                        var dx = Math.abs(axisx.p2c(x) - mouseX),
+                            dy = Math.abs(axisy.p2c(y) - mouseY),
+                            dist = dx * dx + dy * dy; // we save the sqrt
+
+                        // use <= to ensure last point takes precedence
+                        // (last generally means on top of)
+                        if (dist < smallestDistance) {
+                            smallestDistance = dist;
+                            item = [i, j / ps];
+                        }
+                    }
+                }
+
+                if (s.bars.show && !item) { // no other point can be nearby
+
+                    var barLeft, barRight;
+
+                    switch (s.bars.align) {
+                        case "left":
+                            barLeft = 0;
+                            break;
+                        case "right":
+                            barLeft = -s.bars.barWidth;
+                            break;
+                        default:
+                            barLeft = -s.bars.barWidth / 2;
+                    }
+
+                    barRight = barLeft + s.bars.barWidth;
+
+                    for (j = 0; j < points.length; j += ps) {
+                        var x = points[j], y = points[j + 1], b = points[j + 2];
+                        if (x == null)
+                            continue;
+
+                        // for a bar graph, the cursor must be inside the bar
+                        if (series[i].bars.horizontal ?
+                            (mx <= Math.max(b, x) && mx >= Math.min(b, x) &&
+                             my >= y + barLeft && my <= y + barRight) :
+                            (mx >= x + barLeft && mx <= x + barRight &&
+                             my >= Math.min(b, y) && my <= Math.max(b, y)))
+                                item = [i, j / ps];
+                    }
+                }
+            }
+
+            if (item) {
+                i = item[0];
+                j = item[1];
+                ps = series[i].datapoints.pointsize;
+
+                return { datapoint: series[i].datapoints.points.slice(j * ps, (j + 1) * ps),
+                         dataIndex: j,
+                         series: series[i],
+                         seriesIndex: i };
+            }
+
+            return null;
+        }
+
+        function onMouseMove(e) {
+            if (options.grid.hoverable)
+                triggerClickHoverEvent("plothover", e,
+                                       function (s) { return s["hoverable"] != false; });
+        }
+
+        function onMouseLeave(e) {
+            if (options.grid.hoverable)
+                triggerClickHoverEvent("plothover", e,
+                                       function (s) { return false; });
+        }
+
+        function onClick(e) {
+            triggerClickHoverEvent("plotclick", e,
+                                   function (s) { return s["clickable"] != false; });
+        }
+
+        // trigger click or hover event (they send the same parameters
+        // so we share their code)
+        function triggerClickHoverEvent(eventname, event, seriesFilter) {
+            var offset = eventHolder.offset(),
+                canvasX = event.pageX - offset.left - plotOffset.left,
+                canvasY = event.pageY - offset.top - plotOffset.top,
+            pos = canvasToAxisCoords({ left: canvasX, top: canvasY });
+
+            pos.pageX = event.pageX;
+            pos.pageY = event.pageY;
+
+            var item = findNearbyItem(canvasX, canvasY, seriesFilter);
+
+            if (item) {
+                // fill in mouse pos for any listeners out there
+                item.pageX = parseInt(item.series.xaxis.p2c(item.datapoint[0]) + offset.left + plotOffset.left, 10);
+                item.pageY = parseInt(item.series.yaxis.p2c(item.datapoint[1]) + offset.top + plotOffset.top, 10);
+            }
+
+            if (options.grid.autoHighlight) {
+                // clear auto-highlights
+                for (var i = 0; i < highlights.length; ++i) {
+                    var h = highlights[i];
+                    if (h.auto == eventname &&
+                        !(item && h.series == item.series &&
+                          h.point[0] == item.datapoint[0] &&
+                          h.point[1] == item.datapoint[1]))
+                        unhighlight(h.series, h.point);
+                }
+
+                if (item)
+                    highlight(item.series, item.datapoint, eventname);
+            }
+
+            placeholder.trigger(eventname, [ pos, item ]);
+        }
+
+        function triggerRedrawOverlay() {
+            var t = options.interaction.redrawOverlayInterval;
+            if (t == -1) {      // skip event queue
+                drawOverlay();
+                return;
+            }
+
+            if (!redrawTimeout)
+                redrawTimeout = setTimeout(drawOverlay, t);
+        }
+
+        function drawOverlay() {
+            redrawTimeout = null;
+
+            // draw highlights
+            octx.save();
+            overlay.clear();
+            octx.translate(plotOffset.left, plotOffset.top);
+
+            var i, hi;
+            for (i = 0; i < highlights.length; ++i) {
+                hi = highlights[i];
+
+                if (hi.series.bars.show)
+                    drawBarHighlight(hi.series, hi.point);
+                else
+                    drawPointHighlight(hi.series, hi.point);
+            }
+            octx.restore();
+
+            executeHooks(hooks.drawOverlay, [octx]);
+        }
+
+        function highlight(s, point, auto) {
+            if (typeof s == "number")
+                s = series[s];
+
+            if (typeof point == "number") {
+                var ps = s.datapoints.pointsize;
+                point = s.datapoints.points.slice(ps * point, ps * (point + 1));
+            }
+
+            var i = indexOfHighlight(s, point);
+            if (i == -1) {
+                highlights.push({ series: s, point: point, auto: auto });
+
+                triggerRedrawOverlay();
+            }
+            else if (!auto)
+                highlights[i].auto = false;
+        }
+
+        function unhighlight(s, point) {
+            if (s == null && point == null) {
+                highlights = [];
+                triggerRedrawOverlay();
+                return;
+            }
+
+            if (typeof s == "number")
+                s = series[s];
+
+            if (typeof point == "number") {
+                var ps = s.datapoints.pointsize;
+                point = s.datapoints.points.slice(ps * point, ps * (point + 1));
+            }
+
+            var i = indexOfHighlight(s, point);
+            if (i != -1) {
+                highlights.splice(i, 1);
+
+                triggerRedrawOverlay();
+            }
+        }
+
+        function indexOfHighlight(s, p) {
+            for (var i = 0; i < highlights.length; ++i) {
+                var h = highlights[i];
+                if (h.series == s && h.point[0] == p[0]
+                    && h.point[1] == p[1])
+                    return i;
+            }
+            return -1;
+        }
+
+        function drawPointHighlight(series, point) {
+            var x = point[0], y = point[1],
+                axisx = series.xaxis, axisy = series.yaxis,
+                highlightColor = (typeof series.highlightColor === "string") ? series.highlightColor : $.color.parse(series.color).scale('a', 0.5).toString();
+
+            if (x < axisx.min || x > axisx.max || y < axisy.min || y > axisy.max)
+                return;
+
+            var pointRadius = series.points.radius + series.points.lineWidth / 2;
+            octx.lineWidth = pointRadius;
+            octx.strokeStyle = highlightColor;
+            var radius = 1.5 * pointRadius;
+            x = axisx.p2c(x);
+            y = axisy.p2c(y);
+
+            octx.beginPath();
+            if (series.points.symbol == "circle")
+                octx.arc(x, y, radius, 0, 2 * Math.PI, false);
+            else
+                series.points.symbol(octx, x, y, radius, false);
+            octx.closePath();
+            octx.stroke();
+        }
+
+        function drawBarHighlight(series, point) {
+            var highlightColor = (typeof series.highlightColor === "string") ? series.highlightColor : $.color.parse(series.color).scale('a', 0.5).toString(),
+                fillStyle = highlightColor,
+                barLeft;
+
+            switch (series.bars.align) {
+                case "left":
+                    barLeft = 0;
+                    break;
+                case "right":
+                    barLeft = -series.bars.barWidth;
+                    break;
+                default:
+                    barLeft = -series.bars.barWidth / 2;
+            }
+
+            octx.lineWidth = series.bars.lineWidth;
+            octx.strokeStyle = highlightColor;
+
+            drawBar(point[0], point[1], point[2] || 0, barLeft, barLeft + series.bars.barWidth,
+                    function () { return fillStyle; }, series.xaxis, series.yaxis, octx, series.bars.horizontal, series.bars.lineWidth);
+        }
+
+        function getColorOrGradient(spec, bottom, top, defaultColor) {
+            if (typeof spec == "string")
+                return spec;
+            else {
+                // assume this is a gradient spec; IE currently only
+                // supports a simple vertical gradient properly, so that's
+                // what we support too
+                var gradient = ctx.createLinearGradient(0, top, 0, bottom);
+
+                for (var i = 0, l = spec.colors.length; i < l; ++i) {
+                    var c = spec.colors[i];
+                    if (typeof c != "string") {
+                        var co = $.color.parse(defaultColor);
+                        if (c.brightness != null)
+                            co = co.scale('rgb', c.brightness);
+                        if (c.opacity != null)
+                            co.a *= c.opacity;
+                        c = co.toString();
+                    }
+                    gradient.addColorStop(i / (l - 1), c);
+                }
+
+                return gradient;
+            }
+        }
+    }
+
+    // Add the plot function to the top level of the jQuery object
+
+    $.plot = function(placeholder, data, options) {
+        //var t0 = new Date();
+        var plot = new Plot($(placeholder), data, options, $.plot.plugins);
+        //(window.console ? console.log : alert)("time used (msecs): " + ((new Date()).getTime() - t0.getTime()));
+        return plot;
+    };
+
+    $.plot.version = "0.8.3";
+
+    $.plot.plugins = [];
+
+    // Also add the plot function as a chainable property
+
+    $.fn.plot = function(data, options) {
+        return this.each(function() {
+            $.plot(this, data, options);
+        });
+    };
+
+    // round to nearby lower multiple of base
+    function floorInBase(n, base) {
+        return base * Math.floor(n / base);
+    }
+
+})(jQuery);
diff --git a/qa/workunits/erasure-code/jquery.js b/qa/workunits/erasure-code/jquery.js
new file mode 100644
index 00000000..8c24ffc6
--- /dev/null
+++ b/qa/workunits/erasure-code/jquery.js
@@ -0,0 +1,9472 @@
+/*!
+ * jQuery JavaScript Library v1.8.3
+ * http://jquery.com/
+ *
+ * Includes Sizzle.js
+ * http://sizzlejs.com/
+ *
+ * Copyright 2012 jQuery Foundation and other contributors
+ * Released under the MIT license
+ * http://jquery.org/license
+ *
+ * Date: Tue Nov 13 2012 08:20:33 GMT-0500 (Eastern Standard Time)
+ */
+(function( window, undefined ) {
+var
+	// A central reference to the root jQuery(document)
+	rootjQuery,
+
+	// The deferred used on DOM ready
+	readyList,
+
+	// Use the correct document accordingly with window argument (sandbox)
+	document = window.document,
+	location = window.location,
+	navigator = window.navigator,
+
+	// Map over jQuery in case of overwrite
+	_jQuery = window.jQuery,
+
+	// Map over the $ in case of overwrite
+	_$ = window.$,
+
+	// Save a reference to some core methods
+	core_push = Array.prototype.push,
+	core_slice = Array.prototype.slice,
+	core_indexOf = Array.prototype.indexOf,
+	core_toString = Object.prototype.toString,
+	core_hasOwn = Object.prototype.hasOwnProperty,
+	core_trim = String.prototype.trim,
+
+	// Define a local copy of jQuery
+	jQuery = function( selector, context ) {
+		// The jQuery object is actually just the init constructor 'enhanced'
+		return new jQuery.fn.init( selector, context, rootjQuery );
+	},
+
+	// Used for matching numbers
+	core_pnum = /[\-+]?(?:\d*\.|)\d+(?:[eE][\-+]?\d+|)/.source,
+
+	// Used for detecting and trimming whitespace
+	core_rnotwhite = /\S/,
+	core_rspace = /\s+/,
+
+	// Make sure we trim BOM and NBSP (here's looking at you, Safari 5.0 and IE)
+	rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,
+
+	// A simple way to check for HTML strings
+	// Prioritize #id over <tag> to avoid XSS via location.hash (#9521)
+	rquickExpr = /^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/,
+
+	// Match a standalone tag
+	rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>|)$/,
+
+	// JSON RegExp
+	rvalidchars = /^[\],:{}\s]*$/,
+	rvalidbraces = /(?:^|:|,)(?:\s*\[)+/g,
+	rvalidescape = /\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g,
+	rvalidtokens = /"[^"\\\r\n]*"|true|false|null|-?(?:\d\d*\.|)\d+(?:[eE][\-+]?\d+|)/g,
+
+	// Matches dashed string for camelizing
+	rmsPrefix = /^-ms-/,
+	rdashAlpha = /-([\da-z])/gi,
+
+	// Used by jQuery.camelCase as callback to replace()
+	fcamelCase = function( all, letter ) {
+		return ( letter + "" ).toUpperCase();
+	},
+
+	// The ready event handler and self cleanup method
+	DOMContentLoaded = function() {
+		if ( document.addEventListener ) {
+			document.removeEventListener( "DOMContentLoaded", DOMContentLoaded, false );
+			jQuery.ready();
+		} else if ( document.readyState === "complete" ) {
+			// we're here because readyState === "complete" in oldIE
+			// which is good enough for us to call the dom ready!
+			document.detachEvent( "onreadystatechange", DOMContentLoaded );
+			jQuery.ready();
+		}
+	},
+
+	// [[Class]] -> type pairs
+	class2type = {};
+
+jQuery.fn = jQuery.prototype = {
+	constructor: jQuery,
+	init: function( selector, context, rootjQuery ) {
+		var match, elem, ret, doc;
+
+		// Handle $(""), $(null), $(undefined), $(false)
+		if ( !selector ) {
+			return this;
+		}
+
+		// Handle $(DOMElement)
+		if ( selector.nodeType ) {
+			this.context = this[0] = selector;
+			this.length = 1;
+			return this;
+		}
+
+		// Handle HTML strings
+		if ( typeof selector === "string" ) {
+			if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) {
+				// Assume that strings that start and end with <> are HTML and skip the regex check
+				match = [ null, selector, null ];
+
+			} else {
+				match = rquickExpr.exec( selector );
+			}
+
+			// Match html or make sure no context is specified for #id
+			if ( match && (match[1] || !context) ) {
+
+				// HANDLE: $(html) -> $(array)
+				if ( match[1] ) {
+					context = context instanceof jQuery ? context[0] : context;
+					doc = ( context && context.nodeType ? context.ownerDocument || context : document );
+
+					// scripts is true for back-compat
+					selector = jQuery.parseHTML( match[1], doc, true );
+					if ( rsingleTag.test( match[1] ) && jQuery.isPlainObject( context ) ) {
+						this.attr.call( selector, context, true );
+					}
+
+					return jQuery.merge( this, selector );
+
+				// HANDLE: $(#id)
+				} else {
+					elem = document.getElementById( match[2] );
+
+					// Check parentNode to catch when Blackberry 4.6 returns
+					// nodes that are no longer in the document #6963
+					if ( elem && elem.parentNode ) {
+						// Handle the case where IE and Opera return items
+						// by name instead of ID
+						if ( elem.id !== match[2] ) {
+							return rootjQuery.find( selector );
+						}
+
+						// Otherwise, we inject the element directly into the jQuery object
+						this.length = 1;
+						this[0] = elem;
+					}
+
+					this.context = document;
+					this.selector = selector;
+					return this;
+				}
+
+			// HANDLE: $(expr, $(...))
+			} else if ( !context || context.jquery ) {
+				return ( context || rootjQuery ).find( selector );
+
+			// HANDLE: $(expr, context)
+			// (which is just equivalent to: $(context).find(expr)
+			} else {
+				return this.constructor( context ).find( selector );
+			}
+
+		// HANDLE: $(function)
+		// Shortcut for document ready
+		} else if ( jQuery.isFunction( selector ) ) {
+			return rootjQuery.ready( selector );
+		}
+
+		if ( selector.selector !== undefined ) {
+			this.selector = selector.selector;
+			this.context = selector.context;
+		}
+
+		return jQuery.makeArray( selector, this );
+	},
+
+	// Start with an empty selector
+	selector: "",
+
+	// The current version of jQuery being used
+	jquery: "1.8.3",
+
+	// The default length of a jQuery object is 0
+	length: 0,
+
+	// The number of elements contained in the matched element set
+	size: function() {
+		return this.length;
+	},
+
+	toArray: function() {
+		return core_slice.call( this );
+	},
+
+	// Get the Nth element in the matched element set OR
+	// Get the whole matched element set as a clean array
+	get: function( num ) {
+		return num == null ?
+
+			// Return a 'clean' array
+			this.toArray() :
+
+			// Return just the object
+			( num < 0 ? this[ this.length + num ] : this[ num ] );
+	},
+
+	// Take an array of elements and push it onto the stack
+	// (returning the new matched element set)
+	pushStack: function( elems, name, selector ) {
+
+		// Build a new jQuery matched element set
+		var ret = jQuery.merge( this.constructor(), elems );
+
+		// Add the old object onto the stack (as a reference)
+		ret.prevObject = this;
+
+		ret.context = this.context;
+
+		if ( name === "find" ) {
+			ret.selector = this.selector + ( this.selector ? " " : "" ) + selector;
+		} else if ( name ) {
+			ret.selector = this.selector + "." + name + "(" + selector + ")";
+		}
+
+		// Return the newly-formed element set
+		return ret;
+	},
+
+	// Execute a callback for every element in the matched set.
+	// (You can seed the arguments with an array of args, but this is
+	// only used internally.)
+	each: function( callback, args ) {
+		return jQuery.each( this, callback, args );
+	},
+
+	ready: function( fn ) {
+		// Add the callback
+		jQuery.ready.promise().done( fn );
+
+		return this;
+	},
+
+	eq: function( i ) {
+		i = +i;
+		return i === -1 ?
+			this.slice( i ) :
+			this.slice( i, i + 1 );
+	},
+
+	first: function() {
+		return this.eq( 0 );
+	},
+
+	last: function() {
+		return this.eq( -1 );
+	},
+
+	slice: function() {
+		return this.pushStack( core_slice.apply( this, arguments ),
+			"slice", core_slice.call(arguments).join(",") );
+	},
+
+	map: function( callback ) {
+		return this.pushStack( jQuery.map(this, function( elem, i ) {
+			return callback.call( elem, i, elem );
+		}));
+	},
+
+	end: function() {
+		return this.prevObject || this.constructor(null);
+	},
+
+	// For internal use only.
+	// Behaves like an Array's method, not like a jQuery method.
+	push: core_push,
+	sort: [].sort,
+	splice: [].splice
+};
+
+// Give the init function the jQuery prototype for later instantiation
+jQuery.fn.init.prototype = jQuery.fn;
+
+jQuery.extend = jQuery.fn.extend = function() {
+	var options, name, src, copy, copyIsArray, clone,
+		target = arguments[0] || {},
+		i = 1,
+		length = arguments.length,
+		deep = false;
+
+	// Handle a deep copy situation
+	if ( typeof target === "boolean" ) {
+		deep = target;
+		target = arguments[1] || {};
+		// skip the boolean and the target
+		i = 2;
+	}
+
+	// Handle case when target is a string or something (possible in deep copy)
+	if ( typeof target !== "object" && !jQuery.isFunction(target) ) {
+		target = {};
+	}
+
+	// extend jQuery itself if only one argument is passed
+	if ( length === i ) {
+		target = this;
+		--i;
+	}
+
+	for ( ; i < length; i++ ) {
+		// Only deal with non-null/undefined values
+		if ( (options = arguments[ i ]) != null ) {
+			// Extend the base object
+			for ( name in options ) {
+				src = target[ name ];
+				copy = options[ name ];
+
+				// Prevent never-ending loop
+				if ( target === copy ) {
+					continue;
+				}
+
+				// Recurse if we're merging plain objects or arrays
+				if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) {
+					if ( copyIsArray ) {
+						copyIsArray = false;
+						clone = src && jQuery.isArray(src) ? src : [];
+
+					} else {
+						clone = src && jQuery.isPlainObject(src) ? src : {};
+					}
+
+					// Never move original objects, clone them
+					target[ name ] = jQuery.extend( deep, clone, copy );
+
+				// Don't bring in undefined values
+				} else if ( copy !== undefined ) {
+					target[ name ] = copy;
+				}
+			}
+		}
+	}
+
+	// Return the modified object
+	return target;
+};
+
+jQuery.extend({
+	noConflict: function( deep ) {
+		if ( window.$ === jQuery ) {
+			window.$ = _$;
+		}
+
+		if ( deep && window.jQuery === jQuery ) {
+			window.jQuery = _jQuery;
+		}
+
+		return jQuery;
+	},
+
+	// Is the DOM ready to be used? Set to true once it occurs.
+	isReady: false,
+
+	// A counter to track how many items to wait for before
+	// the ready event fires. See #6781
+	readyWait: 1,
+
+	// Hold (or release) the ready event
+	holdReady: function( hold ) {
+		if ( hold ) {
+			jQuery.readyWait++;
+		} else {
+			jQuery.ready( true );
+		}
+	},
+
+	// Handle when the DOM is ready
+	ready: function( wait ) {
+
+		// Abort if there are pending holds or we're already ready
+		if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) {
+			return;
+		}
+
+		// Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443).
+		if ( !document.body ) {
+			return setTimeout( jQuery.ready, 1 );
+		}
+
+		// Remember that the DOM is ready
+		jQuery.isReady = true;
+
+		// If a normal DOM Ready event fired, decrement, and wait if need be
+		if ( wait !== true && --jQuery.readyWait > 0 ) {
+			return;
+		}
+
+		// If there are functions bound, to execute
+		readyList.resolveWith( document, [ jQuery ] );
+
+		// Trigger any bound ready events
+		if ( jQuery.fn.trigger ) {
+			jQuery( document ).trigger("ready").off("ready");
+		}
+	},
+
+	// See test/unit/core.js for details concerning isFunction.
+	// Since version 1.3, DOM methods and functions like alert
+	// aren't supported. They return false on IE (#2968).
+	isFunction: function( obj ) {
+		return jQuery.type(obj) === "function";
+	},
+
+	isArray: Array.isArray || function( obj ) {
+		return jQuery.type(obj) === "array";
+	},
+
+	isWindow: function( obj ) {
+		return obj != null && obj == obj.window;
+	},
+
+	isNumeric: function( obj ) {
+		return !isNaN( parseFloat(obj) ) && isFinite( obj );
+	},
+
+	type: function( obj ) {
+		return obj == null ?
+			String( obj ) :
+			class2type[ core_toString.call(obj) ] || "object";
+	},
+
+	isPlainObject: function( obj ) {
+		// Must be an Object.
+		// Because of IE, we also have to check the presence of the constructor property.
+		// Make sure that DOM nodes and window objects don't pass through, as well
+		if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) {
+			return false;
+		}
+
+		try {
+			// Not own constructor property must be Object
+			if ( obj.constructor &&
+				!core_hasOwn.call(obj, "constructor") &&
+				!core_hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) {
+				return false;
+			}
+		} catch ( e ) {
+			// IE8,9 Will throw exceptions on certain host objects #9897
+			return false;
+		}
+
+		// Own properties are enumerated firstly, so to speed up,
+		// if last one is own, then all properties are own.
+
+		var key;
+		for ( key in obj ) {}
+
+		return key === undefined || core_hasOwn.call( obj, key );
+	},
+
+	isEmptyObject: function( obj ) {
+		var name;
+		for ( name in obj ) {
+			return false;
+		}
+		return true;
+	},
+
+	error: function( msg ) {
+		throw new Error( msg );
+	},
+
+	// data: string of html
+	// context (optional): If specified, the fragment will be created in this context, defaults to document
+	// scripts (optional): If true, will include scripts passed in the html string
+	parseHTML: function( data, context, scripts ) {
+		var parsed;
+		if ( !data || typeof data !== "string" ) {
+			return null;
+		}
+		if ( typeof context === "boolean" ) {
+			scripts = context;
+			context = 0;
+		}
+		context = context || document;
+
+		// Single tag
+		if ( (parsed = rsingleTag.exec( data )) ) {
+			return [ context.createElement( parsed[1] ) ];
+		}
+
+		parsed = jQuery.buildFragment( [ data ], context, scripts ? null : [] );
+		return jQuery.merge( [],
+			(parsed.cacheable ? jQuery.clone( parsed.fragment ) : parsed.fragment).childNodes );
+	},
+
+	parseJSON: function( data ) {
+		if ( !data || typeof data !== "string") {
+			return null;
+		}
+
+		// Make sure leading/trailing whitespace is removed (IE can't handle it)
+		data = jQuery.trim( data );
+
+		// Attempt to parse using the native JSON parser first
+		if ( window.JSON && window.JSON.parse ) {
+			return window.JSON.parse( data );
+		}
+
+		// Make sure the incoming data is actual JSON
+		// Logic borrowed from http://json.org/json2.js
+		if ( rvalidchars.test( data.replace( rvalidescape, "@" )
+			.replace( rvalidtokens, "]" )
+			.replace( rvalidbraces, "")) ) {
+
+			return ( new Function( "return " + data ) )();
+
+		}
+		jQuery.error( "Invalid JSON: " + data );
+	},
+
+	// Cross-browser xml parsing
+	parseXML: function( data ) {
+		var xml, tmp;
+		if ( !data || typeof data !== "string" ) {
+			return null;
+		}
+		try {
+			if ( window.DOMParser ) { // Standard
+				tmp = new DOMParser();
+				xml = tmp.parseFromString( data , "text/xml" );
+			} else { // IE
+				xml = new ActiveXObject( "Microsoft.XMLDOM" );
+				xml.async = "false";
+				xml.loadXML( data );
+			}
+		} catch( e ) {
+			xml = undefined;
+		}
+		if ( !xml || !xml.documentElement || xml.getElementsByTagName( "parsererror" ).length ) {
+			jQuery.error( "Invalid XML: " + data );
+		}
+		return xml;
+	},
+
+	noop: function() {},
+
+	// Evaluates a script in a global context
+	// Workarounds based on findings by Jim Driscoll
+	// http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context
+	globalEval: function( data ) {
+		if ( data && core_rnotwhite.test( data ) ) {
+			// We use execScript on Internet Explorer
+			// We use an anonymous function so that context is window
+			// rather than jQuery in Firefox
+			( window.execScript || function( data ) {
+				window[ "eval" ].call( window, data );
+			} )( data );
+		}
+	},
+
+	// Convert dashed to camelCase; used by the css and data modules
+	// Microsoft forgot to hump their vendor prefix (#9572)
+	camelCase: function( string ) {
+		return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase );
+	},
+
+	nodeName: function( elem, name ) {
+		return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase();
+	},
+
+	// args is for internal usage only
+	each: function( obj, callback, args ) {
+		var name,
+			i = 0,
+			length = obj.length,
+			isObj = length === undefined || jQuery.isFunction( obj );
+
+		if ( args ) {
+			if ( isObj ) {
+				for ( name in obj ) {
+					if ( callback.apply( obj[ name ], args ) === false ) {
+						break;
+					}
+				}
+			} else {
+				for ( ; i < length; ) {
+					if ( callback.apply( obj[ i++ ], args ) === false ) {
+						break;
+					}
+				}
+			}
+
+		// A special, fast, case for the most common use of each
+		} else {
+			if ( isObj ) {
+				for ( name in obj ) {
+					if ( callback.call( obj[ name ], name, obj[ name ] ) === false ) {
+						break;
+					}
+				}
+			} else {
+				for ( ; i < length; ) {
+					if ( callback.call( obj[ i ], i, obj[ i++ ] ) === false ) {
+						break;
+					}
+				}
+			}
+		}
+
+		return obj;
+	},
+
+	// Use native String.trim function wherever possible
+	trim: core_trim && !core_trim.call("\uFEFF\xA0") ?
+		function( text ) {
+			return text == null ?
+				"" :
+				core_trim.call( text );
+		} :
+
+		// Otherwise use our own trimming functionality
+		function( text ) {
+			return text == null ?
+				"" :
+				( text + "" ).replace( rtrim, "" );
+		},
+
+	// results is for internal usage only
+	makeArray: function( arr, results ) {
+		var type,
+			ret = results || [];
+
+		if ( arr != null ) {
+			// The window, strings (and functions) also have 'length'
+			// Tweaked logic slightly to handle Blackberry 4.7 RegExp issues #6930
+			type = jQuery.type( arr );
+
+			if ( arr.length == null || type === "string" || type === "function" || type === "regexp" || jQuery.isWindow( arr ) ) {
+				core_push.call( ret, arr );
+			} else {
+				jQuery.merge( ret, arr );
+			}
+		}
+
+		return ret;
+	},
+
+	inArray: function( elem, arr, i ) {
+		var len;
+
+		if ( arr ) {
+			if ( core_indexOf ) {
+				return core_indexOf.call( arr, elem, i );
+			}
+
+			len = arr.length;
+			i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0;
+
+			for ( ; i < len; i++ ) {
+				// Skip accessing in sparse arrays
+				if ( i in arr && arr[ i ] === elem ) {
+					return i;
+				}
+			}
+		}
+
+		return -1;
+	},
+
+	merge: function( first, second ) {
+		var l = second.length,
+			i = first.length,
+			j = 0;
+
+		if ( typeof l === "number" ) {
+			for ( ; j < l; j++ ) {
+				first[ i++ ] = second[ j ];
+			}
+
+		} else {
+			while ( second[j] !== undefined ) {
+				first[ i++ ] = second[ j++ ];
+			}
+		}
+
+		first.length = i;
+
+		return first;
+	},
+
+	grep: function( elems, callback, inv ) {
+		var retVal,
+			ret = [],
+			i = 0,
+			length = elems.length;
+		inv = !!inv;
+
+		// Go through the array, only saving the items
+		// that pass the validator function
+		for ( ; i < length; i++ ) {
+			retVal = !!callback( elems[ i ], i );
+			if ( inv !== retVal ) {
+				ret.push( elems[ i ] );
+			}
+		}
+
+		return ret;
+	},
+
+	// arg is for internal usage only
+	map: function( elems, callback, arg ) {
+		var value, key,
+			ret = [],
+			i = 0,
+			length = elems.length,
+			// jquery objects are treated as arrays
+			isArray = elems instanceof jQuery || length !== undefined && typeof length === "number" && ( ( length > 0 && elems[ 0 ] && elems[ length -1 ] ) || length === 0 || jQuery.isArray( elems ) ) ;
+
+		// Go through the array, translating each of the items to their
+		if ( isArray ) {
+			for ( ; i < length; i++ ) {
+				value = callback( elems[ i ], i, arg );
+
+				if ( value != null ) {
+					ret[ ret.length ] = value;
+				}
+			}
+
+		// Go through every key on the object,
+		} else {
+			for ( key in elems ) {
+				value = callback( elems[ key ], key, arg );
+
+				if ( value != null ) {
+					ret[ ret.length ] = value;
+				}
+			}
+		}
+
+		// Flatten any nested arrays
+		return ret.concat.apply( [], ret );
+	},
+
+	// A global GUID counter for objects
+	guid: 1,
+
+	// Bind a function to a context, optionally partially applying any
+	// arguments.
+	proxy: function( fn, context ) {
+		var tmp, args, proxy;
+
+		if ( typeof context === "string" ) {
+			tmp = fn[ context ];
+			context = fn;
+			fn = tmp;
+		}
+
+		// Quick check to determine if target is callable, in the spec
+		// this throws a TypeError, but we will just return undefined.
+		if ( !jQuery.isFunction( fn ) ) {
+			return undefined;
+		}
+
+		// Simulated bind
+		args = core_slice.call( arguments, 2 );
+		proxy = function() {
+			return fn.apply( context, args.concat( core_slice.call( arguments ) ) );
+		};
+
+		// Set the guid of unique handler to the same of original handler, so it can be removed
+		proxy.guid = fn.guid = fn.guid || jQuery.guid++;
+
+		return proxy;
+	},
+
+	// Multifunctional method to get and set values of a collection
+	// The value/s can optionally be executed if it's a function
+	access: function( elems, fn, key, value, chainable, emptyGet, pass ) {
+		var exec,
+			bulk = key == null,
+			i = 0,
+			length = elems.length;
+
+		// Sets many values
+		if ( key && typeof key === "object" ) {
+			for ( i in key ) {
+				jQuery.access( elems, fn, i, key[i], 1, emptyGet, value );
+			}
+			chainable = 1;
+
+		// Sets one value
+		} else if ( value !== undefined ) {
+			// Optionally, function values get executed if exec is true
+			exec = pass === undefined && jQuery.isFunction( value );
+
+			if ( bulk ) {
+				// Bulk operations only iterate when executing function values
+				if ( exec ) {
+					exec = fn;
+					fn = function( elem, key, value ) {
+						return exec.call( jQuery( elem ), value );
+					};
+
+				// Otherwise they run against the entire set
+				} else {
+					fn.call( elems, value );
+					fn = null;
+				}
+			}
+
+			if ( fn ) {
+				for (; i < length; i++ ) {
+					fn( elems[i], key, exec ? value.call( elems[i], i, fn( elems[i], key ) ) : value, pass );
+				}
+			}
+
+			chainable = 1;
+		}
+
+		return chainable ?
+			elems :
+
+			// Gets
+			bulk ?
+				fn.call( elems ) :
+				length ? fn( elems[0], key ) : emptyGet;
+	},
+
+	now: function() {
+		return ( new Date() ).getTime();
+	}
+});
+
+jQuery.ready.promise = function( obj ) {
+	if ( !readyList ) {
+
+		readyList = jQuery.Deferred();
+
+		// Catch cases where $(document).ready() is called after the browser event has already occurred.
+		// we once tried to use readyState "interactive" here, but it caused issues like the one
+		// discovered by ChrisS here: http://bugs.jquery.com/ticket/12282#comment:15
+		if ( document.readyState === "complete" ) {
+			// Handle it asynchronously to allow scripts the opportunity to delay ready
+			setTimeout( jQuery.ready, 1 );
+
+		// Standards-based browsers support DOMContentLoaded
+		} else if ( document.addEventListener ) {
+			// Use the handy event callback
+			document.addEventListener( "DOMContentLoaded", DOMContentLoaded, false );
+
+			// A fallback to window.onload, that will always work
+			window.addEventListener( "load", jQuery.ready, false );
+
+		// If IE event model is used
+		} else {
+			// Ensure firing before onload, maybe late but safe also for iframes
+			document.attachEvent( "onreadystatechange", DOMContentLoaded );
+
+			// A fallback to window.onload, that will always work
+			window.attachEvent( "onload", jQuery.ready );
+
+			// If IE and not a frame
+			// continually check to see if the document is ready
+			var top = false;
+
+			try {
+				top = window.frameElement == null && document.documentElement;
+			} catch(e) {}
+
+			if ( top && top.doScroll ) {
+				(function doScrollCheck() {
+					if ( !jQuery.isReady ) {
+
+						try {
+							// Use the trick by Diego Perini
+							// http://javascript.nwbox.com/IEContentLoaded/
+							top.doScroll("left");
+						} catch(e) {
+							return setTimeout( doScrollCheck, 50 );
+						}
+
+						// and execute any waiting functions
+						jQuery.ready();
+					}
+				})();
+			}
+		}
+	}
+	return readyList.promise( obj );
+};
+
+// Populate the class2type map
+jQuery.each("Boolean Number String Function Array Date RegExp Object".split(" "), function(i, name) {
+	class2type[ "[object " + name + "]" ] = name.toLowerCase();
+});
+
+// All jQuery objects should point back to these
+rootjQuery = jQuery(document);
+// String to Object options format cache
+var optionsCache = {};
+
+// Convert String-formatted options into Object-formatted ones and store in cache
+function createOptions( options ) {
+	var object = optionsCache[ options ] = {};
+	jQuery.each( options.split( core_rspace ), function( _, flag ) {
+		object[ flag ] = true;
+	});
+	return object;
+}
+
+/*
+ * Create a callback list using the following parameters:
+ *
+ *	options: an optional list of space-separated options that will change how
+ *			the callback list behaves or a more traditional option object
+ *
+ * By default a callback list will act like an event callback list and can be
+ * "fired" multiple times.
+ *
+ * Possible options:
+ *
+ *	once:			will ensure the callback list can only be fired once (like a Deferred)
+ *
+ *	memory:			will keep track of previous values and will call any callback added
+ *					after the list has been fired right away with the latest "memorized"
+ *					values (like a Deferred)
+ *
+ *	unique:			will ensure a callback can only be added once (no duplicate in the list)
+ *
+ *	stopOnFalse:	interrupt callings when a callback returns false
+ *
+ */
+jQuery.Callbacks = function( options ) {
+
+	// Convert options from String-formatted to Object-formatted if needed
+	// (we check in cache first)
+	options = typeof options === "string" ?
+		( optionsCache[ options ] || createOptions( options ) ) :
+		jQuery.extend( {}, options );
+
+	var // Last fire value (for non-forgettable lists)
+		memory,
+		// Flag to know if list was already fired
+		fired,
+		// Flag to know if list is currently firing
+		firing,
+		// First callback to fire (used internally by add and fireWith)
+		firingStart,
+		// End of the loop when firing
+		firingLength,
+		// Index of currently firing callback (modified by remove if needed)
+		firingIndex,
+		// Actual callback list
+		list = [],
+		// Stack of fire calls for repeatable lists
+		stack = !options.once && [],
+		// Fire callbacks
+		fire = function( data ) {
+			memory = options.memory && data;
+			fired = true;
+			firingIndex = firingStart || 0;
+			firingStart = 0;
+			firingLength = list.length;
+			firing = true;
+			for ( ; list && firingIndex < firingLength; firingIndex++ ) {
+				if ( list[ firingIndex ].apply( data[ 0 ], data[ 1 ] ) === false && options.stopOnFalse ) {
+					memory = false; // To prevent further calls using add
+					break;
+				}
+			}
+			firing = false;
+			if ( list ) {
+				if ( stack ) {
+					if ( stack.length ) {
+						fire( stack.shift() );
+					}
+				} else if ( memory ) {
+					list = [];
+				} else {
+					self.disable();
+				}
+			}
+		},
+		// Actual Callbacks object
+		self = {
+			// Add a callback or a collection of callbacks to the list
+			add: function() {
+				if ( list ) {
+					// First, we save the current length
+					var start = list.length;
+					(function add( args ) {
+						jQuery.each( args, function( _, arg ) {
+							var type = jQuery.type( arg );
+							if ( type === "function" ) {
+								if ( !options.unique || !self.has( arg ) ) {
+									list.push( arg );
+								}
+							} else if ( arg && arg.length && type !== "string" ) {
+								// Inspect recursively
+								add( arg );
+							}
+						});
+					})( arguments );
+					// Do we need to add the callbacks to the
+					// current firing batch?
+					if ( firing ) {
+						firingLength = list.length;
+					// With memory, if we're not firing then
+					// we should call right away
+					} else if ( memory ) {
+						firingStart = start;
+						fire( memory );
+					}
+				}
+				return this;
+			},
+			// Remove a callback from the list
+			remove: function() {
+				if ( list ) {
+					jQuery.each( arguments, function( _, arg ) {
+						var index;
+						while( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) {
+							list.splice( index, 1 );
+							// Handle firing indexes
+							if ( firing ) {
+								if ( index <= firingLength ) {
+									firingLength--;
+								}
+								if ( index <= firingIndex ) {
+									firingIndex--;
+								}
+							}
+						}
+					});
+				}
+				return this;
+			},
+			// Control if a given callback is in the list
+			has: function( fn ) {
+				return jQuery.inArray( fn, list ) > -1;
+			},
+			// Remove all callbacks from the list
+			empty: function() {
+				list = [];
+				return this;
+			},
+			// Have the list do nothing anymore
+			disable: function() {
+				list = stack = memory = undefined;
+				return this;
+			},
+			// Is it disabled?
+			disabled: function() {
+				return !list;
+			},
+			// Lock the list in its current state
+			lock: function() {
+				stack = undefined;
+				if ( !memory ) {
+					self.disable();
+				}
+				return this;
+			},
+			// Is it locked?
+			locked: function() {
+				return !stack;
+			},
+			// Call all callbacks with the given context and arguments
+			fireWith: function( context, args ) {
+				args = args || [];
+				args = [ context, args.slice ? args.slice() : args ];
+				if ( list && ( !fired || stack ) ) {
+					if ( firing ) {
+						stack.push( args );
+					} else {
+						fire( args );
+					}
+				}
+				return this;
+			},
+			// Call all the callbacks with the given arguments
+			fire: function() {
+				self.fireWith( this, arguments );
+				return this;
+			},
+			// To know if the callbacks have already been called at least once
+			fired: function() {
+				return !!fired;
+			}
+		};
+
+	return self;
+};
+jQuery.extend({
+
+	Deferred: function( func ) {
+		var tuples = [
+				// action, add listener, listener list, final state
+				[ "resolve", "done", jQuery.Callbacks("once memory"), "resolved" ],
+				[ "reject", "fail", jQuery.Callbacks("once memory"), "rejected" ],
+				[ "notify", "progress", jQuery.Callbacks("memory") ]
+			],
+			state = "pending",
+			promise = {
+				state: function() {
+					return state;
+				},
+				always: function() {
+					deferred.done( arguments ).fail( arguments );
+					return this;
+				},
+				then: function( /* fnDone, fnFail, fnProgress */ ) {
+					var fns = arguments;
+					return jQuery.Deferred(function( newDefer ) {
+						jQuery.each( tuples, function( i, tuple ) {
+							var action = tuple[ 0 ],
+								fn = fns[ i ];
+							// deferred[ done | fail | progress ] for forwarding actions to newDefer
+							deferred[ tuple[1] ]( jQuery.isFunction( fn ) ?
+								function() {
+									var returned = fn.apply( this, arguments );
+									if ( returned && jQuery.isFunction( returned.promise ) ) {
+										returned.promise()
+											.done( newDefer.resolve )
+											.fail( newDefer.reject )
+											.progress( newDefer.notify );
+									} else {
+										newDefer[ action + "With" ]( this === deferred ? newDefer : this, [ returned ] );
+									}
+								} :
+								newDefer[ action ]
+							);
+						});
+						fns = null;
+					}).promise();
+				},
+				// Get a promise for this deferred
+				// If obj is provided, the promise aspect is added to the object
+				promise: function( obj ) {
+					return obj != null ? jQuery.extend( obj, promise ) : promise;
+				}
+			},
+			deferred = {};
+
+		// Keep pipe for back-compat
+		promise.pipe = promise.then;
+
+		// Add list-specific methods
+		jQuery.each( tuples, function( i, tuple ) {
+			var list = tuple[ 2 ],
+				stateString = tuple[ 3 ];
+
+			// promise[ done | fail | progress ] = list.add
+			promise[ tuple[1] ] = list.add;
+
+			// Handle state
+			if ( stateString ) {
+				list.add(function() {
+					// state = [ resolved | rejected ]
+					state = stateString;
+
+				// [ reject_list | resolve_list ].disable; progress_list.lock
+				}, tuples[ i ^ 1 ][ 2 ].disable, tuples[ 2 ][ 2 ].lock );
+			}
+
+			// deferred[ resolve | reject | notify ] = list.fire
+			deferred[ tuple[0] ] = list.fire;
+			deferred[ tuple[0] + "With" ] = list.fireWith;
+		});
+
+		// Make the deferred a promise
+		promise.promise( deferred );
+
+		// Call given func if any
+		if ( func ) {
+			func.call( deferred, deferred );
+		}
+
+		// All done!
+		return deferred;
+	},
+
+	// Deferred helper
+	when: function( subordinate /* , ..., subordinateN */ ) {
+		var i = 0,
+			resolveValues = core_slice.call( arguments ),
+			length = resolveValues.length,
+
+			// the count of uncompleted subordinates
+			remaining = length !== 1 || ( subordinate && jQuery.isFunction( subordinate.promise ) ) ? length : 0,
+
+			// the master Deferred. If resolveValues consist of only a single Deferred, just use that.
+			deferred = remaining === 1 ? subordinate : jQuery.Deferred(),
+
+			// Update function for both resolve and progress values
+			updateFunc = function( i, contexts, values ) {
+				return function( value ) {
+					contexts[ i ] = this;
+					values[ i ] = arguments.length > 1 ? core_slice.call( arguments ) : value;
+					if( values === progressValues ) {
+						deferred.notifyWith( contexts, values );
+					} else if ( !( --remaining ) ) {
+						deferred.resolveWith( contexts, values );
+					}
+				};
+			},
+
+			progressValues, progressContexts, resolveContexts;
+
+		// add listeners to Deferred subordinates; treat others as resolved
+		if ( length > 1 ) {
+			progressValues = new Array( length );
+			progressContexts = new Array( length );
+			resolveContexts = new Array( length );
+			for ( ; i < length; i++ ) {
+				if ( resolveValues[ i ] && jQuery.isFunction( resolveValues[ i ].promise ) ) {
+					resolveValues[ i ].promise()
+						.done( updateFunc( i, resolveContexts, resolveValues ) )
+						.fail( deferred.reject )
+						.progress( updateFunc( i, progressContexts, progressValues ) );
+				} else {
+					--remaining;
+				}
+			}
+		}
+
+		// if we're not waiting on anything, resolve the master
+		if ( !remaining ) {
+			deferred.resolveWith( resolveContexts, resolveValues );
+		}
+
+		return deferred.promise();
+	}
+});
+jQuery.support = (function() {
+
+	var support,
+		all,
+		a,
+		select,
+		opt,
+		input,
+		fragment,
+		eventName,
+		i,
+		isSupported,
+		clickFn,
+		div = document.createElement("div");
+
+	// Setup
+	div.setAttribute( "className", "t" );
+	div.innerHTML = "  <link/><table></table><a href='/a'>a</a><input type='checkbox'/>";
+
+	// Support tests won't run in some limited or non-browser environments
+	all = div.getElementsByTagName("*");
+	a = div.getElementsByTagName("a")[ 0 ];
+	if ( !all || !a || !all.length ) {
+		return {};
+	}
+
+	// First batch of tests
+	select = document.createElement("select");
+	opt = select.appendChild( document.createElement("option") );
+	input = div.getElementsByTagName("input")[ 0 ];
+
+	a.style.cssText = "top:1px;float:left;opacity:.5";
+	support = {
+		// IE strips leading whitespace when .innerHTML is used
+		leadingWhitespace: ( div.firstChild.nodeType === 3 ),
+
+		// Make sure that tbody elements aren't automatically inserted
+		// IE will insert them into empty tables
+		tbody: !div.getElementsByTagName("tbody").length,
+
+		// Make sure that link elements get serialized correctly by innerHTML
+		// This requires a wrapper element in IE
+		htmlSerialize: !!div.getElementsByTagName("link").length,
+
+		// Get the style information from getAttribute
+		// (IE uses .cssText instead)
+		style: /top/.test( a.getAttribute("style") ),
+
+		// Make sure that URLs aren't manipulated
+		// (IE normalizes it by default)
+		hrefNormalized: ( a.getAttribute("href") === "/a" ),
+
+		// Make sure that element opacity exists
+		// (IE uses filter instead)
+		// Use a regex to work around a WebKit issue. See #5145
+		opacity: /^0.5/.test( a.style.opacity ),
+
+		// Verify style float existence
+		// (IE uses styleFloat instead of cssFloat)
+		cssFloat: !!a.style.cssFloat,
+
+		// Make sure that if no value is specified for a checkbox
+		// that it defaults to "on".
+		// (WebKit defaults to "" instead)
+		checkOn: ( input.value === "on" ),
+
+		// Make sure that a selected-by-default option has a working selected property.
+		// (WebKit defaults to false instead of true, IE too, if it's in an optgroup)
+		optSelected: opt.selected,
+
+		// Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7)
+		getSetAttribute: div.className !== "t",
+
+		// Tests for enctype support on a form (#6743)
+		enctype: !!document.createElement("form").enctype,
+
+		// Makes sure cloning an html5 element does not cause problems
+		// Where outerHTML is undefined, this still works
+		html5Clone: document.createElement("nav").cloneNode( true ).outerHTML !== "<:nav></:nav>",
+
+		// jQuery.support.boxModel DEPRECATED in 1.8 since we don't support Quirks Mode
+		boxModel: ( document.compatMode === "CSS1Compat" ),
+
+		// Will be defined later
+		submitBubbles: true,
+		changeBubbles: true,
+		focusinBubbles: false,
+		deleteExpando: true,
+		noCloneEvent: true,
+		inlineBlockNeedsLayout: false,
+		shrinkWrapBlocks: false,
+		reliableMarginRight: true,
+		boxSizingReliable: true,
+		pixelPosition: false
+	};
+
+	// Make sure checked status is properly cloned
+	input.checked = true;
+	support.noCloneChecked = input.cloneNode( true ).checked;
+
+	// Make sure that the options inside disabled selects aren't marked as disabled
+	// (WebKit marks them as disabled)
+	select.disabled = true;
+	support.optDisabled = !opt.disabled;
+
+	// Test to see if it's possible to delete an expando from an element
+	// Fails in Internet Explorer
+	try {
+		delete div.test;
+	} catch( e ) {
+		support.deleteExpando = false;
+	}
+
+	if ( !div.addEventListener && div.attachEvent && div.fireEvent ) {
+		div.attachEvent( "onclick", clickFn = function() {
+			// Cloning a node shouldn't copy over any
+			// bound event handlers (IE does this)
+			support.noCloneEvent = false;
+		});
+		div.cloneNode( true ).fireEvent("onclick");
+		div.detachEvent( "onclick", clickFn );
+	}
+
+	// Check if a radio maintains its value
+	// after being appended to the DOM
+	input = document.createElement("input");
+	input.value = "t";
+	input.setAttribute( "type", "radio" );
+	support.radioValue = input.value === "t";
+
+	input.setAttribute( "checked", "checked" );
+
+	// #11217 - WebKit loses check when the name is after the checked attribute
+	input.setAttribute( "name", "t" );
+
+	div.appendChild( input );
+	fragment = document.createDocumentFragment();
+	fragment.appendChild( div.lastChild );
+
+	// WebKit doesn't clone checked state correctly in fragments
+	support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked;
+
+	// Check if a disconnected checkbox will retain its checked
+	// value of true after appended to the DOM (IE6/7)
+	support.appendChecked = input.checked;
+
+	fragment.removeChild( input );
+	fragment.appendChild( div );
+
+	// Technique from Juriy Zaytsev
+	// http://perfectionkills.com/detecting-event-support-without-browser-sniffing/
+	// We only care about the case where non-standard event systems
+	// are used, namely in IE. Short-circuiting here helps us to
+	// avoid an eval call (in setAttribute) which can cause CSP
+	// to go haywire. See: https://developer.mozilla.org/en/Security/CSP
+	if ( div.attachEvent ) {
+		for ( i in {
+			submit: true,
+			change: true,
+			focusin: true
+		}) {
+			eventName = "on" + i;
+			isSupported = ( eventName in div );
+			if ( !isSupported ) {
+				div.setAttribute( eventName, "return;" );
+				isSupported = ( typeof div[ eventName ] === "function" );
+			}
+			support[ i + "Bubbles" ] = isSupported;
+		}
+	}
+
+	// Run tests that need a body at doc ready
+	jQuery(function() {
+		var container, div, tds, marginDiv,
+			divReset = "padding:0;margin:0;border:0;display:block;overflow:hidden;",
+			body = document.getElementsByTagName("body")[0];
+
+		if ( !body ) {
+			// Return for frameset docs that don't have a body
+			return;
+		}
+
+		container = document.createElement("div");
+		container.style.cssText = "visibility:hidden;border:0;width:0;height:0;position:static;top:0;margin-top:1px";
+		body.insertBefore( container, body.firstChild );
+
+		// Construct the test element
+		div = document.createElement("div");
+		container.appendChild( div );
+
+		// Check if table cells still have offsetWidth/Height when they are set
+		// to display:none and there are still other visible table cells in a
+		// table row; if so, offsetWidth/Height are not reliable for use when
+		// determining if an element has been hidden directly using
+		// display:none (it is still safe to use offsets if a parent element is
+		// hidden; don safety goggles and see bug #4512 for more information).
+		// (only IE 8 fails this test)
+		div.innerHTML = "<table><tr><td></td><td>t</td></tr></table>";
+		tds = div.getElementsByTagName("td");
+		tds[ 0 ].style.cssText = "padding:0;margin:0;border:0;display:none";
+		isSupported = ( tds[ 0 ].offsetHeight === 0 );
+
+		tds[ 0 ].style.display = "";
+		tds[ 1 ].style.display = "none";
+
+		// Check if empty table cells still have offsetWidth/Height
+		// (IE <= 8 fail this test)
+		support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 );
+
+		// Check box-sizing and margin behavior
+		div.innerHTML = "";
+		div.style.cssText = "box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;";
+		support.boxSizing = ( div.offsetWidth === 4 );
+		support.doesNotIncludeMarginInBodyOffset = ( body.offsetTop !== 1 );
+
+		// NOTE: To any future maintainer, we've window.getComputedStyle
+		// because jsdom on node.js will break without it.
+		if ( window.getComputedStyle ) {
+			support.pixelPosition = ( window.getComputedStyle( div, null ) || {} ).top !== "1%";
+			support.boxSizingReliable = ( window.getComputedStyle( div, null ) || { width: "4px" } ).width === "4px";
+
+			// Check if div with explicit width and no margin-right incorrectly
+			// gets computed margin-right based on width of container. For more
+			// info see bug #3333
+			// Fails in WebKit before Feb 2011 nightlies
+			// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
+			marginDiv = document.createElement("div");
+			marginDiv.style.cssText = div.style.cssText = divReset;
+			marginDiv.style.marginRight = marginDiv.style.width = "0";
+			div.style.width = "1px";
+			div.appendChild( marginDiv );
+			support.reliableMarginRight =
+				!parseFloat( ( window.getComputedStyle( marginDiv, null ) || {} ).marginRight );
+		}
+
+		if ( typeof div.style.zoom !== "undefined" ) {
+			// Check if natively block-level elements act like inline-block
+			// elements when setting their display to 'inline' and giving
+			// them layout
+			// (IE < 8 does this)
+			div.innerHTML = "";
+			div.style.cssText = divReset + "width:1px;padding:1px;display:inline;zoom:1";
+			support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 );
+
+			// Check if elements with layout shrink-wrap their children
+			// (IE 6 does this)
+			div.style.display = "block";
+			div.style.overflow = "visible";
+			div.innerHTML = "<div></div>";
+			div.firstChild.style.width = "5px";
+			support.shrinkWrapBlocks = ( div.offsetWidth !== 3 );
+
+			container.style.zoom = 1;
+		}
+
+		// Null elements to avoid leaks in IE
+		body.removeChild( container );
+		container = div = tds = marginDiv = null;
+	});
+
+	// Null elements to avoid leaks in IE
+	fragment.removeChild( div );
+	all = a = select = opt = input = fragment = div = null;
+
+	return support;
+})();
+var rbrace = /(?:\{[\s\S]*\}|\[[\s\S]*\])$/,
+	rmultiDash = /([A-Z])/g;
+
+jQuery.extend({
+	cache: {},
+
+	deletedIds: [],
+
+	// Remove at next major release (1.9/2.0)
+	uuid: 0,
+
+	// Unique for each copy of jQuery on the page
+	// Non-digits removed to match rinlinejQuery
+	expando: "jQuery" + ( jQuery.fn.jquery + Math.random() ).replace( /\D/g, "" ),
+
+	// The following elements throw uncatchable exceptions if you
+	// attempt to add expando properties to them.
+	noData: {
+		"embed": true,
+		// Ban all objects except for Flash (which handle expandos)
+		"object": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",
+		"applet": true
+	},
+
+	hasData: function( elem ) {
+		elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ];
+		return !!elem && !isEmptyDataObject( elem );
+	},
+
+	data: function( elem, name, data, pvt /* Internal Use Only */ ) {
+		if ( !jQuery.acceptData( elem ) ) {
+			return;
+		}
+
+		var thisCache, ret,
+			internalKey = jQuery.expando,
+			getByName = typeof name === "string",
+
+			// We have to handle DOM nodes and JS objects differently because IE6-7
+			// can't GC object references properly across the DOM-JS boundary
+			isNode = elem.nodeType,
+
+			// Only DOM nodes need the global jQuery cache; JS object data is
+			// attached directly to the object so GC can occur automatically
+			cache = isNode ? jQuery.cache : elem,
+
+			// Only defining an ID for JS objects if its cache already exists allows
+			// the code to shortcut on the same path as a DOM node with no cache
+			id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey;
+
+		// Avoid doing any more work than we need to when trying to get data on an
+		// object that has no data at all
+		if ( (!id || !cache[id] || (!pvt && !cache[id].data)) && getByName && data === undefined ) {
+			return;
+		}
+
+		if ( !id ) {
+			// Only DOM nodes need a new unique ID for each element since their data
+			// ends up in the global cache
+			if ( isNode ) {
+				elem[ internalKey ] = id = jQuery.deletedIds.pop() || jQuery.guid++;
+			} else {
+				id = internalKey;
+			}
+		}
+
+		if ( !cache[ id ] ) {
+			cache[ id ] = {};
+
+			// Avoids exposing jQuery metadata on plain JS objects when the object
+			// is serialized using JSON.stringify
+			if ( !isNode ) {
+				cache[ id ].toJSON = jQuery.noop;
+			}
+		}
+
+		// An object can be passed to jQuery.data instead of a key/value pair; this gets
+		// shallow copied over onto the existing cache
+		if ( typeof name === "object" || typeof name === "function" ) {
+			if ( pvt ) {
+				cache[ id ] = jQuery.extend( cache[ id ], name );
+			} else {
+				cache[ id ].data = jQuery.extend( cache[ id ].data, name );
+			}
+		}
+
+		thisCache = cache[ id ];
+
+		// jQuery data() is stored in a separate object inside the object's internal data
+		// cache in order to avoid key collisions between internal data and user-defined
+		// data.
+		if ( !pvt ) {
+			if ( !thisCache.data ) {
+				thisCache.data = {};
+			}
+
+			thisCache = thisCache.data;
+		}
+
+		if ( data !== undefined ) {
+			thisCache[ jQuery.camelCase( name ) ] = data;
+		}
+
+		// Check for both converted-to-camel and non-converted data property names
+		// If a data property was specified
+		if ( getByName ) {
+
+			// First Try to find as-is property data
+			ret = thisCache[ name ];
+
+			// Test for null|undefined property data
+			if ( ret == null ) {
+
+				// Try to find the camelCased property
+				ret = thisCache[ jQuery.camelCase( name ) ];
+			}
+		} else {
+			ret = thisCache;
+		}
+
+		return ret;
+	},
+
+	removeData: function( elem, name, pvt /* Internal Use Only */ ) {
+		if ( !jQuery.acceptData( elem ) ) {
+			return;
+		}
+
+		var thisCache, i, l,
+
+			isNode = elem.nodeType,
+
+			// See jQuery.data for more information
+			cache = isNode ? jQuery.cache : elem,
+			id = isNode ? elem[ jQuery.expando ] : jQuery.expando;
+
+		// If there is already no cache entry for this object, there is no
+		// purpose in continuing
+		if ( !cache[ id ] ) {
+			return;
+		}
+
+		if ( name ) {
+
+			thisCache = pvt ? cache[ id ] : cache[ id ].data;
+
+			if ( thisCache ) {
+
+				// Support array or space separated string names for data keys
+				if ( !jQuery.isArray( name ) ) {
+
+					// try the string as a key before any manipulation
+					if ( name in thisCache ) {
+						name = [ name ];
+					} else {
+
+						// split the camel cased version by spaces unless a key with the spaces exists
+						name = jQuery.camelCase( name );
+						if ( name in thisCache ) {
+							name = [ name ];
+						} else {
+							name = name.split(" ");
+						}
+					}
+				}
+
+				for ( i = 0, l = name.length; i < l; i++ ) {
+					delete thisCache[ name[i] ];
+				}
+
+				// If there is no data left in the cache, we want to continue
+				// and let the cache object itself get destroyed
+				if ( !( pvt ? isEmptyDataObject : jQuery.isEmptyObject )( thisCache ) ) {
+					return;
+				}
+			}
+		}
+
+		// See jQuery.data for more information
+		if ( !pvt ) {
+			delete cache[ id ].data;
+
+			// Don't destroy the parent cache unless the internal data object
+			// had been the only thing left in it
+			if ( !isEmptyDataObject( cache[ id ] ) ) {
+				return;
+			}
+		}
+
+		// Destroy the cache
+		if ( isNode ) {
+			jQuery.cleanData( [ elem ], true );
+
+		// Use delete when supported for expandos or `cache` is not a window per isWindow (#10080)
+		} else if ( jQuery.support.deleteExpando || cache != cache.window ) {
+			delete cache[ id ];
+
+		// When all else fails, null
+		} else {
+			cache[ id ] = null;
+		}
+	},
+
+	// For internal use only.
+	_data: function( elem, name, data ) {
+		return jQuery.data( elem, name, data, true );
+	},
+
+	// A method for determining if a DOM node can handle the data expando
+	acceptData: function( elem ) {
+		var noData = elem.nodeName && jQuery.noData[ elem.nodeName.toLowerCase() ];
+
+		// nodes accept data unless otherwise specified; rejection can be conditional
+		return !noData || noData !== true && elem.getAttribute("classid") === noData;
+	}
+});
+
+jQuery.fn.extend({
+	data: function( key, value ) {
+		var parts, part, attr, name, l,
+			elem = this[0],
+			i = 0,
+			data = null;
+
+		// Gets all values
+		if ( key === undefined ) {
+			if ( this.length ) {
+				data = jQuery.data( elem );
+
+				if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) {
+					attr = elem.attributes;
+					for ( l = attr.length; i < l; i++ ) {
+						name = attr[i].name;
+
+						if ( !name.indexOf( "data-" ) ) {
+							name = jQuery.camelCase( name.substring(5) );
+
+							dataAttr( elem, name, data[ name ] );
+						}
+					}
+					jQuery._data( elem, "parsedAttrs", true );
+				}
+			}
+
+			return data;
+		}
+
+		// Sets multiple values
+		if ( typeof key === "object" ) {
+			return this.each(function() {
+				jQuery.data( this, key );
+			});
+		}
+
+		parts = key.split( ".", 2 );
+		parts[1] = parts[1] ? "." + parts[1] : "";
+		part = parts[1] + "!";
+
+		return jQuery.access( this, function( value ) {
+
+			if ( value === undefined ) {
+				data = this.triggerHandler( "getData" + part, [ parts[0] ] );
+
+				// Try to fetch any internally stored data first
+				if ( data === undefined && elem ) {
+					data = jQuery.data( elem, key );
+					data = dataAttr( elem, key, data );
+				}
+
+				return data === undefined && parts[1] ?
+					this.data( parts[0] ) :
+					data;
+			}
+
+			parts[1] = value;
+			this.each(function() {
+				var self = jQuery( this );
+
+				self.triggerHandler( "setData" + part, parts );
+				jQuery.data( this, key, value );
+				self.triggerHandler( "changeData" + part, parts );
+			});
+		}, null, value, arguments.length > 1, null, false );
+	},
+
+	removeData: function( key ) {
+		return this.each(function() {
+			jQuery.removeData( this, key );
+		});
+	}
+});
+
+function dataAttr( elem, key, data ) {
+	// If nothing was found internally, try to fetch any
+	// data from the HTML5 data-* attribute
+	if ( data === undefined && elem.nodeType === 1 ) {
+
+		var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase();
+
+		data = elem.getAttribute( name );
+
+		if ( typeof data === "string" ) {
+			try {
+				data = data === "true" ? true :
+				data === "false" ? false :
+				data === "null" ? null :
+				// Only convert to a number if it doesn't change the string
+				+data + "" === data ? +data :
+				rbrace.test( data ) ? jQuery.parseJSON( data ) :
+					data;
+			} catch( e ) {}
+
+			// Make sure we set the data so it isn't changed later
+			jQuery.data( elem, key, data );
+
+		} else {
+			data = undefined;
+		}
+	}
+
+	return data;
+}
+
+// checks a cache object for emptiness
+function isEmptyDataObject( obj ) {
+	var name;
+	for ( name in obj ) {
+
+		// if the public data object is empty, the private is still empty
+		if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) {
+			continue;
+		}
+		if ( name !== "toJSON" ) {
+			return false;
+		}
+	}
+
+	return true;
+}
+jQuery.extend({
+	queue: function( elem, type, data ) {
+		var queue;
+
+		if ( elem ) {
+			type = ( type || "fx" ) + "queue";
+			queue = jQuery._data( elem, type );
+
+			// Speed up dequeue by getting out quickly if this is just a lookup
+			if ( data ) {
+				if ( !queue || jQuery.isArray(data) ) {
+					queue = jQuery._data( elem, type, jQuery.makeArray(data) );
+				} else {
+					queue.push( data );
+				}
+			}
+			return queue || [];
+		}
+	},
+
+	dequeue: function( elem, type ) {
+		type = type || "fx";
+
+		var queue = jQuery.queue( elem, type ),
+			startLength = queue.length,
+			fn = queue.shift(),
+			hooks = jQuery._queueHooks( elem, type ),
+			next = function() {
+				jQuery.dequeue( elem, type );
+			};
+
+		// If the fx queue is dequeued, always remove the progress sentinel
+		if ( fn === "inprogress" ) {
+			fn = queue.shift();
+			startLength--;
+		}
+
+		if ( fn ) {
+
+			// Add a progress sentinel to prevent the fx queue from being
+			// automatically dequeued
+			if ( type === "fx" ) {
+				queue.unshift( "inprogress" );
+			}
+
+			// clear up the last queue stop function
+			delete hooks.stop;
+			fn.call( elem, next, hooks );
+		}
+
+		if ( !startLength && hooks ) {
+			hooks.empty.fire();
+		}
+	},
+
+	// not intended for public consumption - generates a queueHooks object, or returns the current one
+	_queueHooks: function( elem, type ) {
+		var key = type + "queueHooks";
+		return jQuery._data( elem, key ) || jQuery._data( elem, key, {
+			empty: jQuery.Callbacks("once memory").add(function() {
+				jQuery.removeData( elem, type + "queue", true );
+				jQuery.removeData( elem, key, true );
+			})
+		});
+	}
+});
+
+jQuery.fn.extend({
+	queue: function( type, data ) {
+		var setter = 2;
+
+		if ( typeof type !== "string" ) {
+			data = type;
+			type = "fx";
+			setter--;
+		}
+
+		if ( arguments.length < setter ) {
+			return jQuery.queue( this[0], type );
+		}
+
+		return data === undefined ?
+			this :
+			this.each(function() {
+				var queue = jQuery.queue( this, type, data );
+
+				// ensure a hooks for this queue
+				jQuery._queueHooks( this, type );
+
+				if ( type === "fx" && queue[0] !== "inprogress" ) {
+					jQuery.dequeue( this, type );
+				}
+			});
+	},
+	dequeue: function( type ) {
+		return this.each(function() {
+			jQuery.dequeue( this, type );
+		});
+	},
+	// Based off of the plugin by Clint Helfers, with permission.
+	// http://blindsignals.com/index.php/2009/07/jquery-delay/
+	delay: function( time, type ) {
+		time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time;
+		type = type || "fx";
+
+		return this.queue( type, function( next, hooks ) {
+			var timeout = setTimeout( next, time );
+			hooks.stop = function() {
+				clearTimeout( timeout );
+			};
+		});
+	},
+	clearQueue: function( type ) {
+		return this.queue( type || "fx", [] );
+	},
+	// Get a promise resolved when queues of a certain type
+	// are emptied (fx is the type by default)
+	promise: function( type, obj ) {
+		var tmp,
+			count = 1,
+			defer = jQuery.Deferred(),
+			elements = this,
+			i = this.length,
+			resolve = function() {
+				if ( !( --count ) ) {
+					defer.resolveWith( elements, [ elements ] );
+				}
+			};
+
+		if ( typeof type !== "string" ) {
+			obj = type;
+			type = undefined;
+		}
+		type = type || "fx";
+
+		while( i-- ) {
+			tmp = jQuery._data( elements[ i ], type + "queueHooks" );
+			if ( tmp && tmp.empty ) {
+				count++;
+				tmp.empty.add( resolve );
+			}
+		}
+		resolve();
+		return defer.promise( obj );
+	}
+});
+var nodeHook, boolHook, fixSpecified,
+	rclass = /[\t\r\n]/g,
+	rreturn = /\r/g,
+	rtype = /^(?:button|input)$/i,
+	rfocusable = /^(?:button|input|object|select|textarea)$/i,
+	rclickable = /^a(?:rea|)$/i,
+	rboolean = /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i,
+	getSetAttribute = jQuery.support.getSetAttribute;
+
+jQuery.fn.extend({
+	attr: function( name, value ) {
+		return jQuery.access( this, jQuery.attr, name, value, arguments.length > 1 );
+	},
+
+	removeAttr: function( name ) {
+		return this.each(function() {
+			jQuery.removeAttr( this, name );
+		});
+	},
+
+	prop: function( name, value ) {
+		return jQuery.access( this, jQuery.prop, name, value, arguments.length > 1 );
+	},
+
+	removeProp: function( name ) {
+		name = jQuery.propFix[ name ] || name;
+		return this.each(function() {
+			// try/catch handles cases where IE balks (such as removing a property on window)
+			try {
+				this[ name ] = undefined;
+				delete this[ name ];
+			} catch( e ) {}
+		});
+	},
+
+	addClass: function( value ) {
+		var classNames, i, l, elem,
+			setClass, c, cl;
+
+		if ( jQuery.isFunction( value ) ) {
+			return this.each(function( j ) {
+				jQuery( this ).addClass( value.call(this, j, this.className) );
+			});
+		}
+
+		if ( value && typeof value === "string" ) {
+			classNames = value.split( core_rspace );
+
+			for ( i = 0, l = this.length; i < l; i++ ) {
+				elem = this[ i ];
+
+				if ( elem.nodeType === 1 ) {
+					if ( !elem.className && classNames.length === 1 ) {
+						elem.className = value;
+
+					} else {
+						setClass = " " + elem.className + " ";
+
+						for ( c = 0, cl = classNames.length; c < cl; c++ ) {
+							if ( setClass.indexOf( " " + classNames[ c ] + " " ) < 0 ) {
+								setClass += classNames[ c ] + " ";
+							}
+						}
+						elem.className = jQuery.trim( setClass );
+					}
+				}
+			}
+		}
+
+		return this;
+	},
+
+	removeClass: function( value ) {
+		var removes, className, elem, c, cl, i, l;
+
+		if ( jQuery.isFunction( value ) ) {
+			return this.each(function( j ) {
+				jQuery( this ).removeClass( value.call(this, j, this.className) );
+			});
+		}
+		if ( (value && typeof value === "string") || value === undefined ) {
+			removes = ( value || "" ).split( core_rspace );
+
+			for ( i = 0, l = this.length; i < l; i++ ) {
+				elem = this[ i ];
+				if ( elem.nodeType === 1 && elem.className ) {
+
+					className = (" " + elem.className + " ").replace( rclass, " " );
+
+					// loop over each item in the removal list
+					for ( c = 0, cl = removes.length; c < cl; c++ ) {
+						// Remove until there is nothing to remove,
+						while ( className.indexOf(" " + removes[ c ] + " ") >= 0 ) {
+							className = className.replace( " " + removes[ c ] + " " , " " );
+						}
+					}
+					elem.className = value ? jQuery.trim( className ) : "";
+				}
+			}
+		}
+
+		return this;
+	},
+
+	toggleClass: function( value, stateVal ) {
+		var type = typeof value,
+			isBool = typeof stateVal === "boolean";
+
+		if ( jQuery.isFunction( value ) ) {
+			return this.each(function( i ) {
+				jQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal );
+			});
+		}
+
+		return this.each(function() {
+			if ( type === "string" ) {
+				// toggle individual class names
+				var className,
+					i = 0,
+					self = jQuery( this ),
+					state = stateVal,
+					classNames = value.split( core_rspace );
+
+				while ( (className = classNames[ i++ ]) ) {
+					// check each className given, space separated list
+					state = isBool ? state : !self.hasClass( className );
+					self[ state ? "addClass" : "removeClass" ]( className );
+				}
+
+			} else if ( type === "undefined" || type === "boolean" ) {
+				if ( this.className ) {
+					// store className if set
+					jQuery._data( this, "__className__", this.className );
+				}
+
+				// toggle whole className
+				this.className = this.className || value === false ? "" : jQuery._data( this, "__className__" ) || "";
+			}
+		});
+	},
+
+	hasClass: function( selector ) {
+		var className = " " + selector + " ",
+			i = 0,
+			l = this.length;
+		for ( ; i < l; i++ ) {
+			if ( this[i].nodeType === 1 && (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) >= 0 ) {
+				return true;
+			}
+		}
+
+		return false;
+	},
+
+	val: function( value ) {
+		var hooks, ret, isFunction,
+			elem = this[0];
+
+		if ( !arguments.length ) {
+			if ( elem ) {
+				hooks = jQuery.valHooks[ elem.type ] || jQuery.valHooks[ elem.nodeName.toLowerCase() ];
+
+				if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) {
+					return ret;
+				}
+
+				ret = elem.value;
+
+				return typeof ret === "string" ?
+					// handle most common string cases
+					ret.replace(rreturn, "") :
+					// handle cases where value is null/undef or number
+					ret == null ? "" : ret;
+			}
+
+			return;
+		}
+
+		isFunction = jQuery.isFunction( value );
+
+		return this.each(function( i ) {
+			var val,
+				self = jQuery(this);
+
+			if ( this.nodeType !== 1 ) {
+				return;
+			}
+
+			if ( isFunction ) {
+				val = value.call( this, i, self.val() );
+			} else {
+				val = value;
+			}
+
+			// Treat null/undefined as ""; convert numbers to string
+			if ( val == null ) {
+				val = "";
+			} else if ( typeof val === "number" ) {
+				val += "";
+			} else if ( jQuery.isArray( val ) ) {
+				val = jQuery.map(val, function ( value ) {
+					return value == null ? "" : value + "";
+				});
+			}
+
+			hooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ];
+
+			// If set returns undefined, fall back to normal setting
+			if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) {
+				this.value = val;
+			}
+		});
+	}
+});
+
+jQuery.extend({
+	valHooks: {
+		option: {
+			get: function( elem ) {
+				// attributes.value is undefined in Blackberry 4.7 but
+				// uses .value. See #6932
+				var val = elem.attributes.value;
+				return !val || val.specified ? elem.value : elem.text;
+			}
+		},
+		select: {
+			get: function( elem ) {
+				var value, option,
+					options = elem.options,
+					index = elem.selectedIndex,
+					one = elem.type === "select-one" || index < 0,
+					values = one ? null : [],
+					max = one ? index + 1 : options.length,
+					i = index < 0 ?
+						max :
+						one ? index : 0;
+
+				// Loop through all the selected options
+				for ( ; i < max; i++ ) {
+					option = options[ i ];
+
+					// oldIE doesn't update selected after form reset (#2551)
+					if ( ( option.selected || i === index ) &&
+							// Don't return options that are disabled or in a disabled optgroup
+							( jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null ) &&
+							( !option.parentNode.disabled || !jQuery.nodeName( option.parentNode, "optgroup" ) ) ) {
+
+						// Get the specific value for the option
+						value = jQuery( option ).val();
+
+						// We don't need an array for one selects
+						if ( one ) {
+							return value;
+						}
+
+						// Multi-Selects return an array
+						values.push( value );
+					}
+				}
+
+				return values;
+			},
+
+			set: function( elem, value ) {
+				var values = jQuery.makeArray( value );
+
+				jQuery(elem).find("option").each(function() {
+					this.selected = jQuery.inArray( jQuery(this).val(), values ) >= 0;
+				});
+
+				if ( !values.length ) {
+					elem.selectedIndex = -1;
+				}
+				return values;
+			}
+		}
+	},
+
+	// Unused in 1.8, left in so attrFn-stabbers won't die; remove in 1.9
+	attrFn: {},
+
+	attr: function( elem, name, value, pass ) {
+		var ret, hooks, notxml,
+			nType = elem.nodeType;
+
+		// don't get/set attributes on text, comment and attribute nodes
+		if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
+			return;
+		}
+
+		if ( pass && jQuery.isFunction( jQuery.fn[ name ] ) ) {
+			return jQuery( elem )[ name ]( value );
+		}
+
+		// Fallback to prop when attributes are not supported
+		if ( typeof elem.getAttribute === "undefined" ) {
+			return jQuery.prop( elem, name, value );
+		}
+
+		notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
+
+		// All attributes are lowercase
+		// Grab necessary hook if one is defined
+		if ( notxml ) {
+			name = name.toLowerCase();
+			hooks = jQuery.attrHooks[ name ] || ( rboolean.test( name ) ? boolHook : nodeHook );
+		}
+
+		if ( value !== undefined ) {
+
+			if ( value === null ) {
+				jQuery.removeAttr( elem, name );
+				return;
+
+			} else if ( hooks && "set" in hooks && notxml && (ret = hooks.set( elem, value, name )) !== undefined ) {
+				return ret;
+
+			} else {
+				elem.setAttribute( name, value + "" );
+				return value;
+			}
+
+		} else if ( hooks && "get" in hooks && notxml && (ret = hooks.get( elem, name )) !== null ) {
+			return ret;
+
+		} else {
+
+			ret = elem.getAttribute( name );
+
+			// Non-existent attributes return null, we normalize to undefined
+			return ret === null ?
+				undefined :
+				ret;
+		}
+	},
+
+	removeAttr: function( elem, value ) {
+		var propName, attrNames, name, isBool,
+			i = 0;
+
+		if ( value && elem.nodeType === 1 ) {
+
+			attrNames = value.split( core_rspace );
+
+			for ( ; i < attrNames.length; i++ ) {
+				name = attrNames[ i ];
+
+				if ( name ) {
+					propName = jQuery.propFix[ name ] || name;
+					isBool = rboolean.test( name );
+
+					// See #9699 for explanation of this approach (setting first, then removal)
+					// Do not do this for boolean attributes (see #10870)
+					if ( !isBool ) {
+						jQuery.attr( elem, name, "" );
+					}
+					elem.removeAttribute( getSetAttribute ? name : propName );
+
+					// Set corresponding property to false for boolean attributes
+					if ( isBool && propName in elem ) {
+						elem[ propName ] = false;
+					}
+				}
+			}
+		}
+	},
+
+	attrHooks: {
+		type: {
+			set: function( elem, value ) {
+				// We can't allow the type property to be changed (since it causes problems in IE)
+				if ( rtype.test( elem.nodeName ) && elem.parentNode ) {
+					jQuery.error( "type property can't be changed" );
+				} else if ( !jQuery.support.radioValue && value === "radio" && jQuery.nodeName(elem, "input") ) {
+					// Setting the type on a radio button after the value resets the value in IE6-9
+					// Reset value to it's default in case type is set after value
+					// This is for element creation
+					var val = elem.value;
+					elem.setAttribute( "type", value );
+					if ( val ) {
+						elem.value = val;
+					}
+					return value;
+				}
+			}
+		},
+		// Use the value property for back compat
+		// Use the nodeHook for button elements in IE6/7 (#1954)
+		value: {
+			get: function( elem, name ) {
+				if ( nodeHook && jQuery.nodeName( elem, "button" ) ) {
+					return nodeHook.get( elem, name );
+				}
+				return name in elem ?
+					elem.value :
+					null;
+			},
+			set: function( elem, value, name ) {
+				if ( nodeHook && jQuery.nodeName( elem, "button" ) ) {
+					return nodeHook.set( elem, value, name );
+				}
+				// Does not return so that setAttribute is also used
+				elem.value = value;
+			}
+		}
+	},
+
+	propFix: {
+		tabindex: "tabIndex",
+		readonly: "readOnly",
+		"for": "htmlFor",
+		"class": "className",
+		maxlength: "maxLength",
+		cellspacing: "cellSpacing",
+		cellpadding: "cellPadding",
+		rowspan: "rowSpan",
+		colspan: "colSpan",
+		usemap: "useMap",
+		frameborder: "frameBorder",
+		contenteditable: "contentEditable"
+	},
+
+	prop: function( elem, name, value ) {
+		var ret, hooks, notxml,
+			nType = elem.nodeType;
+
+		// don't get/set properties on text, comment and attribute nodes
+		if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
+			return;
+		}
+
+		notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
+
+		if ( notxml ) {
+			// Fix name and attach hooks
+			name = jQuery.propFix[ name ] || name;
+			hooks = jQuery.propHooks[ name ];
+		}
+
+		if ( value !== undefined ) {
+			if ( hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) {
+				return ret;
+
+			} else {
+				return ( elem[ name ] = value );
+			}
+
+		} else {
+			if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ) {
+				return ret;
+
+			} else {
+				return elem[ name ];
+			}
+		}
+	},
+
+	propHooks: {
+		tabIndex: {
+			get: function( elem ) {
+				// elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set
+				// http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/
+				var attributeNode = elem.getAttributeNode("tabindex");
+
+				return attributeNode && attributeNode.specified ?
+					parseInt( attributeNode.value, 10 ) :
+					rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ?
+						0 :
+						undefined;
+			}
+		}
+	}
+});
+
+// Hook for boolean attributes
+boolHook = {
+	get: function( elem, name ) {
+		// Align boolean attributes with corresponding properties
+		// Fall back to attribute presence where some booleans are not supported
+		var attrNode,
+			property = jQuery.prop( elem, name );
+		return property === true || typeof property !== "boolean" && ( attrNode = elem.getAttributeNode(name) ) && attrNode.nodeValue !== false ?
+			name.toLowerCase() :
+			undefined;
+	},
+	set: function( elem, value, name ) {
+		var propName;
+		if ( value === false ) {
+			// Remove boolean attributes when set to false
+			jQuery.removeAttr( elem, name );
+		} else {
+			// value is true since we know at this point it's type boolean and not false
+			// Set boolean attributes to the same name and set the DOM property
+			propName = jQuery.propFix[ name ] || name;
+			if ( propName in elem ) {
+				// Only set the IDL specifically if it already exists on the element
+				elem[ propName ] = true;
+			}
+
+			elem.setAttribute( name, name.toLowerCase() );
+		}
+		return name;
+	}
+};
+
+// IE6/7 do not support getting/setting some attributes with get/setAttribute
+if ( !getSetAttribute ) {
+
+	fixSpecified = {
+		name: true,
+		id: true,
+		coords: true
+	};
+
+	// Use this for any attribute in IE6/7
+	// This fixes almost every IE6/7 issue
+	nodeHook = jQuery.valHooks.button = {
+		get: function( elem, name ) {
+			var ret;
+			ret = elem.getAttributeNode( name );
+			return ret && ( fixSpecified[ name ] ? ret.value !== "" : ret.specified ) ?
+				ret.value :
+				undefined;
+		},
+		set: function( elem, value, name ) {
+			// Set the existing or create a new attribute node
+			var ret = elem.getAttributeNode( name );
+			if ( !ret ) {
+				ret = document.createAttribute( name );
+				elem.setAttributeNode( ret );
+			}
+			return ( ret.value = value + "" );
+		}
+	};
+
+	// Set width and height to auto instead of 0 on empty string( Bug #8150 )
+	// This is for removals
+	jQuery.each([ "width", "height" ], function( i, name ) {
+		jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], {
+			set: function( elem, value ) {
+				if ( value === "" ) {
+					elem.setAttribute( name, "auto" );
+					return value;
+				}
+			}
+		});
+	});
+
+	// Set contenteditable to false on removals(#10429)
+	// Setting to empty string throws an error as an invalid value
+	jQuery.attrHooks.contenteditable = {
+		get: nodeHook.get,
+		set: function( elem, value, name ) {
+			if ( value === "" ) {
+				value = "false";
+			}
+			nodeHook.set( elem, value, name );
+		}
+	};
+}
+
+
+// Some attributes require a special call on IE
+if ( !jQuery.support.hrefNormalized ) {
+	jQuery.each([ "href", "src", "width", "height" ], function( i, name ) {
+		jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], {
+			get: function( elem ) {
+				var ret = elem.getAttribute( name, 2 );
+				return ret === null ? undefined : ret;
+			}
+		});
+	});
+}
+
+if ( !jQuery.support.style ) {
+	jQuery.attrHooks.style = {
+		get: function( elem ) {
+			// Return undefined in the case of empty string
+			// Normalize to lowercase since IE uppercases css property names
+			return elem.style.cssText.toLowerCase() || undefined;
+		},
+		set: function( elem, value ) {
+			return ( elem.style.cssText = value + "" );
+		}
+	};
+}
+
+// Safari mis-reports the default selected property of an option
+// Accessing the parent's selectedIndex property fixes it
+if ( !jQuery.support.optSelected ) {
+	jQuery.propHooks.selected = jQuery.extend( jQuery.propHooks.selected, {
+		get: function( elem ) {
+			var parent = elem.parentNode;
+
+			if ( parent ) {
+				parent.selectedIndex;
+
+				// Make sure that it also works with optgroups, see #5701
+				if ( parent.parentNode ) {
+					parent.parentNode.selectedIndex;
+				}
+			}
+			return null;
+		}
+	});
+}
+
+// IE6/7 call enctype encoding
+if ( !jQuery.support.enctype ) {
+	jQuery.propFix.enctype = "encoding";
+}
+
+// Radios and checkboxes getter/setter
+if ( !jQuery.support.checkOn ) {
+	jQuery.each([ "radio", "checkbox" ], function() {
+		jQuery.valHooks[ this ] = {
+			get: function( elem ) {
+				// Handle the case where in Webkit "" is returned instead of "on" if a value isn't specified
+				return elem.getAttribute("value") === null ? "on" : elem.value;
+			}
+		};
+	});
+}
+jQuery.each([ "radio", "checkbox" ], function() {
+	jQuery.valHooks[ this ] = jQuery.extend( jQuery.valHooks[ this ], {
+		set: function( elem, value ) {
+			if ( jQuery.isArray( value ) ) {
+				return ( elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0 );
+			}
+		}
+	});
+});
+var rformElems = /^(?:textarea|input|select)$/i,
+	rtypenamespace = /^([^\.]*|)(?:\.(.+)|)$/,
+	rhoverHack = /(?:^|\s)hover(\.\S+|)\b/,
+	rkeyEvent = /^key/,
+	rmouseEvent = /^(?:mouse|contextmenu)|click/,
+	rfocusMorph = /^(?:focusinfocus|focusoutblur)$/,
+	hoverHack = function( events ) {
+		return jQuery.event.special.hover ? events : events.replace( rhoverHack, "mouseenter$1 mouseleave$1" );
+	};
+
+/*
+ * Helper functions for managing events -- not part of the public interface.
+ * Props to Dean Edwards' addEvent library for many of the ideas.
+ */
+jQuery.event = {
+
+	add: function( elem, types, handler, data, selector ) {
+
+		var elemData, eventHandle, events,
+			t, tns, type, namespaces, handleObj,
+			handleObjIn, handlers, special;
+
+		// Don't attach events to noData or text/comment nodes (allow plain objects tho)
+		if ( elem.nodeType === 3 || elem.nodeType === 8 || !types || !handler || !(elemData = jQuery._data( elem )) ) {
+			return;
+		}
+
+		// Caller can pass in an object of custom data in lieu of the handler
+		if ( handler.handler ) {
+			handleObjIn = handler;
+			handler = handleObjIn.handler;
+			selector = handleObjIn.selector;
+		}
+
+		// Make sure that the handler has a unique ID, used to find/remove it later
+		if ( !handler.guid ) {
+			handler.guid = jQuery.guid++;
+		}
+
+		// Init the element's event structure and main handler, if this is the first
+		events = elemData.events;
+		if ( !events ) {
+			elemData.events = events = {};
+		}
+		eventHandle = elemData.handle;
+		if ( !eventHandle ) {
+			elemData.handle = eventHandle = function( e ) {
+				// Discard the second event of a jQuery.event.trigger() and
+				// when an event is called after a page has unloaded
+				return typeof jQuery !== "undefined" && (!e || jQuery.event.triggered !== e.type) ?
+					jQuery.event.dispatch.apply( eventHandle.elem, arguments ) :
+					undefined;
+			};
+			// Add elem as a property of the handle fn to prevent a memory leak with IE non-native events
+			eventHandle.elem = elem;
+		}
+
+		// Handle multiple events separated by a space
+		// jQuery(...).bind("mouseover mouseout", fn);
+		types = jQuery.trim( hoverHack(types) ).split( " " );
+		for ( t = 0; t < types.length; t++ ) {
+
+			tns = rtypenamespace.exec( types[t] ) || [];
+			type = tns[1];
+			namespaces = ( tns[2] || "" ).split( "." ).sort();
+
+			// If event changes its type, use the special event handlers for the changed type
+			special = jQuery.event.special[ type ] || {};
+
+			// If selector defined, determine special event api type, otherwise given type
+			type = ( selector ? special.delegateType : special.bindType ) || type;
+
+			// Update special based on newly reset type
+			special = jQuery.event.special[ type ] || {};
+
+			// handleObj is passed to all event handlers
+			handleObj = jQuery.extend({
+				type: type,
+				origType: tns[1],
+				data: data,
+				handler: handler,
+				guid: handler.guid,
+				selector: selector,
+				needsContext: selector && jQuery.expr.match.needsContext.test( selector ),
+				namespace: namespaces.join(".")
+			}, handleObjIn );
+
+			// Init the event handler queue if we're the first
+			handlers = events[ type ];
+			if ( !handlers ) {
+				handlers = events[ type ] = [];
+				handlers.delegateCount = 0;
+
+				// Only use addEventListener/attachEvent if the special events handler returns false
+				if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) {
+					// Bind the global event handler to the element
+					if ( elem.addEventListener ) {
+						elem.addEventListener( type, eventHandle, false );
+
+					} else if ( elem.attachEvent ) {
+						elem.attachEvent( "on" + type, eventHandle );
+					}
+				}
+			}
+
+			if ( special.add ) {
+				special.add.call( elem, handleObj );
+
+				if ( !handleObj.handler.guid ) {
+					handleObj.handler.guid = handler.guid;
+				}
+			}
+
+			// Add to the element's handler list, delegates in front
+			if ( selector ) {
+				handlers.splice( handlers.delegateCount++, 0, handleObj );
+			} else {
+				handlers.push( handleObj );
+			}
+
+			// Keep track of which events have ever been used, for event optimization
+			jQuery.event.global[ type ] = true;
+		}
+
+		// Nullify elem to prevent memory leaks in IE
+		elem = null;
+	},
+
+	global: {},
+
+	// Detach an event or set of events from an element
+	remove: function( elem, types, handler, selector, mappedTypes ) {
+
+		var t, tns, type, origType, namespaces, origCount,
+			j, events, special, eventType, handleObj,
+			elemData = jQuery.hasData( elem ) && jQuery._data( elem );
+
+		if ( !elemData || !(events = elemData.events) ) {
+			return;
+		}
+
+		// Once for each type.namespace in types; type may be omitted
+		types = jQuery.trim( hoverHack( types || "" ) ).split(" ");
+		for ( t = 0; t < types.length; t++ ) {
+			tns = rtypenamespace.exec( types[t] ) || [];
+			type = origType = tns[1];
+			namespaces = tns[2];
+
+			// Unbind all events (on this namespace, if provided) for the element
+			if ( !type ) {
+				for ( type in events ) {
+					jQuery.event.remove( elem, type + types[ t ], handler, selector, true );
+				}
+				continue;
+			}
+
+			special = jQuery.event.special[ type ] || {};
+			type = ( selector? special.delegateType : special.bindType ) || type;
+			eventType = events[ type ] || [];
+			origCount = eventType.length;
+			namespaces = namespaces ? new RegExp("(^|\\.)" + namespaces.split(".").sort().join("\\.(?:.*\\.|)") + "(\\.|$)") : null;
+
+			// Remove matching events
+			for ( j = 0; j < eventType.length; j++ ) {
+				handleObj = eventType[ j ];
+
+				if ( ( mappedTypes || origType === handleObj.origType ) &&
+					 ( !handler || handler.guid === handleObj.guid ) &&
+					 ( !namespaces || namespaces.test( handleObj.namespace ) ) &&
+					 ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) {
+					eventType.splice( j--, 1 );
+
+					if ( handleObj.selector ) {
+						eventType.delegateCount--;
+					}
+					if ( special.remove ) {
+						special.remove.call( elem, handleObj );
+					}
+				}
+			}
+
+			// Remove generic event handler if we removed something and no more handlers exist
+			// (avoids potential for endless recursion during removal of special event handlers)
+			if ( eventType.length === 0 && origCount !== eventType.length ) {
+				if ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) {
+					jQuery.removeEvent( elem, type, elemData.handle );
+				}
+
+				delete events[ type ];
+			}
+		}
+
+		// Remove the expando if it's no longer used
+		if ( jQuery.isEmptyObject( events ) ) {
+			delete elemData.handle;
+
+			// removeData also checks for emptiness and clears the expando if empty
+			// so use it instead of delete
+			jQuery.removeData( elem, "events", true );
+		}
+	},
+
+	// Events that are safe to short-circuit if no handlers are attached.
+	// Native DOM events should not be added, they may have inline handlers.
+	customEvent: {
+		"getData": true,
+		"setData": true,
+		"changeData": true
+	},
+
+	trigger: function( event, data, elem, onlyHandlers ) {
+		// Don't do events on text and comment nodes
+		if ( elem && (elem.nodeType === 3 || elem.nodeType === 8) ) {
+			return;
+		}
+
+		// Event object or event type
+		var cache, exclusive, i, cur, old, ontype, special, handle, eventPath, bubbleType,
+			type = event.type || event,
+			namespaces = [];
+
+		// focus/blur morphs to focusin/out; ensure we're not firing them right now
+		if ( rfocusMorph.test( type + jQuery.event.triggered ) ) {
+			return;
+		}
+
+		if ( type.indexOf( "!" ) >= 0 ) {
+			// Exclusive events trigger only for the exact event (no namespaces)
+			type = type.slice(0, -1);
+			exclusive = true;
+		}
+
+		if ( type.indexOf( "." ) >= 0 ) {
+			// Namespaced trigger; create a regexp to match event type in handle()
+			namespaces = type.split(".");
+			type = namespaces.shift();
+			namespaces.sort();
+		}
+
+		if ( (!elem || jQuery.event.customEvent[ type ]) && !jQuery.event.global[ type ] ) {
+			// No jQuery handlers for this event type, and it can't have inline handlers
+			return;
+		}
+
+		// Caller can pass in an Event, Object, or just an event type string
+		event = typeof event === "object" ?
+			// jQuery.Event object
+			event[ jQuery.expando ] ? event :
+			// Object literal
+			new jQuery.Event( type, event ) :
+			// Just the event type (string)
+			new jQuery.Event( type );
+
+		event.type = type;
+		event.isTrigger = true;
+		event.exclusive = exclusive;
+		event.namespace = namespaces.join( "." );
+		event.namespace_re = event.namespace? new RegExp("(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)") : null;
+		ontype = type.indexOf( ":" ) < 0 ? "on" + type : "";
+
+		// Handle a global trigger
+		if ( !elem ) {
+
+			// TODO: Stop taunting the data cache; remove global events and always attach to document
+			cache = jQuery.cache;
+			for ( i in cache ) {
+				if ( cache[ i ].events && cache[ i ].events[ type ] ) {
+					jQuery.event.trigger( event, data, cache[ i ].handle.elem, true );
+				}
+			}
+			return;
+		}
+
+		// Clean up the event in case it is being reused
+		event.result = undefined;
+		if ( !event.target ) {
+			event.target = elem;
+		}
+
+		// Clone any incoming data and prepend the event, creating the handler arg list
+		data = data != null ? jQuery.makeArray( data ) : [];
+		data.unshift( event );
+
+		// Allow special events to draw outside the lines
+		special = jQuery.event.special[ type ] || {};
+		if ( special.trigger && special.trigger.apply( elem, data ) === false ) {
+			return;
+		}
+
+		// Determine event propagation path in advance, per W3C events spec (#9951)
+		// Bubble up to document, then to window; watch for a global ownerDocument var (#9724)
+		eventPath = [[ elem, special.bindType || type ]];
+		if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) {
+
+			bubbleType = special.delegateType || type;
+			cur = rfocusMorph.test( bubbleType + type ) ? elem : elem.parentNode;
+			for ( old = elem; cur; cur = cur.parentNode ) {
+				eventPath.push([ cur, bubbleType ]);
+				old = cur;
+			}
+
+			// Only add window if we got to document (e.g., not plain obj or detached DOM)
+			if ( old === (elem.ownerDocument || document) ) {
+				eventPath.push([ old.defaultView || old.parentWindow || window, bubbleType ]);
+			}
+		}
+
+		// Fire handlers on the event path
+		for ( i = 0; i < eventPath.length && !event.isPropagationStopped(); i++ ) {
+
+			cur = eventPath[i][0];
+			event.type = eventPath[i][1];
+
+			handle = ( jQuery._data( cur, "events" ) || {} )[ event.type ] && jQuery._data( cur, "handle" );
+			if ( handle ) {
+				handle.apply( cur, data );
+			}
+			// Note that this is a bare JS function and not a jQuery handler
+			handle = ontype && cur[ ontype ];
+			if ( handle && jQuery.acceptData( cur ) && handle.apply && handle.apply( cur, data ) === false ) {
+				event.preventDefault();
+			}
+		}
+		event.type = type;
+
+		// If nobody prevented the default action, do it now
+		if ( !onlyHandlers && !event.isDefaultPrevented() ) {
+
+			if ( (!special._default || special._default.apply( elem.ownerDocument, data ) === false) &&
+				!(type === "click" && jQuery.nodeName( elem, "a" )) && jQuery.acceptData( elem ) ) {
+
+				// Call a native DOM method on the target with the same name name as the event.
+				// Can't use an .isFunction() check here because IE6/7 fails that test.
+				// Don't do default actions on window, that's where global variables be (#6170)
+				// IE<9 dies on focus/blur to hidden element (#1486)
+				if ( ontype && elem[ type ] && ((type !== "focus" && type !== "blur") || event.target.offsetWidth !== 0) && !jQuery.isWindow( elem ) ) {
+
+					// Don't re-trigger an onFOO event when we call its FOO() method
+					old = elem[ ontype ];
+
+					if ( old ) {
+						elem[ ontype ] = null;
+					}
+
+					// Prevent re-triggering of the same event, since we already bubbled it above
+					jQuery.event.triggered = type;
+					elem[ type ]();
+					jQuery.event.triggered = undefined;
+
+					if ( old ) {
+						elem[ ontype ] = old;
+					}
+				}
+			}
+		}
+
+		return event.result;
+	},
+
+	dispatch: function( event ) {
+
+		// Make a writable jQuery.Event from the native event object
+		event = jQuery.event.fix( event || window.event );
+
+		var i, j, cur, ret, selMatch, matched, matches, handleObj, sel, related,
+			handlers = ( (jQuery._data( this, "events" ) || {} )[ event.type ] || []),
+			delegateCount = handlers.delegateCount,
+			args = core_slice.call( arguments ),
+			run_all = !event.exclusive && !event.namespace,
+			special = jQuery.event.special[ event.type ] || {},
+			handlerQueue = [];
+
+		// Use the fix-ed jQuery.Event rather than the (read-only) native event
+		args[0] = event;
+		event.delegateTarget = this;
+
+		// Call the preDispatch hook for the mapped type, and let it bail if desired
+		if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) {
+			return;
+		}
+
+		// Determine handlers that should run if there are delegated events
+		// Avoid non-left-click bubbling in Firefox (#3861)
+		if ( delegateCount && !(event.button && event.type === "click") ) {
+
+			for ( cur = event.target; cur != this; cur = cur.parentNode || this ) {
+
+				// Don't process clicks (ONLY) on disabled elements (#6911, #8165, #11382, #11764)
+				if ( cur.disabled !== true || event.type !== "click" ) {
+					selMatch = {};
+					matches = [];
+					for ( i = 0; i < delegateCount; i++ ) {
+						handleObj = handlers[ i ];
+						sel = handleObj.selector;
+
+						if ( selMatch[ sel ] === undefined ) {
+							selMatch[ sel ] = handleObj.needsContext ?
+								jQuery( sel, this ).index( cur ) >= 0 :
+								jQuery.find( sel, this, null, [ cur ] ).length;
+						}
+						if ( selMatch[ sel ] ) {
+							matches.push( handleObj );
+						}
+					}
+					if ( matches.length ) {
+						handlerQueue.push({ elem: cur, matches: matches });
+					}
+				}
+			}
+		}
+
+		// Add the remaining (directly-bound) handlers
+		if ( handlers.length > delegateCount ) {
+			handlerQueue.push({ elem: this, matches: handlers.slice( delegateCount ) });
+		}
+
+		// Run delegates first; they may want to stop propagation beneath us
+		for ( i = 0; i < handlerQueue.length && !event.isPropagationStopped(); i++ ) {
+			matched = handlerQueue[ i ];
+			event.currentTarget = matched.elem;
+
+			for ( j = 0; j < matched.matches.length && !event.isImmediatePropagationStopped(); j++ ) {
+				handleObj = matched.matches[ j ];
+
+				// Triggered event must either 1) be non-exclusive and have no namespace, or
+				// 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace).
+				if ( run_all || (!event.namespace && !handleObj.namespace) || event.namespace_re && event.namespace_re.test( handleObj.namespace ) ) {
+
+					event.data = handleObj.data;
+					event.handleObj = handleObj;
+
+					ret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler )
+							.apply( matched.elem, args );
+
+					if ( ret !== undefined ) {
+						event.result = ret;
+						if ( ret === false ) {
+							event.preventDefault();
+							event.stopPropagation();
+						}
+					}
+				}
+			}
+		}
+
+		// Call the postDispatch hook for the mapped type
+		if ( special.postDispatch ) {
+			special.postDispatch.call( this, event );
+		}
+
+		return event.result;
+	},
+
+	// Includes some event props shared by KeyEvent and MouseEvent
+	// *** attrChange attrName relatedNode srcElement  are not normalized, non-W3C, deprecated, will be removed in 1.8 ***
+	props: "attrChange attrName relatedNode srcElement altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "),
+
+	fixHooks: {},
+
+	keyHooks: {
+		props: "char charCode key keyCode".split(" "),
+		filter: function( event, original ) {
+
+			// Add which for key events
+			if ( event.which == null ) {
+				event.which = original.charCode != null ? original.charCode : original.keyCode;
+			}
+
+			return event;
+		}
+	},
+
+	mouseHooks: {
+		props: "button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "),
+		filter: function( event, original ) {
+			var eventDoc, doc, body,
+				button = original.button,
+				fromElement = original.fromElement;
+
+			// Calculate pageX/Y if missing and clientX/Y available
+			if ( event.pageX == null && original.clientX != null ) {
+				eventDoc = event.target.ownerDocument || document;
+				doc = eventDoc.documentElement;
+				body = eventDoc.body;
+
+				event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 );
+				event.pageY = original.clientY + ( doc && doc.scrollTop  || body && body.scrollTop  || 0 ) - ( doc && doc.clientTop  || body && body.clientTop  || 0 );
+			}
+
+			// Add relatedTarget, if necessary
+			if ( !event.relatedTarget && fromElement ) {
+				event.relatedTarget = fromElement === event.target ? original.toElement : fromElement;
+			}
+
+			// Add which for click: 1 === left; 2 === middle; 3 === right
+			// Note: button is not normalized, so don't use it
+			if ( !event.which && button !== undefined ) {
+				event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) );
+			}
+
+			return event;
+		}
+	},
+
+	fix: function( event ) {
+		if ( event[ jQuery.expando ] ) {
+			return event;
+		}
+
+		// Create a writable copy of the event object and normalize some properties
+		var i, prop,
+			originalEvent = event,
+			fixHook = jQuery.event.fixHooks[ event.type ] || {},
+			copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props;
+
+		event = jQuery.Event( originalEvent );
+
+		for ( i = copy.length; i; ) {
+			prop = copy[ --i ];
+			event[ prop ] = originalEvent[ prop ];
+		}
+
+		// Fix target property, if necessary (#1925, IE 6/7/8 & Safari2)
+		if ( !event.target ) {
+			event.target = originalEvent.srcElement || document;
+		}
+
+		// Target should not be a text node (#504, Safari)
+		if ( event.target.nodeType === 3 ) {
+			event.target = event.target.parentNode;
+		}
+
+		// For mouse/key events, metaKey==false if it's undefined (#3368, #11328; IE6/7/8)
+		event.metaKey = !!event.metaKey;
+
+		return fixHook.filter? fixHook.filter( event, originalEvent ) : event;
+	},
+
+	special: {
+		load: {
+			// Prevent triggered image.load events from bubbling to window.load
+			noBubble: true
+		},
+
+		focus: {
+			delegateType: "focusin"
+		},
+		blur: {
+			delegateType: "focusout"
+		},
+
+		beforeunload: {
+			setup: function( data, namespaces, eventHandle ) {
+				// We only want to do this special case on windows
+				if ( jQuery.isWindow( this ) ) {
+					this.onbeforeunload = eventHandle;
+				}
+			},
+
+			teardown: function( namespaces, eventHandle ) {
+				if ( this.onbeforeunload === eventHandle ) {
+					this.onbeforeunload = null;
+				}
+			}
+		}
+	},
+
+	simulate: function( type, elem, event, bubble ) {
+		// Piggyback on a donor event to simulate a different one.
+		// Fake originalEvent to avoid donor's stopPropagation, but if the
+		// simulated event prevents default then we do the same on the donor.
+		var e = jQuery.extend(
+			new jQuery.Event(),
+			event,
+			{ type: type,
+				isSimulated: true,
+				originalEvent: {}
+			}
+		);
+		if ( bubble ) {
+			jQuery.event.trigger( e, null, elem );
+		} else {
+			jQuery.event.dispatch.call( elem, e );
+		}
+		if ( e.isDefaultPrevented() ) {
+			event.preventDefault();
+		}
+	}
+};
+
+// Some plugins are using, but it's undocumented/deprecated and will be removed.
+// The 1.7 special event interface should provide all the hooks needed now.
+jQuery.event.handle = jQuery.event.dispatch;
+
+jQuery.removeEvent = document.removeEventListener ?
+	function( elem, type, handle ) {
+		if ( elem.removeEventListener ) {
+			elem.removeEventListener( type, handle, false );
+		}
+	} :
+	function( elem, type, handle ) {
+		var name = "on" + type;
+
+		if ( elem.detachEvent ) {
+
+			// #8545, #7054, preventing memory leaks for custom events in IE6-8
+			// detachEvent needed property on element, by name of that event, to properly expose it to GC
+			if ( typeof elem[ name ] === "undefined" ) {
+				elem[ name ] = null;
+			}
+
+			elem.detachEvent( name, handle );
+		}
+	};
+
+jQuery.Event = function( src, props ) {
+	// Allow instantiation without the 'new' keyword
+	if ( !(this instanceof jQuery.Event) ) {
+		return new jQuery.Event( src, props );
+	}
+
+	// Event object
+	if ( src && src.type ) {
+		this.originalEvent = src;
+		this.type = src.type;
+
+		// Events bubbling up the document may have been marked as prevented
+		// by a handler lower down the tree; reflect the correct value.
+		this.isDefaultPrevented = ( src.defaultPrevented || src.returnValue === false ||
+			src.getPreventDefault && src.getPreventDefault() ) ? returnTrue : returnFalse;
+
+	// Event type
+	} else {
+		this.type = src;
+	}
+
+	// Put explicitly provided properties onto the event object
+	if ( props ) {
+		jQuery.extend( this, props );
+	}
+
+	// Create a timestamp if incoming event doesn't have one
+	this.timeStamp = src && src.timeStamp || jQuery.now();
+
+	// Mark it as fixed
+	this[ jQuery.expando ] = true;
+};
+
+function returnFalse() {
+	return false;
+}
+function returnTrue() {
+	return true;
+}
+
+// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding
+// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html
+jQuery.Event.prototype = {
+	preventDefault: function() {
+		this.isDefaultPrevented = returnTrue;
+
+		var e = this.originalEvent;
+		if ( !e ) {
+			return;
+		}
+
+		// if preventDefault exists run it on the original event
+		if ( e.preventDefault ) {
+			e.preventDefault();
+
+		// otherwise set the returnValue property of the original event to false (IE)
+		} else {
+			e.returnValue = false;
+		}
+	},
+	stopPropagation: function() {
+		this.isPropagationStopped = returnTrue;
+
+		var e = this.originalEvent;
+		if ( !e ) {
+			return;
+		}
+		// if stopPropagation exists run it on the original event
+		if ( e.stopPropagation ) {
+			e.stopPropagation();
+		}
+		// otherwise set the cancelBubble property of the original event to true (IE)
+		e.cancelBubble = true;
+	},
+	stopImmediatePropagation: function() {
+		this.isImmediatePropagationStopped = returnTrue;
+		this.stopPropagation();
+	},
+	isDefaultPrevented: returnFalse,
+	isPropagationStopped: returnFalse,
+	isImmediatePropagationStopped: returnFalse
+};
+
+// Create mouseenter/leave events using mouseover/out and event-time checks
+jQuery.each({
+	mouseenter: "mouseover",
+	mouseleave: "mouseout"
+}, function( orig, fix ) {
+	jQuery.event.special[ orig ] = {
+		delegateType: fix,
+		bindType: fix,
+
+		handle: function( event ) {
+			var ret,
+				target = this,
+				related = event.relatedTarget,
+				handleObj = event.handleObj,
+				selector = handleObj.selector;
+
+			// For mousenter/leave call the handler if related is outside the target.
+			// NB: No relatedTarget if the mouse left/entered the browser window
+			if ( !related || (related !== target && !jQuery.contains( target, related )) ) {
+				event.type = handleObj.origType;
+				ret = handleObj.handler.apply( this, arguments );
+				event.type = fix;
+			}
+			return ret;
+		}
+	};
+});
+
+// IE submit delegation
+if ( !jQuery.support.submitBubbles ) {
+
+	jQuery.event.special.submit = {
+		setup: function() {
+			// Only need this for delegated form submit events
+			if ( jQuery.nodeName( this, "form" ) ) {
+				return false;
+			}
+
+			// Lazy-add a submit handler when a descendant form may potentially be submitted
+			jQuery.event.add( this, "click._submit keypress._submit", function( e ) {
+				// Node name check avoids a VML-related crash in IE (#9807)
+				var elem = e.target,
+					form = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.form : undefined;
+				if ( form && !jQuery._data( form, "_submit_attached" ) ) {
+					jQuery.event.add( form, "submit._submit", function( event ) {
+						event._submit_bubble = true;
+					});
+					jQuery._data( form, "_submit_attached", true );
+				}
+			});
+			// return undefined since we don't need an event listener
+		},
+
+		postDispatch: function( event ) {
+			// If form was submitted by the user, bubble the event up the tree
+			if ( event._submit_bubble ) {
+				delete event._submit_bubble;
+				if ( this.parentNode && !event.isTrigger ) {
+					jQuery.event.simulate( "submit", this.parentNode, event, true );
+				}
+			}
+		},
+
+		teardown: function() {
+			// Only need this for delegated form submit events
+			if ( jQuery.nodeName( this, "form" ) ) {
+				return false;
+			}
+
+			// Remove delegated handlers; cleanData eventually reaps submit handlers attached above
+			jQuery.event.remove( this, "._submit" );
+		}
+	};
+}
+
+// IE change delegation and checkbox/radio fix
+if ( !jQuery.support.changeBubbles ) {
+
+	jQuery.event.special.change = {
+
+		setup: function() {
+
+			if ( rformElems.test( this.nodeName ) ) {
+				// IE doesn't fire change on a check/radio until blur; trigger it on click
+				// after a propertychange. Eat the blur-change in special.change.handle.
+				// This still fires onchange a second time for check/radio after blur.
+				if ( this.type === "checkbox" || this.type === "radio" ) {
+					jQuery.event.add( this, "propertychange._change", function( event ) {
+						if ( event.originalEvent.propertyName === "checked" ) {
+							this._just_changed = true;
+						}
+					});
+					jQuery.event.add( this, "click._change", function( event ) {
+						if ( this._just_changed && !event.isTrigger ) {
+							this._just_changed = false;
+						}
+						// Allow triggered, simulated change events (#11500)
+						jQuery.event.simulate( "change", this, event, true );
+					});
+				}
+				return false;
+			}
+			// Delegated event; lazy-add a change handler on descendant inputs
+			jQuery.event.add( this, "beforeactivate._change", function( e ) {
+				var elem = e.target;
+
+				if ( rformElems.test( elem.nodeName ) && !jQuery._data( elem, "_change_attached" ) ) {
+					jQuery.event.add( elem, "change._change", function( event ) {
+						if ( this.parentNode && !event.isSimulated && !event.isTrigger ) {
+							jQuery.event.simulate( "change", this.parentNode, event, true );
+						}
+					});
+					jQuery._data( elem, "_change_attached", true );
+				}
+			});
+		},
+
+		handle: function( event ) {
+			var elem = event.target;
+
+			// Swallow native change events from checkbox/radio, we already triggered them above
+			if ( this !== elem || event.isSimulated || event.isTrigger || (elem.type !== "radio" && elem.type !== "checkbox") ) {
+				return event.handleObj.handler.apply( this, arguments );
+			}
+		},
+
+		teardown: function() {
+			jQuery.event.remove( this, "._change" );
+
+			return !rformElems.test( this.nodeName );
+		}
+	};
+}
+
+// Create "bubbling" focus and blur events
+if ( !jQuery.support.focusinBubbles ) {
+	jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) {
+
+		// Attach a single capturing handler while someone wants focusin/focusout
+		var attaches = 0,
+			handler = function( event ) {
+				jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true );
+			};
+
+		jQuery.event.special[ fix ] = {
+			setup: function() {
+				if ( attaches++ === 0 ) {
+					document.addEventListener( orig, handler, true );
+				}
+			},
+			teardown: function() {
+				if ( --attaches === 0 ) {
+					document.removeEventListener( orig, handler, true );
+				}
+			}
+		};
+	});
+}
+
+jQuery.fn.extend({
+
+	on: function( types, selector, data, fn, /*INTERNAL*/ one ) {
+		var origFn, type;
+
+		// Types can be a map of types/handlers
+		if ( typeof types === "object" ) {
+			// ( types-Object, selector, data )
+			if ( typeof selector !== "string" ) { // && selector != null
+				// ( types-Object, data )
+				data = data || selector;
+				selector = undefined;
+			}
+			for ( type in types ) {
+				this.on( type, selector, data, types[ type ], one );
+			}
+			return this;
+		}
+
+		if ( data == null && fn == null ) {
+			// ( types, fn )
+			fn = selector;
+			data = selector = undefined;
+		} else if ( fn == null ) {
+			if ( typeof selector === "string" ) {
+				// ( types, selector, fn )
+				fn = data;
+				data = undefined;
+			} else {
+				// ( types, data, fn )
+				fn = data;
+				data = selector;
+				selector = undefined;
+			}
+		}
+		if ( fn === false ) {
+			fn = returnFalse;
+		} else if ( !fn ) {
+			return this;
+		}
+
+		if ( one === 1 ) {
+			origFn = fn;
+			fn = function( event ) {
+				// Can use an empty set, since event contains the info
+				jQuery().off( event );
+				return origFn.apply( this, arguments );
+			};
+			// Use same guid so caller can remove using origFn
+			fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ );
+		}
+		return this.each( function() {
+			jQuery.event.add( this, types, fn, data, selector );
+		});
+	},
+	one: function( types, selector, data, fn ) {
+		return this.on( types, selector, data, fn, 1 );
+	},
+	off: function( types, selector, fn ) {
+		var handleObj, type;
+		if ( types && types.preventDefault && types.handleObj ) {
+			// ( event )  dispatched jQuery.Event
+			handleObj = types.handleObj;
+			jQuery( types.delegateTarget ).off(
+				handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType,
+				handleObj.selector,
+				handleObj.handler
+			);
+			return this;
+		}
+		if ( typeof types === "object" ) {
+			// ( types-object [, selector] )
+			for ( type in types ) {
+				this.off( type, selector, types[ type ] );
+			}
+			return this;
+		}
+		if ( selector === false || typeof selector === "function" ) {
+			// ( types [, fn] )
+			fn = selector;
+			selector = undefined;
+		}
+		if ( fn === false ) {
+			fn = returnFalse;
+		}
+		return this.each(function() {
+			jQuery.event.remove( this, types, fn, selector );
+		});
+	},
+
+	bind: function( types, data, fn ) {
+		return this.on( types, null, data, fn );
+	},
+	unbind: function( types, fn ) {
+		return this.off( types, null, fn );
+	},
+
+	live: function( types, data, fn ) {
+		jQuery( this.context ).on( types, this.selector, data, fn );
+		return this;
+	},
+	die: function( types, fn ) {
+		jQuery( this.context ).off( types, this.selector || "**", fn );
+		return this;
+	},
+
+	delegate: function( selector, types, data, fn ) {
+		return this.on( types, selector, data, fn );
+	},
+	undelegate: function( selector, types, fn ) {
+		// ( namespace ) or ( selector, types [, fn] )
+		return arguments.length === 1 ? this.off( selector, "**" ) : this.off( types, selector || "**", fn );
+	},
+
+	trigger: function( type, data ) {
+		return this.each(function() {
+			jQuery.event.trigger( type, data, this );
+		});
+	},
+	triggerHandler: function( type, data ) {
+		if ( this[0] ) {
+			return jQuery.event.trigger( type, data, this[0], true );
+		}
+	},
+
+	toggle: function( fn ) {
+		// Save reference to arguments for access in closure
+		var args = arguments,
+			guid = fn.guid || jQuery.guid++,
+			i = 0,
+			toggler = function( event ) {
+				// Figure out which function to execute
+				var lastToggle = ( jQuery._data( this, "lastToggle" + fn.guid ) || 0 ) % i;
+				jQuery._data( this, "lastToggle" + fn.guid, lastToggle + 1 );
+
+				// Make sure that clicks stop
+				event.preventDefault();
+
+				// and execute the function
+				return args[ lastToggle ].apply( this, arguments ) || false;
+			};
+
+		// link all the functions, so any of them can unbind this click handler
+		toggler.guid = guid;
+		while ( i < args.length ) {
+			args[ i++ ].guid = guid;
+		}
+
+		return this.click( toggler );
+	},
+
+	hover: function( fnOver, fnOut ) {
+		return this.mouseenter( fnOver ).mouseleave( fnOut || fnOver );
+	}
+});
+
+jQuery.each( ("blur focus focusin focusout load resize scroll unload click dblclick " +
+	"mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave " +
+	"change select submit keydown keypress keyup error contextmenu").split(" "), function( i, name ) {
+
+	// Handle event binding
+	jQuery.fn[ name ] = function( data, fn ) {
+		if ( fn == null ) {
+			fn = data;
+			data = null;
+		}
+
+		return arguments.length > 0 ?
+			this.on( name, null, data, fn ) :
+			this.trigger( name );
+	};
+
+	if ( rkeyEvent.test( name ) ) {
+		jQuery.event.fixHooks[ name ] = jQuery.event.keyHooks;
+	}
+
+	if ( rmouseEvent.test( name ) ) {
+		jQuery.event.fixHooks[ name ] = jQuery.event.mouseHooks;
+	}
+});
+/*!
+ * Sizzle CSS Selector Engine
+ * Copyright 2012 jQuery Foundation and other contributors
+ * Released under the MIT license
+ * http://sizzlejs.com/
+ */
+(function( window, undefined ) {
+
+var cachedruns,
+	assertGetIdNotName,
+	Expr,
+	getText,
+	isXML,
+	contains,
+	compile,
+	sortOrder,
+	hasDuplicate,
+	outermostContext,
+
+	baseHasDuplicate = true,
+	strundefined = "undefined",
+
+	expando = ( "sizcache" + Math.random() ).replace( ".", "" ),
+
+	Token = String,
+	document = window.document,
+	docElem = document.documentElement,
+	dirruns = 0,
+	done = 0,
+	pop = [].pop,
+	push = [].push,
+	slice = [].slice,
+	// Use a stripped-down indexOf if a native one is unavailable
+	indexOf = [].indexOf || function( elem ) {
+		var i = 0,
+			len = this.length;
+		for ( ; i < len; i++ ) {
+			if ( this[i] === elem ) {
+				return i;
+			}
+		}
+		return -1;
+	},
+
+	// Augment a function for special use by Sizzle
+	markFunction = function( fn, value ) {
+		fn[ expando ] = value == null || value;
+		return fn;
+	},
+
+	createCache = function() {
+		var cache = {},
+			keys = [];
+
+		return markFunction(function( key, value ) {
+			// Only keep the most recent entries
+			if ( keys.push( key ) > Expr.cacheLength ) {
+				delete cache[ keys.shift() ];
+			}
+
+			// Retrieve with (key + " ") to avoid collision with native Object.prototype properties (see Issue #157)
+			return (cache[ key + " " ] = value);
+		}, cache );
+	},
+
+	classCache = createCache(),
+	tokenCache = createCache(),
+	compilerCache = createCache(),
+
+	// Regex
+
+	// Whitespace characters http://www.w3.org/TR/css3-selectors/#whitespace
+	whitespace = "[\\x20\\t\\r\\n\\f]",
+	// http://www.w3.org/TR/css3-syntax/#characters
+	characterEncoding = "(?:\\\\.|[-\\w]|[^\\x00-\\xa0])+",
+
+	// Loosely modeled on CSS identifier characters
+	// An unquoted value should be a CSS identifier (http://www.w3.org/TR/css3-selectors/#attribute-selectors)
+	// Proper syntax: http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier
+	identifier = characterEncoding.replace( "w", "w#" ),
+
+	// Acceptable operators http://www.w3.org/TR/selectors/#attribute-selectors
+	operators = "([*^$|!~]?=)",
+	attributes = "\\[" + whitespace + "*(" + characterEncoding + ")" + whitespace +
+		"*(?:" + operators + whitespace + "*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|(" + identifier + ")|)|)" + whitespace + "*\\]",
+
+	// Prefer arguments not in parens/brackets,
+	//   then attribute selectors and non-pseudos (denoted by :),
+	//   then anything else
+	// These preferences are here to reduce the number of selectors
+	//   needing tokenize in the PSEUDO preFilter
+	pseudos = ":(" + characterEncoding + ")(?:\\((?:(['\"])((?:\\\\.|[^\\\\])*?)\\2|([^()[\\]]*|(?:(?:" + attributes + ")|[^:]|\\\\.)*|.*))\\)|)",
+
+	// For matchExpr.POS and matchExpr.needsContext
+	pos = ":(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + whitespace +
+		"*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)",
+
+	// Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter
+	rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ),
+
+	rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ),
+	rcombinators = new RegExp( "^" + whitespace + "*([\\x20\\t\\r\\n\\f>+~])" + whitespace + "*" ),
+	rpseudo = new RegExp( pseudos ),
+
+	// Easily-parseable/retrievable ID or TAG or CLASS selectors
+	rquickExpr = /^(?:#([\w\-]+)|(\w+)|\.([\w\-]+))$/,
+
+	rnot = /^:not/,
+	rsibling = /[\x20\t\r\n\f]*[+~]/,
+	rendsWithNot = /:not\($/,
+
+	rheader = /h\d/i,
+	rinputs = /input|select|textarea|button/i,
+
+	rbackslash = /\\(?!\\)/g,
+
+	matchExpr = {
+		"ID": new RegExp( "^#(" + characterEncoding + ")" ),
+		"CLASS": new RegExp( "^\\.(" + characterEncoding + ")" ),
+		"NAME": new RegExp( "^\\[name=['\"]?(" + characterEncoding + ")['\"]?\\]" ),
+		"TAG": new RegExp( "^(" + characterEncoding.replace( "w", "w*" ) + ")" ),
+		"ATTR": new RegExp( "^" + attributes ),
+		"PSEUDO": new RegExp( "^" + pseudos ),
+		"POS": new RegExp( pos, "i" ),
+		"CHILD": new RegExp( "^:(only|nth|first|last)-child(?:\\(" + whitespace +
+			"*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace +
+			"*(\\d+)|))" + whitespace + "*\\)|)", "i" ),
+		// For use in libraries implementing .is()
+		"needsContext": new RegExp( "^" + whitespace + "*[>+~]|" + pos, "i" )
+	},
+
+	// Support
+
+	// Used for testing something on an element
+	assert = function( fn ) {
+		var div = document.createElement("div");
+
+		try {
+			return fn( div );
+		} catch (e) {
+			return false;
+		} finally {
+			// release memory in IE
+			div = null;
+		}
+	},
+
+	// Check if getElementsByTagName("*") returns only elements
+	assertTagNameNoComments = assert(function( div ) {
+		div.appendChild( document.createComment("") );
+		return !div.getElementsByTagName("*").length;
+	}),
+
+	// Check if getAttribute returns normalized href attributes
+	assertHrefNotNormalized = assert(function( div ) {
+		div.innerHTML = "<a href='#'></a>";
+		return div.firstChild && typeof div.firstChild.getAttribute !== strundefined &&
+			div.firstChild.getAttribute("href") === "#";
+	}),
+
+	// Check if attributes should be retrieved by attribute nodes
+	assertAttributes = assert(function( div ) {
+		div.innerHTML = "<select></select>";
+		var type = typeof div.lastChild.getAttribute("multiple");
+		// IE8 returns a string for some attributes even when not present
+		return type !== "boolean" && type !== "string";
+	}),
+
+	// Check if getElementsByClassName can be trusted
+	assertUsableClassName = assert(function( div ) {
+		// Opera can't find a second classname (in 9.6)
+		div.innerHTML = "<div class='hidden e'></div><div class='hidden'></div>";
+		if ( !div.getElementsByClassName || !div.getElementsByClassName("e").length ) {
+			return false;
+		}
+
+		// Safari 3.2 caches class attributes and doesn't catch changes
+		div.lastChild.className = "e";
+		return div.getElementsByClassName("e").length === 2;
+	}),
+
+	// Check if getElementById returns elements by name
+	// Check if getElementsByName privileges form controls or returns elements by ID
+	assertUsableName = assert(function( div ) {
+		// Inject content
+		div.id = expando + 0;
+		div.innerHTML = "<a name='" + expando + "'></a><div name='" + expando + "'></div>";
+		docElem.insertBefore( div, docElem.firstChild );
+
+		// Test
+		var pass = document.getElementsByName &&
+			// buggy browsers will return fewer than the correct 2
+			document.getElementsByName( expando ).length === 2 +
+			// buggy browsers will return more than the correct 0
+			document.getElementsByName( expando + 0 ).length;
+		assertGetIdNotName = !document.getElementById( expando );
+
+		// Cleanup
+		docElem.removeChild( div );
+
+		return pass;
+	});
+
+// If slice is not available, provide a backup
+try {
+	slice.call( docElem.childNodes, 0 )[0].nodeType;
+} catch ( e ) {
+	slice = function( i ) {
+		var elem,
+			results = [];
+		for ( ; (elem = this[i]); i++ ) {
+			results.push( elem );
+		}
+		return results;
+	};
+}
+
+function Sizzle( selector, context, results, seed ) {
+	results = results || [];
+	context = context || document;
+	var match, elem, xml, m,
+		nodeType = context.nodeType;
+
+	if ( !selector || typeof selector !== "string" ) {
+		return results;
+	}
+
+	if ( nodeType !== 1 && nodeType !== 9 ) {
+		return [];
+	}
+
+	xml = isXML( context );
+
+	if ( !xml && !seed ) {
+		if ( (match = rquickExpr.exec( selector )) ) {
+			// Speed-up: Sizzle("#ID")
+			if ( (m = match[1]) ) {
+				if ( nodeType === 9 ) {
+					elem = context.getElementById( m );
+					// Check parentNode to catch when Blackberry 4.6 returns
+					// nodes that are no longer in the document #6963
+					if ( elem && elem.parentNode ) {
+						// Handle the case where IE, Opera, and Webkit return items
+						// by name instead of ID
+						if ( elem.id === m ) {
+							results.push( elem );
+							return results;
+						}
+					} else {
+						return results;
+					}
+				} else {
+					// Context is not a document
+					if ( context.ownerDocument && (elem = context.ownerDocument.getElementById( m )) &&
+						contains( context, elem ) && elem.id === m ) {
+						results.push( elem );
+						return results;
+					}
+				}
+
+			// Speed-up: Sizzle("TAG")
+			} else if ( match[2] ) {
+				push.apply( results, slice.call(context.getElementsByTagName( selector ), 0) );
+				return results;
+
+			// Speed-up: Sizzle(".CLASS")
+			} else if ( (m = match[3]) && assertUsableClassName && context.getElementsByClassName ) {
+				push.apply( results, slice.call(context.getElementsByClassName( m ), 0) );
+				return results;
+			}
+		}
+	}
+
+	// All others
+	return select( selector.replace( rtrim, "$1" ), context, results, seed, xml );
+}
+
+Sizzle.matches = function( expr, elements ) {
+	return Sizzle( expr, null, null, elements );
+};
+
+Sizzle.matchesSelector = function( elem, expr ) {
+	return Sizzle( expr, null, null, [ elem ] ).length > 0;
+};
+
+// Returns a function to use in pseudos for input types
+function createInputPseudo( type ) {
+	return function( elem ) {
+		var name = elem.nodeName.toLowerCase();
+		return name === "input" && elem.type === type;
+	};
+}
+
+// Returns a function to use in pseudos for buttons
+function createButtonPseudo( type ) {
+	return function( elem ) {
+		var name = elem.nodeName.toLowerCase();
+		return (name === "input" || name === "button") && elem.type === type;
+	};
+}
+
+// Returns a function to use in pseudos for positionals
+function createPositionalPseudo( fn ) {
+	return markFunction(function( argument ) {
+		argument = +argument;
+		return markFunction(function( seed, matches ) {
+			var j,
+				matchIndexes = fn( [], seed.length, argument ),
+				i = matchIndexes.length;
+
+			// Match elements found at the specified indexes
+			while ( i-- ) {
+				if ( seed[ (j = matchIndexes[i]) ] ) {
+					seed[j] = !(matches[j] = seed[j]);
+				}
+			}
+		});
+	});
+}
+
+/**
+ * Utility function for retrieving the text value of an array of DOM nodes
+ * @param {Array|Element} elem
+ */
+getText = Sizzle.getText = function( elem ) {
+	var node,
+		ret = "",
+		i = 0,
+		nodeType = elem.nodeType;
+
+	if ( nodeType ) {
+		if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) {
+			// Use textContent for elements
+			// innerText usage removed for consistency of new lines (see #11153)
+			if ( typeof elem.textContent === "string" ) {
+				return elem.textContent;
+			} else {
+				// Traverse its children
+				for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) {
+					ret += getText( elem );
+				}
+			}
+		} else if ( nodeType === 3 || nodeType === 4 ) {
+			return elem.nodeValue;
+		}
+		// Do not include comment or processing instruction nodes
+	} else {
+
+		// If no nodeType, this is expected to be an array
+		for ( ; (node = elem[i]); i++ ) {
+			// Do not traverse comment nodes
+			ret += getText( node );
+		}
+	}
+	return ret;
+};
+
+isXML = Sizzle.isXML = function( elem ) {
+	// documentElement is verified for cases where it doesn't yet exist
+	// (such as loading iframes in IE - #4833)
+	var documentElement = elem && (elem.ownerDocument || elem).documentElement;
+	return documentElement ? documentElement.nodeName !== "HTML" : false;
+};
+
+// Element contains another
+contains = Sizzle.contains = docElem.contains ?
+	function( a, b ) {
+		var adown = a.nodeType === 9 ? a.documentElement : a,
+			bup = b && b.parentNode;
+		return a === bup || !!( bup && bup.nodeType === 1 && adown.contains && adown.contains(bup) );
+	} :
+	docElem.compareDocumentPosition ?
+	function( a, b ) {
+		return b && !!( a.compareDocumentPosition( b ) & 16 );
+	} :
+	function( a, b ) {
+		while ( (b = b.parentNode) ) {
+			if ( b === a ) {
+				return true;
+			}
+		}
+		return false;
+	};
+
+Sizzle.attr = function( elem, name ) {
+	var val,
+		xml = isXML( elem );
+
+	if ( !xml ) {
+		name = name.toLowerCase();
+	}
+	if ( (val = Expr.attrHandle[ name ]) ) {
+		return val( elem );
+	}
+	if ( xml || assertAttributes ) {
+		return elem.getAttribute( name );
+	}
+	val = elem.getAttributeNode( name );
+	return val ?
+		typeof elem[ name ] === "boolean" ?
+			elem[ name ] ? name : null :
+			val.specified ? val.value : null :
+		null;
+};
+
+Expr = Sizzle.selectors = {
+
+	// Can be adjusted by the user
+	cacheLength: 50,
+
+	createPseudo: markFunction,
+
+	match: matchExpr,
+
+	// IE6/7 return a modified href
+	attrHandle: assertHrefNotNormalized ?
+		{} :
+		{
+			"href": function( elem ) {
+				return elem.getAttribute( "href", 2 );
+			},
+			"type": function( elem ) {
+				return elem.getAttribute("type");
+			}
+		},
+
+	find: {
+		"ID": assertGetIdNotName ?
+			function( id, context, xml ) {
+				if ( typeof context.getElementById !== strundefined && !xml ) {
+					var m = context.getElementById( id );
+					// Check parentNode to catch when Blackberry 4.6 returns
+					// nodes that are no longer in the document #6963
+					return m && m.parentNode ? [m] : [];
+				}
+			} :
+			function( id, context, xml ) {
+				if ( typeof context.getElementById !== strundefined && !xml ) {
+					var m = context.getElementById( id );
+
+					return m ?
+						m.id === id || typeof m.getAttributeNode !== strundefined && m.getAttributeNode("id").value === id ?
+							[m] :
+							undefined :
+						[];
+				}
+			},
+
+		"TAG": assertTagNameNoComments ?
+			function( tag, context ) {
+				if ( typeof context.getElementsByTagName !== strundefined ) {
+					return context.getElementsByTagName( tag );
+				}
+			} :
+			function( tag, context ) {
+				var results = context.getElementsByTagName( tag );
+
+				// Filter out possible comments
+				if ( tag === "*" ) {
+					var elem,
+						tmp = [],
+						i = 0;
+
+					for ( ; (elem = results[i]); i++ ) {
+						if ( elem.nodeType === 1 ) {
+							tmp.push( elem );
+						}
+					}
+
+					return tmp;
+				}
+				return results;
+			},
+
+		"NAME": assertUsableName && function( tag, context ) {
+			if ( typeof context.getElementsByName !== strundefined ) {
+				return context.getElementsByName( name );
+			}
+		},
+
+		"CLASS": assertUsableClassName && function( className, context, xml ) {
+			if ( typeof context.getElementsByClassName !== strundefined && !xml ) {
+				return context.getElementsByClassName( className );
+			}
+		}
+	},
+
+	relative: {
+		">": { dir: "parentNode", first: true },
+		" ": { dir: "parentNode" },
+		"+": { dir: "previousSibling", first: true },
+		"~": { dir: "previousSibling" }
+	},
+
+	preFilter: {
+		"ATTR": function( match ) {
+			match[1] = match[1].replace( rbackslash, "" );
+
+			// Move the given value to match[3] whether quoted or unquoted
+			match[3] = ( match[4] || match[5] || "" ).replace( rbackslash, "" );
+
+			if ( match[2] === "~=" ) {
+				match[3] = " " + match[3] + " ";
+			}
+
+			return match.slice( 0, 4 );
+		},
+
+		"CHILD": function( match ) {
+			/* matches from matchExpr["CHILD"]
+				1 type (only|nth|...)
+				2 argument (even|odd|\d*|\d*n([+-]\d+)?|...)
+				3 xn-component of xn+y argument ([+-]?\d*n|)
+				4 sign of xn-component
+				5 x of xn-component
+				6 sign of y-component
+				7 y of y-component
+			*/
+			match[1] = match[1].toLowerCase();
+
+			if ( match[1] === "nth" ) {
+				// nth-child requires argument
+				if ( !match[2] ) {
+					Sizzle.error( match[0] );
+				}
+
+				// numeric x and y parameters for Expr.filter.CHILD
+				// remember that false/true cast respectively to 0/1
+				match[3] = +( match[3] ? match[4] + (match[5] || 1) : 2 * ( match[2] === "even" || match[2] === "odd" ) );
+				match[4] = +( ( match[6] + match[7] ) || match[2] === "odd" );
+
+			// other types prohibit arguments
+			} else if ( match[2] ) {
+				Sizzle.error( match[0] );
+			}
+
+			return match;
+		},
+
+		"PSEUDO": function( match ) {
+			var unquoted, excess;
+			if ( matchExpr["CHILD"].test( match[0] ) ) {
+				return null;
+			}
+
+			if ( match[3] ) {
+				match[2] = match[3];
+			} else if ( (unquoted = match[4]) ) {
+				// Only check arguments that contain a pseudo
+				if ( rpseudo.test(unquoted) &&
+					// Get excess from tokenize (recursively)
+					(excess = tokenize( unquoted, true )) &&
+					// advance to the next closing parenthesis
+					(excess = unquoted.indexOf( ")", unquoted.length - excess ) - unquoted.length) ) {
+
+					// excess is a negative index
+					unquoted = unquoted.slice( 0, excess );
+					match[0] = match[0].slice( 0, excess );
+				}
+				match[2] = unquoted;
+			}
+
+			// Return only captures needed by the pseudo filter method (type and argument)
+			return match.slice( 0, 3 );
+		}
+	},
+
+	filter: {
+		"ID": assertGetIdNotName ?
+			function( id ) {
+				id = id.replace( rbackslash, "" );
+				return function( elem ) {
+					return elem.getAttribute("id") === id;
+				};
+			} :
+			function( id ) {
+				id = id.replace( rbackslash, "" );
+				return function( elem ) {
+					var node = typeof elem.getAttributeNode !== strundefined && elem.getAttributeNode("id");
+					return node && node.value === id;
+				};
+			},
+
+		"TAG": function( nodeName ) {
+			if ( nodeName === "*" ) {
+				return function() { return true; };
+			}
+			nodeName = nodeName.replace( rbackslash, "" ).toLowerCase();
+
+			return function( elem ) {
+				return elem.nodeName && elem.nodeName.toLowerCase() === nodeName;
+			};
+		},
+
+		"CLASS": function( className ) {
+			var pattern = classCache[ expando ][ className + " " ];
+
+			return pattern ||
+				(pattern = new RegExp( "(^|" + whitespace + ")" + className + "(" + whitespace + "|$)" )) &&
+				classCache( className, function( elem ) {
+					return pattern.test( elem.className || (typeof elem.getAttribute !== strundefined && elem.getAttribute("class")) || "" );
+				});
+		},
+
+		"ATTR": function( name, operator, check ) {
+			return function( elem, context ) {
+				var result = Sizzle.attr( elem, name );
+
+				if ( result == null ) {
+					return operator === "!=";
+				}
+				if ( !operator ) {
+					return true;
+				}
+
+				result += "";
+
+				return operator === "=" ? result === check :
+					operator === "!=" ? result !== check :
+					operator === "^=" ? check && result.indexOf( check ) === 0 :
+					operator === "*=" ? check && result.indexOf( check ) > -1 :
+					operator === "$=" ? check && result.substr( result.length - check.length ) === check :
+					operator === "~=" ? ( " " + result + " " ).indexOf( check ) > -1 :
+					operator === "|=" ? result === check || result.substr( 0, check.length + 1 ) === check + "-" :
+					false;
+			};
+		},
+
+		"CHILD": function( type, argument, first, last ) {
+
+			if ( type === "nth" ) {
+				return function( elem ) {
+					var node, diff,
+						parent = elem.parentNode;
+
+					if ( first === 1 && last === 0 ) {
+						return true;
+					}
+
+					if ( parent ) {
+						diff = 0;
+						for ( node = parent.firstChild; node; node = node.nextSibling ) {
+							if ( node.nodeType === 1 ) {
+								diff++;
+								if ( elem === node ) {
+									break;
+								}
+							}
+						}
+					}
+
+					// Incorporate the offset (or cast to NaN), then check against cycle size
+					diff -= last;
+					return diff === first || ( diff % first === 0 && diff / first >= 0 );
+				};
+			}
+
+			return function( elem ) {
+				var node = elem;
+
+				switch ( type ) {
+					case "only":
+					case "first":
+						while ( (node = node.previousSibling) ) {
+							if ( node.nodeType === 1 ) {
+								return false;
+							}
+						}
+
+						if ( type === "first" ) {
+							return true;
+						}
+
+						node = elem;
+
+						/* falls through */
+					case "last":
+						while ( (node = node.nextSibling) ) {
+							if ( node.nodeType === 1 ) {
+								return false;
+							}
+						}
+
+						return true;
+				}
+			};
+		},
+
+		"PSEUDO": function( pseudo, argument ) {
+			// pseudo-class names are case-insensitive
+			// http://www.w3.org/TR/selectors/#pseudo-classes
+			// Prioritize by case sensitivity in case custom pseudos are added with uppercase letters
+			// Remember that setFilters inherits from pseudos
+			var args,
+				fn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] ||
+					Sizzle.error( "unsupported pseudo: " + pseudo );
+
+			// The user may use createPseudo to indicate that
+			// arguments are needed to create the filter function
+			// just as Sizzle does
+			if ( fn[ expando ] ) {
+				return fn( argument );
+			}
+
+			// But maintain support for old signatures
+			if ( fn.length > 1 ) {
+				args = [ pseudo, pseudo, "", argument ];
+				return Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ?
+					markFunction(function( seed, matches ) {
+						var idx,
+							matched = fn( seed, argument ),
+							i = matched.length;
+						while ( i-- ) {
+							idx = indexOf.call( seed, matched[i] );
+							seed[ idx ] = !( matches[ idx ] = matched[i] );
+						}
+					}) :
+					function( elem ) {
+						return fn( elem, 0, args );
+					};
+			}
+
+			return fn;
+		}
+	},
+
+	pseudos: {
+		"not": markFunction(function( selector ) {
+			// Trim the selector passed to compile
+			// to avoid treating leading and trailing
+			// spaces as combinators
+			var input = [],
+				results = [],
+				matcher = compile( selector.replace( rtrim, "$1" ) );
+
+			return matcher[ expando ] ?
+				markFunction(function( seed, matches, context, xml ) {
+					var elem,
+						unmatched = matcher( seed, null, xml, [] ),
+						i = seed.length;
+
+					// Match elements unmatched by `matcher`
+					while ( i-- ) {
+						if ( (elem = unmatched[i]) ) {
+							seed[i] = !(matches[i] = elem);
+						}
+					}
+				}) :
+				function( elem, context, xml ) {
+					input[0] = elem;
+					matcher( input, null, xml, results );
+					return !results.pop();
+				};
+		}),
+
+		"has": markFunction(function( selector ) {
+			return function( elem ) {
+				return Sizzle( selector, elem ).length > 0;
+			};
+		}),
+
+		"contains": markFunction(function( text ) {
+			return function( elem ) {
+				return ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1;
+			};
+		}),
+
+		"enabled": function( elem ) {
+			return elem.disabled === false;
+		},
+
+		"disabled": function( elem ) {
+			return elem.disabled === true;
+		},
+
+		"checked": function( elem ) {
+			// In CSS3, :checked should return both checked and selected elements
+			// http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked
+			var nodeName = elem.nodeName.toLowerCase();
+			return (nodeName === "input" && !!elem.checked) || (nodeName === "option" && !!elem.selected);
+		},
+
+		"selected": function( elem ) {
+			// Accessing this property makes selected-by-default
+			// options in Safari work properly
+			if ( elem.parentNode ) {
+				elem.parentNode.selectedIndex;
+			}
+
+			return elem.selected === true;
+		},
+
+		"parent": function( elem ) {
+			return !Expr.pseudos["empty"]( elem );
+		},
+
+		"empty": function( elem ) {
+			// http://www.w3.org/TR/selectors/#empty-pseudo
+			// :empty is only affected by element nodes and content nodes(including text(3), cdata(4)),
+			//   not comment, processing instructions, or others
+			// Thanks to Diego Perini for the nodeName shortcut
+			//   Greater than "@" means alpha characters (specifically not starting with "#" or "?")
+			var nodeType;
+			elem = elem.firstChild;
+			while ( elem ) {
+				if ( elem.nodeName > "@" || (nodeType = elem.nodeType) === 3 || nodeType === 4 ) {
+					return false;
+				}
+				elem = elem.nextSibling;
+			}
+			return true;
+		},
+
+		"header": function( elem ) {
+			return rheader.test( elem.nodeName );
+		},
+
+		"text": function( elem ) {
+			var type, attr;
+			// IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc)
+			// use getAttribute instead to test this case
+			return elem.nodeName.toLowerCase() === "input" &&
+				(type = elem.type) === "text" &&
+				( (attr = elem.getAttribute("type")) == null || attr.toLowerCase() === type );
+		},
+
+		// Input types
+		"radio": createInputPseudo("radio"),
+		"checkbox": createInputPseudo("checkbox"),
+		"file": createInputPseudo("file"),
+		"password": createInputPseudo("password"),
+		"image": createInputPseudo("image"),
+
+		"submit": createButtonPseudo("submit"),
+		"reset": createButtonPseudo("reset"),
+
+		"button": function( elem ) {
+			var name = elem.nodeName.toLowerCase();
+			return name === "input" && elem.type === "button" || name === "button";
+		},
+
+		"input": function( elem ) {
+			return rinputs.test( elem.nodeName );
+		},
+
+		"focus": function( elem ) {
+			var doc = elem.ownerDocument;
+			return elem === doc.activeElement && (!doc.hasFocus || doc.hasFocus()) && !!(elem.type || elem.href || ~elem.tabIndex);
+		},
+
+		"active": function( elem ) {
+			return elem === elem.ownerDocument.activeElement;
+		},
+
+		// Positional types
+		"first": createPositionalPseudo(function() {
+			return [ 0 ];
+		}),
+
+		"last": createPositionalPseudo(function( matchIndexes, length ) {
+			return [ length - 1 ];
+		}),
+
+		"eq": createPositionalPseudo(function( matchIndexes, length, argument ) {
+			return [ argument < 0 ? argument + length : argument ];
+		}),
+
+		"even": createPositionalPseudo(function( matchIndexes, length ) {
+			for ( var i = 0; i < length; i += 2 ) {
+				matchIndexes.push( i );
+			}
+			return matchIndexes;
+		}),
+
+		"odd": createPositionalPseudo(function( matchIndexes, length ) {
+			for ( var i = 1; i < length; i += 2 ) {
+				matchIndexes.push( i );
+			}
+			return matchIndexes;
+		}),
+
+		"lt": createPositionalPseudo(function( matchIndexes, length, argument ) {
+			for ( var i = argument < 0 ? argument + length : argument; --i >= 0; ) {
+				matchIndexes.push( i );
+			}
+			return matchIndexes;
+		}),
+
+		"gt": createPositionalPseudo(function( matchIndexes, length, argument ) {
+			for ( var i = argument < 0 ? argument + length : argument; ++i < length; ) {
+				matchIndexes.push( i );
+			}
+			return matchIndexes;
+		})
+	}
+};
+
+function siblingCheck( a, b, ret ) {
+	if ( a === b ) {
+		return ret;
+	}
+
+	var cur = a.nextSibling;
+
+	while ( cur ) {
+		if ( cur === b ) {
+			return -1;
+		}
+
+		cur = cur.nextSibling;
+	}
+
+	return 1;
+}
+
+sortOrder = docElem.compareDocumentPosition ?
+	function( a, b ) {
+		if ( a === b ) {
+			hasDuplicate = true;
+			return 0;
+		}
+
+		return ( !a.compareDocumentPosition || !b.compareDocumentPosition ?
+			a.compareDocumentPosition :
+			a.compareDocumentPosition(b) & 4
+		) ? -1 : 1;
+	} :
+	function( a, b ) {
+		// The nodes are identical, we can exit early
+		if ( a === b ) {
+			hasDuplicate = true;
+			return 0;
+
+		// Fallback to using sourceIndex (in IE) if it's available on both nodes
+		} else if ( a.sourceIndex && b.sourceIndex ) {
+			return a.sourceIndex - b.sourceIndex;
+		}
+
+		var al, bl,
+			ap = [],
+			bp = [],
+			aup = a.parentNode,
+			bup = b.parentNode,
+			cur = aup;
+
+		// If the nodes are siblings (or identical) we can do a quick check
+		if ( aup === bup ) {
+			return siblingCheck( a, b );
+
+		// If no parents were found then the nodes are disconnected
+		} else if ( !aup ) {
+			return -1;
+
+		} else if ( !bup ) {
+			return 1;
+		}
+
+		// Otherwise they're somewhere else in the tree so we need
+		// to build up a full list of the parentNodes for comparison
+		while ( cur ) {
+			ap.unshift( cur );
+			cur = cur.parentNode;
+		}
+
+		cur = bup;
+
+		while ( cur ) {
+			bp.unshift( cur );
+			cur = cur.parentNode;
+		}
+
+		al = ap.length;
+		bl = bp.length;
+
+		// Start walking down the tree looking for a discrepancy
+		for ( var i = 0; i < al && i < bl; i++ ) {
+			if ( ap[i] !== bp[i] ) {
+				return siblingCheck( ap[i], bp[i] );
+			}
+		}
+
+		// We ended someplace up the tree so do a sibling check
+		return i === al ?
+			siblingCheck( a, bp[i], -1 ) :
+			siblingCheck( ap[i], b, 1 );
+	};
+
+// Always assume the presence of duplicates if sort doesn't
+// pass them to our comparison function (as in Google Chrome).
+[0, 0].sort( sortOrder );
+baseHasDuplicate = !hasDuplicate;
+
+// Document sorting and removing duplicates
+Sizzle.uniqueSort = function( results ) {
+	var elem,
+		duplicates = [],
+		i = 1,
+		j = 0;
+
+	hasDuplicate = baseHasDuplicate;
+	results.sort( sortOrder );
+
+	if ( hasDuplicate ) {
+		for ( ; (elem = results[i]); i++ ) {
+			if ( elem === results[ i - 1 ] ) {
+				j = duplicates.push( i );
+			}
+		}
+		while ( j-- ) {
+			results.splice( duplicates[ j ], 1 );
+		}
+	}
+
+	return results;
+};
+
+Sizzle.error = function( msg ) {
+	throw new Error( "Syntax error, unrecognized expression: " + msg );
+};
+
+function tokenize( selector, parseOnly ) {
+	var matched, match, tokens, type,
+		soFar, groups, preFilters,
+		cached = tokenCache[ expando ][ selector + " " ];
+
+	if ( cached ) {
+		return parseOnly ? 0 : cached.slice( 0 );
+	}
+
+	soFar = selector;
+	groups = [];
+	preFilters = Expr.preFilter;
+
+	while ( soFar ) {
+
+		// Comma and first run
+		if ( !matched || (match = rcomma.exec( soFar )) ) {
+			if ( match ) {
+				// Don't consume trailing commas as valid
+				soFar = soFar.slice( match[0].length ) || soFar;
+			}
+			groups.push( tokens = [] );
+		}
+
+		matched = false;
+
+		// Combinators
+		if ( (match = rcombinators.exec( soFar )) ) {
+			tokens.push( matched = new Token( match.shift() ) );
+			soFar = soFar.slice( matched.length );
+
+			// Cast descendant combinators to space
+			matched.type = match[0].replace( rtrim, " " );
+		}
+
+		// Filters
+		for ( type in Expr.filter ) {
+			if ( (match = matchExpr[ type ].exec( soFar )) && (!preFilters[ type ] ||
+				(match = preFilters[ type ]( match ))) ) {
+
+				tokens.push( matched = new Token( match.shift() ) );
+				soFar = soFar.slice( matched.length );
+				matched.type = type;
+				matched.matches = match;
+			}
+		}
+
+		if ( !matched ) {
+			break;
+		}
+	}
+
+	// Return the length of the invalid excess
+	// if we're just parsing
+	// Otherwise, throw an error or return tokens
+	return parseOnly ?
+		soFar.length :
+		soFar ?
+			Sizzle.error( selector ) :
+			// Cache the tokens
+			tokenCache( selector, groups ).slice( 0 );
+}
+
+function addCombinator( matcher, combinator, base ) {
+	var dir = combinator.dir,
+		checkNonElements = base && combinator.dir === "parentNode",
+		doneName = done++;
+
+	return combinator.first ?
+		// Check against closest ancestor/preceding element
+		function( elem, context, xml ) {
+			while ( (elem = elem[ dir ]) ) {
+				if ( checkNonElements || elem.nodeType === 1  ) {
+					return matcher( elem, context, xml );
+				}
+			}
+		} :
+
+		// Check against all ancestor/preceding elements
+		function( elem, context, xml ) {
+			// We can't set arbitrary data on XML nodes, so they don't benefit from dir caching
+			if ( !xml ) {
+				var cache,
+					dirkey = dirruns + " " + doneName + " ",
+					cachedkey = dirkey + cachedruns;
+				while ( (elem = elem[ dir ]) ) {
+					if ( checkNonElements || elem.nodeType === 1 ) {
+						if ( (cache = elem[ expando ]) === cachedkey ) {
+							return elem.sizset;
+						} else if ( typeof cache === "string" && cache.indexOf(dirkey) === 0 ) {
+							if ( elem.sizset ) {
+								return elem;
+							}
+						} else {
+							elem[ expando ] = cachedkey;
+							if ( matcher( elem, context, xml ) ) {
+								elem.sizset = true;
+								return elem;
+							}
+							elem.sizset = false;
+						}
+					}
+				}
+			} else {
+				while ( (elem = elem[ dir ]) ) {
+					if ( checkNonElements || elem.nodeType === 1 ) {
+						if ( matcher( elem, context, xml ) ) {
+							return elem;
+						}
+					}
+				}
+			}
+		};
+}
+
+function elementMatcher( matchers ) {
+	return matchers.length > 1 ?
+		function( elem, context, xml ) {
+			var i = matchers.length;
+			while ( i-- ) {
+				if ( !matchers[i]( elem, context, xml ) ) {
+					return false;
+				}
+			}
+			return true;
+		} :
+		matchers[0];
+}
+
+function condense( unmatched, map, filter, context, xml ) {
+	var elem,
+		newUnmatched = [],
+		i = 0,
+		len = unmatched.length,
+		mapped = map != null;
+
+	for ( ; i < len; i++ ) {
+		if ( (elem = unmatched[i]) ) {
+			if ( !filter || filter( elem, context, xml ) ) {
+				newUnmatched.push( elem );
+				if ( mapped ) {
+					map.push( i );
+				}
+			}
+		}
+	}
+
+	return newUnmatched;
+}
+
+function setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) {
+	if ( postFilter && !postFilter[ expando ] ) {
+		postFilter = setMatcher( postFilter );
+	}
+	if ( postFinder && !postFinder[ expando ] ) {
+		postFinder = setMatcher( postFinder, postSelector );
+	}
+	return markFunction(function( seed, results, context, xml ) {
+		var temp, i, elem,
+			preMap = [],
+			postMap = [],
+			preexisting = results.length,
+
+			// Get initial elements from seed or context
+			elems = seed || multipleContexts( selector || "*", context.nodeType ? [ context ] : context, [] ),
+
+			// Prefilter to get matcher input, preserving a map for seed-results synchronization
+			matcherIn = preFilter && ( seed || !selector ) ?
+				condense( elems, preMap, preFilter, context, xml ) :
+				elems,
+
+			matcherOut = matcher ?
+				// If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results,
+				postFinder || ( seed ? preFilter : preexisting || postFilter ) ?
+
+					// ...intermediate processing is necessary
+					[] :
+
+					// ...otherwise use results directly
+					results :
+				matcherIn;
+
+		// Find primary matches
+		if ( matcher ) {
+			matcher( matcherIn, matcherOut, context, xml );
+		}
+
+		// Apply postFilter
+		if ( postFilter ) {
+			temp = condense( matcherOut, postMap );
+			postFilter( temp, [], context, xml );
+
+			// Un-match failing elements by moving them back to matcherIn
+			i = temp.length;
+			while ( i-- ) {
+				if ( (elem = temp[i]) ) {
+					matcherOut[ postMap[i] ] = !(matcherIn[ postMap[i] ] = elem);
+				}
+			}
+		}
+
+		if ( seed ) {
+			if ( postFinder || preFilter ) {
+				if ( postFinder ) {
+					// Get the final matcherOut by condensing this intermediate into postFinder contexts
+					temp = [];
+					i = matcherOut.length;
+					while ( i-- ) {
+						if ( (elem = matcherOut[i]) ) {
+							// Restore matcherIn since elem is not yet a final match
+							temp.push( (matcherIn[i] = elem) );
+						}
+					}
+					postFinder( null, (matcherOut = []), temp, xml );
+				}
+
+				// Move matched elements from seed to results to keep them synchronized
+				i = matcherOut.length;
+				while ( i-- ) {
+					if ( (elem = matcherOut[i]) &&
+						(temp = postFinder ? indexOf.call( seed, elem ) : preMap[i]) > -1 ) {
+
+						seed[temp] = !(results[temp] = elem);
+					}
+				}
+			}
+
+		// Add elements to results, through postFinder if defined
+		} else {
+			matcherOut = condense(
+				matcherOut === results ?
+					matcherOut.splice( preexisting, matcherOut.length ) :
+					matcherOut
+			);
+			if ( postFinder ) {
+				postFinder( null, results, matcherOut, xml );
+			} else {
+				push.apply( results, matcherOut );
+			}
+		}
+	});
+}
+
+function matcherFromTokens( tokens ) {
+	var checkContext, matcher, j,
+		len = tokens.length,
+		leadingRelative = Expr.relative[ tokens[0].type ],
+		implicitRelative = leadingRelative || Expr.relative[" "],
+		i = leadingRelative ? 1 : 0,
+
+		// The foundational matcher ensures that elements are reachable from top-level context(s)
+		matchContext = addCombinator( function( elem ) {
+			return elem === checkContext;
+		}, implicitRelative, true ),
+		matchAnyContext = addCombinator( function( elem ) {
+			return indexOf.call( checkContext, elem ) > -1;
+		}, implicitRelative, true ),
+		matchers = [ function( elem, context, xml ) {
+			return ( !leadingRelative && ( xml || context !== outermostContext ) ) || (
+				(checkContext = context).nodeType ?
+					matchContext( elem, context, xml ) :
+					matchAnyContext( elem, context, xml ) );
+		} ];
+
+	for ( ; i < len; i++ ) {
+		if ( (matcher = Expr.relative[ tokens[i].type ]) ) {
+			matchers = [ addCombinator( elementMatcher( matchers ), matcher ) ];
+		} else {
+			matcher = Expr.filter[ tokens[i].type ].apply( null, tokens[i].matches );
+
+			// Return special upon seeing a positional matcher
+			if ( matcher[ expando ] ) {
+				// Find the next relative operator (if any) for proper handling
+				j = ++i;
+				for ( ; j < len; j++ ) {
+					if ( Expr.relative[ tokens[j].type ] ) {
+						break;
+					}
+				}
+				return setMatcher(
+					i > 1 && elementMatcher( matchers ),
+					i > 1 && tokens.slice( 0, i - 1 ).join("").replace( rtrim, "$1" ),
+					matcher,
+					i < j && matcherFromTokens( tokens.slice( i, j ) ),
+					j < len && matcherFromTokens( (tokens = tokens.slice( j )) ),
+					j < len && tokens.join("")
+				);
+			}
+			matchers.push( matcher );
+		}
+	}
+
+	return elementMatcher( matchers );
+}
+
+function matcherFromGroupMatchers( elementMatchers, setMatchers ) {
+	var bySet = setMatchers.length > 0,
+		byElement = elementMatchers.length > 0,
+		superMatcher = function( seed, context, xml, results, expandContext ) {
+			var elem, j, matcher,
+				setMatched = [],
+				matchedCount = 0,
+				i = "0",
+				unmatched = seed && [],
+				outermost = expandContext != null,
+				contextBackup = outermostContext,
+				// We must always have either seed elements or context
+				elems = seed || byElement && Expr.find["TAG"]( "*", expandContext && context.parentNode || context ),
+				// Nested matchers should use non-integer dirruns
+				dirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.E);
+
+			if ( outermost ) {
+				outermostContext = context !== document && context;
+				cachedruns = superMatcher.el;
+			}
+
+			// Add elements passing elementMatchers directly to results
+			for ( ; (elem = elems[i]) != null; i++ ) {
+				if ( byElement && elem ) {
+					for ( j = 0; (matcher = elementMatchers[j]); j++ ) {
+						if ( matcher( elem, context, xml ) ) {
+							results.push( elem );
+							break;
+						}
+					}
+					if ( outermost ) {
+						dirruns = dirrunsUnique;
+						cachedruns = ++superMatcher.el;
+					}
+				}
+
+				// Track unmatched elements for set filters
+				if ( bySet ) {
+					// They will have gone through all possible matchers
+					if ( (elem = !matcher && elem) ) {
+						matchedCount--;
+					}
+
+					// Lengthen the array for every element, matched or not
+					if ( seed ) {
+						unmatched.push( elem );
+					}
+				}
+			}
+
+			// Apply set filters to unmatched elements
+			matchedCount += i;
+			if ( bySet && i !== matchedCount ) {
+				for ( j = 0; (matcher = setMatchers[j]); j++ ) {
+					matcher( unmatched, setMatched, context, xml );
+				}
+
+				if ( seed ) {
+					// Reintegrate element matches to eliminate the need for sorting
+					if ( matchedCount > 0 ) {
+						while ( i-- ) {
+							if ( !(unmatched[i] || setMatched[i]) ) {
+								setMatched[i] = pop.call( results );
+							}
+						}
+					}
+
+					// Discard index placeholder values to get only actual matches
+					setMatched = condense( setMatched );
+				}
+
+				// Add matches to results
+				push.apply( results, setMatched );
+
+				// Seedless set matches succeeding multiple successful matchers stipulate sorting
+				if ( outermost && !seed && setMatched.length > 0 &&
+					( matchedCount + setMatchers.length ) > 1 ) {
+
+					Sizzle.uniqueSort( results );
+				}
+			}
+
+			// Override manipulation of globals by nested matchers
+			if ( outermost ) {
+				dirruns = dirrunsUnique;
+				outermostContext = contextBackup;
+			}
+
+			return unmatched;
+		};
+
+	superMatcher.el = 0;
+	return bySet ?
+		markFunction( superMatcher ) :
+		superMatcher;
+}
+
+compile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) {
+	var i,
+		setMatchers = [],
+		elementMatchers = [],
+		cached = compilerCache[ expando ][ selector + " " ];
+
+	if ( !cached ) {
+		// Generate a function of recursive functions that can be used to check each element
+		if ( !group ) {
+			group = tokenize( selector );
+		}
+		i = group.length;
+		while ( i-- ) {
+			cached = matcherFromTokens( group[i] );
+			if ( cached[ expando ] ) {
+				setMatchers.push( cached );
+			} else {
+				elementMatchers.push( cached );
+			}
+		}
+
+		// Cache the compiled function
+		cached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) );
+	}
+	return cached;
+};
+
+function multipleContexts( selector, contexts, results ) {
+	var i = 0,
+		len = contexts.length;
+	for ( ; i < len; i++ ) {
+		Sizzle( selector, contexts[i], results );
+	}
+	return results;
+}
+
+function select( selector, context, results, seed, xml ) {
+	var i, tokens, token, type, find,
+		match = tokenize( selector ),
+		j = match.length;
+
+	if ( !seed ) {
+		// Try to minimize operations if there is only one group
+		if ( match.length === 1 ) {
+
+			// Take a shortcut and set the context if the root selector is an ID
+			tokens = match[0] = match[0].slice( 0 );
+			if ( tokens.length > 2 && (token = tokens[0]).type === "ID" &&
+					context.nodeType === 9 && !xml &&
+					Expr.relative[ tokens[1].type ] ) {
+
+				context = Expr.find["ID"]( token.matches[0].replace( rbackslash, "" ), context, xml )[0];
+				if ( !context ) {
+					return results;
+				}
+
+				selector = selector.slice( tokens.shift().length );
+			}
+
+			// Fetch a seed set for right-to-left matching
+			for ( i = matchExpr["POS"].test( selector ) ? -1 : tokens.length - 1; i >= 0; i-- ) {
+				token = tokens[i];
+
+				// Abort if we hit a combinator
+				if ( Expr.relative[ (type = token.type) ] ) {
+					break;
+				}
+				if ( (find = Expr.find[ type ]) ) {
+					// Search, expanding context for leading sibling combinators
+					if ( (seed = find(
+						token.matches[0].replace( rbackslash, "" ),
+						rsibling.test( tokens[0].type ) && context.parentNode || context,
+						xml
+					)) ) {
+
+						// If seed is empty or no tokens remain, we can return early
+						tokens.splice( i, 1 );
+						selector = seed.length && tokens.join("");
+						if ( !selector ) {
+							push.apply( results, slice.call( seed, 0 ) );
+							return results;
+						}
+
+						break;
+					}
+				}
+			}
+		}
+	}
+
+	// Compile and execute a filtering function
+	// Provide `match` to avoid retokenization if we modified the selector above
+	compile( selector, match )(
+		seed,
+		context,
+		xml,
+		results,
+		rsibling.test( selector )
+	);
+	return results;
+}
+
+if ( document.querySelectorAll ) {
+	(function() {
+		var disconnectedMatch,
+			oldSelect = select,
+			rescape = /'|\\/g,
+			rattributeQuotes = /\=[\x20\t\r\n\f]*([^'"\]]*)[\x20\t\r\n\f]*\]/g,
+
+			// qSa(:focus) reports false when true (Chrome 21), no need to also add to buggyMatches since matches checks buggyQSA
+			// A support test would require too much code (would include document ready)
+			rbuggyQSA = [ ":focus" ],
+
+			// matchesSelector(:active) reports false when true (IE9/Opera 11.5)
+			// A support test would require too much code (would include document ready)
+			// just skip matchesSelector for :active
+			rbuggyMatches = [ ":active" ],
+			matches = docElem.matchesSelector ||
+				docElem.mozMatchesSelector ||
+				docElem.webkitMatchesSelector ||
+				docElem.oMatchesSelector ||
+				docElem.msMatchesSelector;
+
+		// Build QSA regex
+		// Regex strategy adopted from Diego Perini
+		assert(function( div ) {
+			// Select is set to empty string on purpose
+			// This is to test IE's treatment of not explictly
+			// setting a boolean content attribute,
+			// since its presence should be enough
+			// http://bugs.jquery.com/ticket/12359
+			div.innerHTML = "<select><option selected=''></option></select>";
+
+			// IE8 - Some boolean attributes are not treated correctly
+			if ( !div.querySelectorAll("[selected]").length ) {
+				rbuggyQSA.push( "\\[" + whitespace + "*(?:checked|disabled|ismap|multiple|readonly|selected|value)" );
+			}
+
+			// Webkit/Opera - :checked should return selected option elements
+			// http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked
+			// IE8 throws error here (do not put tests after this one)
+			if ( !div.querySelectorAll(":checked").length ) {
+				rbuggyQSA.push(":checked");
+			}
+		});
+
+		assert(function( div ) {
+
+			// Opera 10-12/IE9 - ^= $= *= and empty values
+			// Should not select anything
+			div.innerHTML = "<p test=''></p>";
+			if ( div.querySelectorAll("[test^='']").length ) {
+				rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:\"\"|'')" );
+			}
+
+			// FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled)
+			// IE8 throws error here (do not put tests after this one)
+			div.innerHTML = "<input type='hidden'/>";
+			if ( !div.querySelectorAll(":enabled").length ) {
+				rbuggyQSA.push(":enabled", ":disabled");
+			}
+		});
+
+		// rbuggyQSA always contains :focus, so no need for a length check
+		rbuggyQSA = /* rbuggyQSA.length && */ new RegExp( rbuggyQSA.join("|") );
+
+		select = function( selector, context, results, seed, xml ) {
+			// Only use querySelectorAll when not filtering,
+			// when this is not xml,
+			// and when no QSA bugs apply
+			if ( !seed && !xml && !rbuggyQSA.test( selector ) ) {
+				var groups, i,
+					old = true,
+					nid = expando,
+					newContext = context,
+					newSelector = context.nodeType === 9 && selector;
+
+				// qSA works strangely on Element-rooted queries
+				// We can work around this by specifying an extra ID on the root
+				// and working up from there (Thanks to Andrew Dupont for the technique)
+				// IE 8 doesn't work on object elements
+				if ( context.nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) {
+					groups = tokenize( selector );
+
+					if ( (old = context.getAttribute("id")) ) {
+						nid = old.replace( rescape, "\\$&" );
+					} else {
+						context.setAttribute( "id", nid );
+					}
+					nid = "[id='" + nid + "'] ";
+
+					i = groups.length;
+					while ( i-- ) {
+						groups[i] = nid + groups[i].join("");
+					}
+					newContext = rsibling.test( selector ) && context.parentNode || context;
+					newSelector = groups.join(",");
+				}
+
+				if ( newSelector ) {
+					try {
+						push.apply( results, slice.call( newContext.querySelectorAll(
+							newSelector
+						), 0 ) );
+						return results;
+					} catch(qsaError) {
+					} finally {
+						if ( !old ) {
+							context.removeAttribute("id");
+						}
+					}
+				}
+			}
+
+			return oldSelect( selector, context, results, seed, xml );
+		};
+
+		if ( matches ) {
+			assert(function( div ) {
+				// Check to see if it's possible to do matchesSelector
+				// on a disconnected node (IE 9)
+				disconnectedMatch = matches.call( div, "div" );
+
+				// This should fail with an exception
+				// Gecko does not error, returns false instead
+				try {
+					matches.call( div, "[test!='']:sizzle" );
+					rbuggyMatches.push( "!=", pseudos );
+				} catch ( e ) {}
+			});
+
+			// rbuggyMatches always contains :active and :focus, so no need for a length check
+			rbuggyMatches = /* rbuggyMatches.length && */ new RegExp( rbuggyMatches.join("|") );
+
+			Sizzle.matchesSelector = function( elem, expr ) {
+				// Make sure that attribute selectors are quoted
+				expr = expr.replace( rattributeQuotes, "='$1']" );
+
+				// rbuggyMatches always contains :active, so no need for an existence check
+				if ( !isXML( elem ) && !rbuggyMatches.test( expr ) && !rbuggyQSA.test( expr ) ) {
+					try {
+						var ret = matches.call( elem, expr );
+
+						// IE 9's matchesSelector returns false on disconnected nodes
+						if ( ret || disconnectedMatch ||
+								// As well, disconnected nodes are said to be in a document
+								// fragment in IE 9
+								elem.document && elem.document.nodeType !== 11 ) {
+							return ret;
+						}
+					} catch(e) {}
+				}
+
+				return Sizzle( expr, null, null, [ elem ] ).length > 0;
+			};
+		}
+	})();
+}
+
+// Deprecated
+Expr.pseudos["nth"] = Expr.pseudos["eq"];
+
+// Back-compat
+function setFilters() {}
+Expr.filters = setFilters.prototype = Expr.pseudos;
+Expr.setFilters = new setFilters();
+
+// Override sizzle attribute retrieval
+Sizzle.attr = jQuery.attr;
+jQuery.find = Sizzle;
+jQuery.expr = Sizzle.selectors;
+jQuery.expr[":"] = jQuery.expr.pseudos;
+jQuery.unique = Sizzle.uniqueSort;
+jQuery.text = Sizzle.getText;
+jQuery.isXMLDoc = Sizzle.isXML;
+jQuery.contains = Sizzle.contains;
+
+
+})( window );
+var runtil = /Until$/,
+	rparentsprev = /^(?:parents|prev(?:Until|All))/,
+	isSimple = /^.[^:#\[\.,]*$/,
+	rneedsContext = jQuery.expr.match.needsContext,
+	// methods guaranteed to produce a unique set when starting from a unique set
+	guaranteedUnique = {
+		children: true,
+		contents: true,
+		next: true,
+		prev: true
+	};
+
+jQuery.fn.extend({
+	find: function( selector ) {
+		var i, l, length, n, r, ret,
+			self = this;
+
+		if ( typeof selector !== "string" ) {
+			return jQuery( selector ).filter(function() {
+				for ( i = 0, l = self.length; i < l; i++ ) {
+					if ( jQuery.contains( self[ i ], this ) ) {
+						return true;
+					}
+				}
+			});
+		}
+
+		ret = this.pushStack( "", "find", selector );
+
+		for ( i = 0, l = this.length; i < l; i++ ) {
+			length = ret.length;
+			jQuery.find( selector, this[i], ret );
+
+			if ( i > 0 ) {
+				// Make sure that the results are unique
+				for ( n = length; n < ret.length; n++ ) {
+					for ( r = 0; r < length; r++ ) {
+						if ( ret[r] === ret[n] ) {
+							ret.splice(n--, 1);
+							break;
+						}
+					}
+				}
+			}
+		}
+
+		return ret;
+	},
+
+	has: function( target ) {
+		var i,
+			targets = jQuery( target, this ),
+			len = targets.length;
+
+		return this.filter(function() {
+			for ( i = 0; i < len; i++ ) {
+				if ( jQuery.contains( this, targets[i] ) ) {
+					return true;
+				}
+			}
+		});
+	},
+
+	not: function( selector ) {
+		return this.pushStack( winnow(this, selector, false), "not", selector);
+	},
+
+	filter: function( selector ) {
+		return this.pushStack( winnow(this, selector, true), "filter", selector );
+	},
+
+	is: function( selector ) {
+		return !!selector && (
+			typeof selector === "string" ?
+				// If this is a positional/relative selector, check membership in the returned set
+				// so $("p:first").is("p:last") won't return true for a doc with two "p".
+				rneedsContext.test( selector ) ?
+					jQuery( selector, this.context ).index( this[0] ) >= 0 :
+					jQuery.filter( selector, this ).length > 0 :
+				this.filter( selector ).length > 0 );
+	},
+
+	closest: function( selectors, context ) {
+		var cur,
+			i = 0,
+			l = this.length,
+			ret = [],
+			pos = rneedsContext.test( selectors ) || typeof selectors !== "string" ?
+				jQuery( selectors, context || this.context ) :
+				0;
+
+		for ( ; i < l; i++ ) {
+			cur = this[i];
+
+			while ( cur && cur.ownerDocument && cur !== context && cur.nodeType !== 11 ) {
+				if ( pos ? pos.index(cur) > -1 : jQuery.find.matchesSelector(cur, selectors) ) {
+					ret.push( cur );
+					break;
+				}
+				cur = cur.parentNode;
+			}
+		}
+
+		ret = ret.length > 1 ? jQuery.unique( ret ) : ret;
+
+		return this.pushStack( ret, "closest", selectors );
+	},
+
+	// Determine the position of an element within
+	// the matched set of elements
+	index: function( elem ) {
+
+		// No argument, return index in parent
+		if ( !elem ) {
+			return ( this[0] && this[0].parentNode ) ? this.prevAll().length : -1;
+		}
+
+		// index in selector
+		if ( typeof elem === "string" ) {
+			return jQuery.inArray( this[0], jQuery( elem ) );
+		}
+
+		// Locate the position of the desired element
+		return jQuery.inArray(
+			// If it receives a jQuery object, the first element is used
+			elem.jquery ? elem[0] : elem, this );
+	},
+
+	add: function( selector, context ) {
+		var set = typeof selector === "string" ?
+				jQuery( selector, context ) :
+				jQuery.makeArray( selector && selector.nodeType ? [ selector ] : selector ),
+			all = jQuery.merge( this.get(), set );
+
+		return this.pushStack( isDisconnected( set[0] ) || isDisconnected( all[0] ) ?
+			all :
+			jQuery.unique( all ) );
+	},
+
+	addBack: function( selector ) {
+		return this.add( selector == null ?
+			this.prevObject : this.prevObject.filter(selector)
+		);
+	}
+});
+
+jQuery.fn.andSelf = jQuery.fn.addBack;
+
+// A painfully simple check to see if an element is disconnected
+// from a document (should be improved, where feasible).
+function isDisconnected( node ) {
+	return !node || !node.parentNode || node.parentNode.nodeType === 11;
+}
+
+function sibling( cur, dir ) {
+	do {
+		cur = cur[ dir ];
+	} while ( cur && cur.nodeType !== 1 );
+
+	return cur;
+}
+
+jQuery.each({
+	parent: function( elem ) {
+		var parent = elem.parentNode;
+		return parent && parent.nodeType !== 11 ? parent : null;
+	},
+	parents: function( elem ) {
+		return jQuery.dir( elem, "parentNode" );
+	},
+	parentsUntil: function( elem, i, until ) {
+		return jQuery.dir( elem, "parentNode", until );
+	},
+	next: function( elem ) {
+		return sibling( elem, "nextSibling" );
+	},
+	prev: function( elem ) {
+		return sibling( elem, "previousSibling" );
+	},
+	nextAll: function( elem ) {
+		return jQuery.dir( elem, "nextSibling" );
+	},
+	prevAll: function( elem ) {
+		return jQuery.dir( elem, "previousSibling" );
+	},
+	nextUntil: function( elem, i, until ) {
+		return jQuery.dir( elem, "nextSibling", until );
+	},
+	prevUntil: function( elem, i, until ) {
+		return jQuery.dir( elem, "previousSibling", until );
+	},
+	siblings: function( elem ) {
+		return jQuery.sibling( ( elem.parentNode || {} ).firstChild, elem );
+	},
+	children: function( elem ) {
+		return jQuery.sibling( elem.firstChild );
+	},
+	contents: function( elem ) {
+		return jQuery.nodeName( elem, "iframe" ) ?
+			elem.contentDocument || elem.contentWindow.document :
+			jQuery.merge( [], elem.childNodes );
+	}
+}, function( name, fn ) {
+	jQuery.fn[ name ] = function( until, selector ) {
+		var ret = jQuery.map( this, fn, until );
+
+		if ( !runtil.test( name ) ) {
+			selector = until;
+		}
+
+		if ( selector && typeof selector === "string" ) {
+			ret = jQuery.filter( selector, ret );
+		}
+
+		ret = this.length > 1 && !guaranteedUnique[ name ] ? jQuery.unique( ret ) : ret;
+
+		if ( this.length > 1 && rparentsprev.test( name ) ) {
+			ret = ret.reverse();
+		}
+
+		return this.pushStack( ret, name, core_slice.call( arguments ).join(",") );
+	};
+});
+
+jQuery.extend({
+	filter: function( expr, elems, not ) {
+		if ( not ) {
+			expr = ":not(" + expr + ")";
+		}
+
+		return elems.length === 1 ?
+			jQuery.find.matchesSelector(elems[0], expr) ? [ elems[0] ] : [] :
+			jQuery.find.matches(expr, elems);
+	},
+
+	dir: function( elem, dir, until ) {
+		var matched = [],
+			cur = elem[ dir ];
+
+		while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) {
+			if ( cur.nodeType === 1 ) {
+				matched.push( cur );
+			}
+			cur = cur[dir];
+		}
+		return matched;
+	},
+
+	sibling: function( n, elem ) {
+		var r = [];
+
+		for ( ; n; n = n.nextSibling ) {
+			if ( n.nodeType === 1 && n !== elem ) {
+				r.push( n );
+			}
+		}
+
+		return r;
+	}
+});
+
+// Implement the identical functionality for filter and not
+function winnow( elements, qualifier, keep ) {
+
+	// Can't pass null or undefined to indexOf in Firefox 4
+	// Set to 0 to skip string check
+	qualifier = qualifier || 0;
+
+	if ( jQuery.isFunction( qualifier ) ) {
+		return jQuery.grep(elements, function( elem, i ) {
+			var retVal = !!qualifier.call( elem, i, elem );
+			return retVal === keep;
+		});
+
+	} else if ( qualifier.nodeType ) {
+		return jQuery.grep(elements, function( elem, i ) {
+			return ( elem === qualifier ) === keep;
+		});
+
+	} else if ( typeof qualifier === "string" ) {
+		var filtered = jQuery.grep(elements, function( elem ) {
+			return elem.nodeType === 1;
+		});
+
+		if ( isSimple.test( qualifier ) ) {
+			return jQuery.filter(qualifier, filtered, !keep);
+		} else {
+			qualifier = jQuery.filter( qualifier, filtered );
+		}
+	}
+
+	return jQuery.grep(elements, function( elem, i ) {
+		return ( jQuery.inArray( elem, qualifier ) >= 0 ) === keep;
+	});
+}
+function createSafeFragment( document ) {
+	var list = nodeNames.split( "|" ),
+	safeFrag = document.createDocumentFragment();
+
+	if ( safeFrag.createElement ) {
+		while ( list.length ) {
+			safeFrag.createElement(
+				list.pop()
+			);
+		}
+	}
+	return safeFrag;
+}
+
+var nodeNames = "abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|" +
+		"header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",
+	rinlinejQuery = / jQuery\d+="(?:null|\d+)"/g,
+	rleadingWhitespace = /^\s+/,
+	rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,
+	rtagName = /<([\w:]+)/,
+	rtbody = /<tbody/i,
+	rhtml = /<|&#?\w+;/,
+	rnoInnerhtml = /<(?:script|style|link)/i,
+	rnocache = /<(?:script|object|embed|option|style)/i,
+	rnoshimcache = new RegExp("<(?:" + nodeNames + ")[\\s/>]", "i"),
+	rcheckableType = /^(?:checkbox|radio)$/,
+	// checked="checked" or checked
+	rchecked = /checked\s*(?:[^=]|=\s*.checked.)/i,
+	rscriptType = /\/(java|ecma)script/i,
+	rcleanScript = /^\s*<!(?:\[CDATA\[|\-\-)|[\]\-]{2}>\s*$/g,
+	wrapMap = {
+		option: [ 1, "<select multiple='multiple'>", "</select>" ],
+		legend: [ 1, "<fieldset>", "</fieldset>" ],
+		thead: [ 1, "<table>", "</table>" ],
+		tr: [ 2, "<table><tbody>", "</tbody></table>" ],
+		td: [ 3, "<table><tbody><tr>", "</tr></tbody></table>" ],
+		col: [ 2, "<table><tbody></tbody><colgroup>", "</colgroup></table>" ],
+		area: [ 1, "<map>", "</map>" ],
+		_default: [ 0, "", "" ]
+	},
+	safeFragment = createSafeFragment( document ),
+	fragmentDiv = safeFragment.appendChild( document.createElement("div") );
+
+wrapMap.optgroup = wrapMap.option;
+wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead;
+wrapMap.th = wrapMap.td;
+
+// IE6-8 can't serialize link, script, style, or any html5 (NoScope) tags,
+// unless wrapped in a div with non-breaking characters in front of it.
+if ( !jQuery.support.htmlSerialize ) {
+	wrapMap._default = [ 1, "X<div>", "</div>" ];
+}
+
+jQuery.fn.extend({
+	text: function( value ) {
+		return jQuery.access( this, function( value ) {
+			return value === undefined ?
+				jQuery.text( this ) :
+				this.empty().append( ( this[0] && this[0].ownerDocument || document ).createTextNode( value ) );
+		}, null, value, arguments.length );
+	},
+
+	wrapAll: function( html ) {
+		if ( jQuery.isFunction( html ) ) {
+			return this.each(function(i) {
+				jQuery(this).wrapAll( html.call(this, i) );
+			});
+		}
+
+		if ( this[0] ) {
+			// The elements to wrap the target around
+			var wrap = jQuery( html, this[0].ownerDocument ).eq(0).clone(true);
+
+			if ( this[0].parentNode ) {
+				wrap.insertBefore( this[0] );
+			}
+
+			wrap.map(function() {
+				var elem = this;
+
+				while ( elem.firstChild && elem.firstChild.nodeType === 1 ) {
+					elem = elem.firstChild;
+				}
+
+				return elem;
+			}).append( this );
+		}
+
+		return this;
+	},
+
+	wrapInner: function( html ) {
+		if ( jQuery.isFunction( html ) ) {
+			return this.each(function(i) {
+				jQuery(this).wrapInner( html.call(this, i) );
+			});
+		}
+
+		return this.each(function() {
+			var self = jQuery( this ),
+				contents = self.contents();
+
+			if ( contents.length ) {
+				contents.wrapAll( html );
+
+			} else {
+				self.append( html );
+			}
+		});
+	},
+
+	wrap: function( html ) {
+		var isFunction = jQuery.isFunction( html );
+
+		return this.each(function(i) {
+			jQuery( this ).wrapAll( isFunction ? html.call(this, i) : html );
+		});
+	},
+
+	unwrap: function() {
+		return this.parent().each(function() {
+			if ( !jQuery.nodeName( this, "body" ) ) {
+				jQuery( this ).replaceWith( this.childNodes );
+			}
+		}).end();
+	},
+
+	append: function() {
+		return this.domManip(arguments, true, function( elem ) {
+			if ( this.nodeType === 1 || this.nodeType === 11 ) {
+				this.appendChild( elem );
+			}
+		});
+	},
+
+	prepend: function() {
+		return this.domManip(arguments, true, function( elem ) {
+			if ( this.nodeType === 1 || this.nodeType === 11 ) {
+				this.insertBefore( elem, this.firstChild );
+			}
+		});
+	},
+
+	before: function() {
+		if ( !isDisconnected( this[0] ) ) {
+			return this.domManip(arguments, false, function( elem ) {
+				this.parentNode.insertBefore( elem, this );
+			});
+		}
+
+		if ( arguments.length ) {
+			var set = jQuery.clean( arguments );
+			return this.pushStack( jQuery.merge( set, this ), "before", this.selector );
+		}
+	},
+
+	after: function() {
+		if ( !isDisconnected( this[0] ) ) {
+			return this.domManip(arguments, false, function( elem ) {
+				this.parentNode.insertBefore( elem, this.nextSibling );
+			});
+		}
+
+		if ( arguments.length ) {
+			var set = jQuery.clean( arguments );
+			return this.pushStack( jQuery.merge( this, set ), "after", this.selector );
+		}
+	},
+
+	// keepData is for internal use only--do not document
+	remove: function( selector, keepData ) {
+		var elem,
+			i = 0;
+
+		for ( ; (elem = this[i]) != null; i++ ) {
+			if ( !selector || jQuery.filter( selector, [ elem ] ).length ) {
+				if ( !keepData && elem.nodeType === 1 ) {
+					jQuery.cleanData( elem.getElementsByTagName("*") );
+					jQuery.cleanData( [ elem ] );
+				}
+
+				if ( elem.parentNode ) {
+					elem.parentNode.removeChild( elem );
+				}
+			}
+		}
+
+		return this;
+	},
+
+	empty: function() {
+		var elem,
+			i = 0;
+
+		for ( ; (elem = this[i]) != null; i++ ) {
+			// Remove element nodes and prevent memory leaks
+			if ( elem.nodeType === 1 ) {
+				jQuery.cleanData( elem.getElementsByTagName("*") );
+			}
+
+			// Remove any remaining nodes
+			while ( elem.firstChild ) {
+				elem.removeChild( elem.firstChild );
+			}
+		}
+
+		return this;
+	},
+
+	clone: function( dataAndEvents, deepDataAndEvents ) {
+		dataAndEvents = dataAndEvents == null ? false : dataAndEvents;
+		deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents;
+
+		return this.map( function () {
+			return jQuery.clone( this, dataAndEvents, deepDataAndEvents );
+		});
+	},
+
+	html: function( value ) {
+		return jQuery.access( this, function( value ) {
+			var elem = this[0] || {},
+				i = 0,
+				l = this.length;
+
+			if ( value === undefined ) {
+				return elem.nodeType === 1 ?
+					elem.innerHTML.replace( rinlinejQuery, "" ) :
+					undefined;
+			}
+
+			// See if we can take a shortcut and just use innerHTML
+			if ( typeof value === "string" && !rnoInnerhtml.test( value ) &&
+				( jQuery.support.htmlSerialize || !rnoshimcache.test( value )  ) &&
+				( jQuery.support.leadingWhitespace || !rleadingWhitespace.test( value ) ) &&
+				!wrapMap[ ( rtagName.exec( value ) || ["", ""] )[1].toLowerCase() ] ) {
+
+				value = value.replace( rxhtmlTag, "<$1></$2>" );
+
+				try {
+					for (; i < l; i++ ) {
+						// Remove element nodes and prevent memory leaks
+						elem = this[i] || {};
+						if ( elem.nodeType === 1 ) {
+							jQuery.cleanData( elem.getElementsByTagName( "*" ) );
+							elem.innerHTML = value;
+						}
+					}
+
+					elem = 0;
+
+				// If using innerHTML throws an exception, use the fallback method
+				} catch(e) {}
+			}
+
+			if ( elem ) {
+				this.empty().append( value );
+			}
+		}, null, value, arguments.length );
+	},
+
+	replaceWith: function( value ) {
+		if ( !isDisconnected( this[0] ) ) {
+			// Make sure that the elements are removed from the DOM before they are inserted
+			// this can help fix replacing a parent with child elements
+			if ( jQuery.isFunction( value ) ) {
+				return this.each(function(i) {
+					var self = jQuery(this), old = self.html();
+					self.replaceWith( value.call( this, i, old ) );
+				});
+			}
+
+			if ( typeof value !== "string" ) {
+				value = jQuery( value ).detach();
+			}
+
+			return this.each(function() {
+				var next = this.nextSibling,
+					parent = this.parentNode;
+
+				jQuery( this ).remove();
+
+				if ( next ) {
+					jQuery(next).before( value );
+				} else {
+					jQuery(parent).append( value );
+				}
+			});
+		}
+
+		return this.length ?
+			this.pushStack( jQuery(jQuery.isFunction(value) ? value() : value), "replaceWith", value ) :
+			this;
+	},
+
+	detach: function( selector ) {
+		return this.remove( selector, true );
+	},
+
+	domManip: function( args, table, callback ) {
+
+		// Flatten any nested arrays
+		args = [].concat.apply( [], args );
+
+		var results, first, fragment, iNoClone,
+			i = 0,
+			value = args[0],
+			scripts = [],
+			l = this.length;
+
+		// We can't cloneNode fragments that contain checked, in WebKit
+		if ( !jQuery.support.checkClone && l > 1 && typeof value === "string" && rchecked.test( value ) ) {
+			return this.each(function() {
+				jQuery(this).domManip( args, table, callback );
+			});
+		}
+
+		if ( jQuery.isFunction(value) ) {
+			return this.each(function(i) {
+				var self = jQuery(this);
+				args[0] = value.call( this, i, table ? self.html() : undefined );
+				self.domManip( args, table, callback );
+			});
+		}
+
+		if ( this[0] ) {
+			results = jQuery.buildFragment( args, this, scripts );
+			fragment = results.fragment;
+			first = fragment.firstChild;
+
+			if ( fragment.childNodes.length === 1 ) {
+				fragment = first;
+			}
+
+			if ( first ) {
+				table = table && jQuery.nodeName( first, "tr" );
+
+				// Use the original fragment for the last item instead of the first because it can end up
+				// being emptied incorrectly in certain situations (#8070).
+				// Fragments from the fragment cache must always be cloned and never used in place.
+				for ( iNoClone = results.cacheable || l - 1; i < l; i++ ) {
+					callback.call(
+						table && jQuery.nodeName( this[i], "table" ) ?
+							findOrAppend( this[i], "tbody" ) :
+							this[i],
+						i === iNoClone ?
+							fragment :
+							jQuery.clone( fragment, true, true )
+					);
+				}
+			}
+
+			// Fix #11809: Avoid leaking memory
+			fragment = first = null;
+
+			if ( scripts.length ) {
+				jQuery.each( scripts, function( i, elem ) {
+					if ( elem.src ) {
+						if ( jQuery.ajax ) {
+							jQuery.ajax({
+								url: elem.src,
+								type: "GET",
+								dataType: "script",
+								async: false,
+								global: false,
+								"throws": true
+							});
+						} else {
+							jQuery.error("no ajax");
+						}
+					} else {
+						jQuery.globalEval( ( elem.text || elem.textContent || elem.innerHTML || "" ).replace( rcleanScript, "" ) );
+					}
+
+					if ( elem.parentNode ) {
+						elem.parentNode.removeChild( elem );
+					}
+				});
+			}
+		}
+
+		return this;
+	}
+});
+
+function findOrAppend( elem, tag ) {
+	return elem.getElementsByTagName( tag )[0] || elem.appendChild( elem.ownerDocument.createElement( tag ) );
+}
+
+function cloneCopyEvent( src, dest ) {
+
+	if ( dest.nodeType !== 1 || !jQuery.hasData( src ) ) {
+		return;
+	}
+
+	var type, i, l,
+		oldData = jQuery._data( src ),
+		curData = jQuery._data( dest, oldData ),
+		events = oldData.events;
+
+	if ( events ) {
+		delete curData.handle;
+		curData.events = {};
+
+		for ( type in events ) {
+			for ( i = 0, l = events[ type ].length; i < l; i++ ) {
+				jQuery.event.add( dest, type, events[ type ][ i ] );
+			}
+		}
+	}
+
+	// make the cloned public data object a copy from the original
+	if ( curData.data ) {
+		curData.data = jQuery.extend( {}, curData.data );
+	}
+}
+
+function cloneFixAttributes( src, dest ) {
+	var nodeName;
+
+	// We do not need to do anything for non-Elements
+	if ( dest.nodeType !== 1 ) {
+		return;
+	}
+
+	// clearAttributes removes the attributes, which we don't want,
+	// but also removes the attachEvent events, which we *do* want
+	if ( dest.clearAttributes ) {
+		dest.clearAttributes();
+	}
+
+	// mergeAttributes, in contrast, only merges back on the
+	// original attributes, not the events
+	if ( dest.mergeAttributes ) {
+		dest.mergeAttributes( src );
+	}
+
+	nodeName = dest.nodeName.toLowerCase();
+
+	if ( nodeName === "object" ) {
+		// IE6-10 improperly clones children of object elements using classid.
+		// IE10 throws NoModificationAllowedError if parent is null, #12132.
+		if ( dest.parentNode ) {
+			dest.outerHTML = src.outerHTML;
+		}
+
+		// This path appears unavoidable for IE9. When cloning an object
+		// element in IE9, the outerHTML strategy above is not sufficient.
+		// If the src has innerHTML and the destination does not,
+		// copy the src.innerHTML into the dest.innerHTML. #10324
+		if ( jQuery.support.html5Clone && (src.innerHTML && !jQuery.trim(dest.innerHTML)) ) {
+			dest.innerHTML = src.innerHTML;
+		}
+
+	} else if ( nodeName === "input" && rcheckableType.test( src.type ) ) {
+		// IE6-8 fails to persist the checked state of a cloned checkbox
+		// or radio button. Worse, IE6-7 fail to give the cloned element
+		// a checked appearance if the defaultChecked value isn't also set
+
+		dest.defaultChecked = dest.checked = src.checked;
+
+		// IE6-7 get confused and end up setting the value of a cloned
+		// checkbox/radio button to an empty string instead of "on"
+		if ( dest.value !== src.value ) {
+			dest.value = src.value;
+		}
+
+	// IE6-8 fails to return the selected option to the default selected
+	// state when cloning options
+	} else if ( nodeName === "option" ) {
+		dest.selected = src.defaultSelected;
+
+	// IE6-8 fails to set the defaultValue to the correct value when
+	// cloning other types of input fields
+	} else if ( nodeName === "input" || nodeName === "textarea" ) {
+		dest.defaultValue = src.defaultValue;
+
+	// IE blanks contents when cloning scripts
+	} else if ( nodeName === "script" && dest.text !== src.text ) {
+		dest.text = src.text;
+	}
+
+	// Event data gets referenced instead of copied if the expando
+	// gets copied too
+	dest.removeAttribute( jQuery.expando );
+}
+
+jQuery.buildFragment = function( args, context, scripts ) {
+	var fragment, cacheable, cachehit,
+		first = args[ 0 ];
+
+	// Set context from what may come in as undefined or a jQuery collection or a node
+	// Updated to fix #12266 where accessing context[0] could throw an exception in IE9/10 &
+	// also doubles as fix for #8950 where plain objects caused createDocumentFragment exception
+	context = context || document;
+	context = !context.nodeType && context[0] || context;
+	context = context.ownerDocument || context;
+
+	// Only cache "small" (1/2 KB) HTML strings that are associated with the main document
+	// Cloning options loses the selected state, so don't cache them
+	// IE 6 doesn't like it when you put <object> or <embed> elements in a fragment
+	// Also, WebKit does not clone 'checked' attributes on cloneNode, so don't cache
+	// Lastly, IE6,7,8 will not correctly reuse cached fragments that were created from unknown elems #10501
+	if ( args.length === 1 && typeof first === "string" && first.length < 512 && context === document &&
+		first.charAt(0) === "<" && !rnocache.test( first ) &&
+		(jQuery.support.checkClone || !rchecked.test( first )) &&
+		(jQuery.support.html5Clone || !rnoshimcache.test( first )) ) {
+
+		// Mark cacheable and look for a hit
+		cacheable = true;
+		fragment = jQuery.fragments[ first ];
+		cachehit = fragment !== undefined;
+	}
+
+	if ( !fragment ) {
+		fragment = context.createDocumentFragment();
+		jQuery.clean( args, context, fragment, scripts );
+
+		// Update the cache, but only store false
+		// unless this is a second parsing of the same content
+		if ( cacheable ) {
+			jQuery.fragments[ first ] = cachehit && fragment;
+		}
+	}
+
+	return { fragment: fragment, cacheable: cacheable };
+};
+
+jQuery.fragments = {};
+
+jQuery.each({
+	appendTo: "append",
+	prependTo: "prepend",
+	insertBefore: "before",
+	insertAfter: "after",
+	replaceAll: "replaceWith"
+}, function( name, original ) {
+	jQuery.fn[ name ] = function( selector ) {
+		var elems,
+			i = 0,
+			ret = [],
+			insert = jQuery( selector ),
+			l = insert.length,
+			parent = this.length === 1 && this[0].parentNode;
+
+		if ( (parent == null || parent && parent.nodeType === 11 && parent.childNodes.length === 1) && l === 1 ) {
+			insert[ original ]( this[0] );
+			return this;
+		} else {
+			for ( ; i < l; i++ ) {
+				elems = ( i > 0 ? this.clone(true) : this ).get();
+				jQuery( insert[i] )[ original ]( elems );
+				ret = ret.concat( elems );
+			}
+
+			return this.pushStack( ret, name, insert.selector );
+		}
+	};
+});
+
+function getAll( elem ) {
+	if ( typeof elem.getElementsByTagName !== "undefined" ) {
+		return elem.getElementsByTagName( "*" );
+
+	} else if ( typeof elem.querySelectorAll !== "undefined" ) {
+		return elem.querySelectorAll( "*" );
+
+	} else {
+		return [];
+	}
+}
+
+// Used in clean, fixes the defaultChecked property
+function fixDefaultChecked( elem ) {
+	if ( rcheckableType.test( elem.type ) ) {
+		elem.defaultChecked = elem.checked;
+	}
+}
+
+jQuery.extend({
+	clone: function( elem, dataAndEvents, deepDataAndEvents ) {
+		var srcElements,
+			destElements,
+			i,
+			clone;
+
+		if ( jQuery.support.html5Clone || jQuery.isXMLDoc(elem) || !rnoshimcache.test( "<" + elem.nodeName + ">" ) ) {
+			clone = elem.cloneNode( true );
+
+		// IE<=8 does not properly clone detached, unknown element nodes
+		} else {
+			fragmentDiv.innerHTML = elem.outerHTML;
+			fragmentDiv.removeChild( clone = fragmentDiv.firstChild );
+		}
+
+		if ( (!jQuery.support.noCloneEvent || !jQuery.support.noCloneChecked) &&
+				(elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) {
+			// IE copies events bound via attachEvent when using cloneNode.
+			// Calling detachEvent on the clone will also remove the events
+			// from the original. In order to get around this, we use some
+			// proprietary methods to clear the events. Thanks to MooTools
+			// guys for this hotness.
+
+			cloneFixAttributes( elem, clone );
+
+			// Using Sizzle here is crazy slow, so we use getElementsByTagName instead
+			srcElements = getAll( elem );
+			destElements = getAll( clone );
+
+			// Weird iteration because IE will replace the length property
+			// with an element if you are cloning the body and one of the
+			// elements on the page has a name or id of "length"
+			for ( i = 0; srcElements[i]; ++i ) {
+				// Ensure that the destination node is not null; Fixes #9587
+				if ( destElements[i] ) {
+					cloneFixAttributes( srcElements[i], destElements[i] );
+				}
+			}
+		}
+
+		// Copy the events from the original to the clone
+		if ( dataAndEvents ) {
+			cloneCopyEvent( elem, clone );
+
+			if ( deepDataAndEvents ) {
+				srcElements = getAll( elem );
+				destElements = getAll( clone );
+
+				for ( i = 0; srcElements[i]; ++i ) {
+					cloneCopyEvent( srcElements[i], destElements[i] );
+				}
+			}
+		}
+
+		srcElements = destElements = null;
+
+		// Return the cloned set
+		return clone;
+	},
+
+	clean: function( elems, context, fragment, scripts ) {
+		var i, j, elem, tag, wrap, depth, div, hasBody, tbody, len, handleScript, jsTags,
+			safe = context === document && safeFragment,
+			ret = [];
+
+		// Ensure that context is a document
+		if ( !context || typeof context.createDocumentFragment === "undefined" ) {
+			context = document;
+		}
+
+		// Use the already-created safe fragment if context permits
+		for ( i = 0; (elem = elems[i]) != null; i++ ) {
+			if ( typeof elem === "number" ) {
+				elem += "";
+			}
+
+			if ( !elem ) {
+				continue;
+			}
+
+			// Convert html string into DOM nodes
+			if ( typeof elem === "string" ) {
+				if ( !rhtml.test( elem ) ) {
+					elem = context.createTextNode( elem );
+				} else {
+					// Ensure a safe container in which to render the html
+					safe = safe || createSafeFragment( context );
+					div = context.createElement("div");
+					safe.appendChild( div );
+
+					// Fix "XHTML"-style tags in all browsers
+					elem = elem.replace(rxhtmlTag, "<$1></$2>");
+
+					// Go to html and back, then peel off extra wrappers
+					tag = ( rtagName.exec( elem ) || ["", ""] )[1].toLowerCase();
+					wrap = wrapMap[ tag ] || wrapMap._default;
+					depth = wrap[0];
+					div.innerHTML = wrap[1] + elem + wrap[2];
+
+					// Move to the right depth
+					while ( depth-- ) {
+						div = div.lastChild;
+					}
+
+					// Remove IE's autoinserted <tbody> from table fragments
+					if ( !jQuery.support.tbody ) {
+
+						// String was a <table>, *may* have spurious <tbody>
+						hasBody = rtbody.test(elem);
+							tbody = tag === "table" && !hasBody ?
+								div.firstChild && div.firstChild.childNodes :
+
+								// String was a bare <thead> or <tfoot>
+								wrap[1] === "<table>" && !hasBody ?
+									div.childNodes :
+									[];
+
+						for ( j = tbody.length - 1; j >= 0 ; --j ) {
+							if ( jQuery.nodeName( tbody[ j ], "tbody" ) && !tbody[ j ].childNodes.length ) {
+								tbody[ j ].parentNode.removeChild( tbody[ j ] );
+							}
+						}
+					}
+
+					// IE completely kills leading whitespace when innerHTML is used
+					if ( !jQuery.support.leadingWhitespace && rleadingWhitespace.test( elem ) ) {
+						div.insertBefore( context.createTextNode( rleadingWhitespace.exec(elem)[0] ), div.firstChild );
+					}
+
+					elem = div.childNodes;
+
+					// Take out of fragment container (we need a fresh div each time)
+					div.parentNode.removeChild( div );
+				}
+			}
+
+			if ( elem.nodeType ) {
+				ret.push( elem );
+			} else {
+				jQuery.merge( ret, elem );
+			}
+		}
+
+		// Fix #11356: Clear elements from safeFragment
+		if ( div ) {
+			elem = div = safe = null;
+		}
+
+		// Reset defaultChecked for any radios and checkboxes
+		// about to be appended to the DOM in IE 6/7 (#8060)
+		if ( !jQuery.support.appendChecked ) {
+			for ( i = 0; (elem = ret[i]) != null; i++ ) {
+				if ( jQuery.nodeName( elem, "input" ) ) {
+					fixDefaultChecked( elem );
+				} else if ( typeof elem.getElementsByTagName !== "undefined" ) {
+					jQuery.grep( elem.getElementsByTagName("input"), fixDefaultChecked );
+				}
+			}
+		}
+
+		// Append elements to a provided document fragment
+		if ( fragment ) {
+			// Special handling of each script element
+			handleScript = function( elem ) {
+				// Check if we consider it executable
+				if ( !elem.type || rscriptType.test( elem.type ) ) {
+					// Detach the script and store it in the scripts array (if provided) or the fragment
+					// Return truthy to indicate that it has been handled
+					return scripts ?
+						scripts.push( elem.parentNode ? elem.parentNode.removeChild( elem ) : elem ) :
+						fragment.appendChild( elem );
+				}
+			};
+
+			for ( i = 0; (elem = ret[i]) != null; i++ ) {
+				// Check if we're done after handling an executable script
+				if ( !( jQuery.nodeName( elem, "script" ) && handleScript( elem ) ) ) {
+					// Append to fragment and handle embedded scripts
+					fragment.appendChild( elem );
+					if ( typeof elem.getElementsByTagName !== "undefined" ) {
+						// handleScript alters the DOM, so use jQuery.merge to ensure snapshot iteration
+						jsTags = jQuery.grep( jQuery.merge( [], elem.getElementsByTagName("script") ), handleScript );
+
+						// Splice the scripts into ret after their former ancestor and advance our index beyond them
+						ret.splice.apply( ret, [i + 1, 0].concat( jsTags ) );
+						i += jsTags.length;
+					}
+				}
+			}
+		}
+
+		return ret;
+	},
+
+	cleanData: function( elems, /* internal */ acceptData ) {
+		var data, id, elem, type,
+			i = 0,
+			internalKey = jQuery.expando,
+			cache = jQuery.cache,
+			deleteExpando = jQuery.support.deleteExpando,
+			special = jQuery.event.special;
+
+		for ( ; (elem = elems[i]) != null; i++ ) {
+
+			if ( acceptData || jQuery.acceptData( elem ) ) {
+
+				id = elem[ internalKey ];
+				data = id && cache[ id ];
+
+				if ( data ) {
+					if ( data.events ) {
+						for ( type in data.events ) {
+							if ( special[ type ] ) {
+								jQuery.event.remove( elem, type );
+
+							// This is a shortcut to avoid jQuery.event.remove's overhead
+							} else {
+								jQuery.removeEvent( elem, type, data.handle );
+							}
+						}
+					}
+
+					// Remove cache only if it was not already removed by jQuery.event.remove
+					if ( cache[ id ] ) {
+
+						delete cache[ id ];
+
+						// IE does not allow us to delete expando properties from nodes,
+						// nor does it have a removeAttribute function on Document nodes;
+						// we must handle all of these cases
+						if ( deleteExpando ) {
+							delete elem[ internalKey ];
+
+						} else if ( elem.removeAttribute ) {
+							elem.removeAttribute( internalKey );
+
+						} else {
+							elem[ internalKey ] = null;
+						}
+
+						jQuery.deletedIds.push( id );
+					}
+				}
+			}
+		}
+	}
+});
+// Limit scope pollution from any deprecated API
+(function() {
+
+var matched, browser;
+
+// Use of jQuery.browser is frowned upon.
+// More details: http://api.jquery.com/jQuery.browser
+// jQuery.uaMatch maintained for back-compat
+jQuery.uaMatch = function( ua ) {
+	ua = ua.toLowerCase();
+
+	var match = /(chrome)[ \/]([\w.]+)/.exec( ua ) ||
+		/(webkit)[ \/]([\w.]+)/.exec( ua ) ||
+		/(opera)(?:.*version|)[ \/]([\w.]+)/.exec( ua ) ||
+		/(msie) ([\w.]+)/.exec( ua ) ||
+		ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec( ua ) ||
+		[];
+
+	return {
+		browser: match[ 1 ] || "",
+		version: match[ 2 ] || "0"
+	};
+};
+
+matched = jQuery.uaMatch( navigator.userAgent );
+browser = {};
+
+if ( matched.browser ) {
+	browser[ matched.browser ] = true;
+	browser.version = matched.version;
+}
+
+// Chrome is Webkit, but Webkit is also Safari.
+if ( browser.chrome ) {
+	browser.webkit = true;
+} else if ( browser.webkit ) {
+	browser.safari = true;
+}
+
+jQuery.browser = browser;
+
+jQuery.sub = function() {
+	function jQuerySub( selector, context ) {
+		return new jQuerySub.fn.init( selector, context );
+	}
+	jQuery.extend( true, jQuerySub, this );
+	jQuerySub.superclass = this;
+	jQuerySub.fn = jQuerySub.prototype = this();
+	jQuerySub.fn.constructor = jQuerySub;
+	jQuerySub.sub = this.sub;
+	jQuerySub.fn.init = function init( selector, context ) {
+		if ( context && context instanceof jQuery && !(context instanceof jQuerySub) ) {
+			context = jQuerySub( context );
+		}
+
+		return jQuery.fn.init.call( this, selector, context, rootjQuerySub );
+	};
+	jQuerySub.fn.init.prototype = jQuerySub.fn;
+	var rootjQuerySub = jQuerySub(document);
+	return jQuerySub;
+};
+
+})();
+var curCSS, iframe, iframeDoc,
+	ralpha = /alpha\([^)]*\)/i,
+	ropacity = /opacity=([^)]*)/,
+	rposition = /^(top|right|bottom|left)$/,
+	// swappable if display is none or starts with table except "table", "table-cell", or "table-caption"
+	// see here for display values: https://developer.mozilla.org/en-US/docs/CSS/display
+	rdisplayswap = /^(none|table(?!-c[ea]).+)/,
+	rmargin = /^margin/,
+	rnumsplit = new RegExp( "^(" + core_pnum + ")(.*)$", "i" ),
+	rnumnonpx = new RegExp( "^(" + core_pnum + ")(?!px)[a-z%]+$", "i" ),
+	rrelNum = new RegExp( "^([-+])=(" + core_pnum + ")", "i" ),
+	elemdisplay = { BODY: "block" },
+
+	cssShow = { position: "absolute", visibility: "hidden", display: "block" },
+	cssNormalTransform = {
+		letterSpacing: 0,
+		fontWeight: 400
+	},
+
+	cssExpand = [ "Top", "Right", "Bottom", "Left" ],
+	cssPrefixes = [ "Webkit", "O", "Moz", "ms" ],
+
+	eventsToggle = jQuery.fn.toggle;
+
+// return a css property mapped to a potentially vendor prefixed property
+function vendorPropName( style, name ) {
+
+	// shortcut for names that are not vendor prefixed
+	if ( name in style ) {
+		return name;
+	}
+
+	// check for vendor prefixed names
+	var capName = name.charAt(0).toUpperCase() + name.slice(1),
+		origName = name,
+		i = cssPrefixes.length;
+
+	while ( i-- ) {
+		name = cssPrefixes[ i ] + capName;
+		if ( name in style ) {
+			return name;
+		}
+	}
+
+	return origName;
+}
+
+function isHidden( elem, el ) {
+	elem = el || elem;
+	return jQuery.css( elem, "display" ) === "none" || !jQuery.contains( elem.ownerDocument, elem );
+}
+
+function showHide( elements, show ) {
+	var elem, display,
+		values = [],
+		index = 0,
+		length = elements.length;
+
+	for ( ; index < length; index++ ) {
+		elem = elements[ index ];
+		if ( !elem.style ) {
+			continue;
+		}
+		values[ index ] = jQuery._data( elem, "olddisplay" );
+		if ( show ) {
+			// Reset the inline display of this element to learn if it is
+			// being hidden by cascaded rules or not
+			if ( !values[ index ] && elem.style.display === "none" ) {
+				elem.style.display = "";
+			}
+
+			// Set elements which have been overridden with display: none
+			// in a stylesheet to whatever the default browser style is
+			// for such an element
+			if ( elem.style.display === "" && isHidden( elem ) ) {
+				values[ index ] = jQuery._data( elem, "olddisplay", css_defaultDisplay(elem.nodeName) );
+			}
+		} else {
+			display = curCSS( elem, "display" );
+
+			if ( !values[ index ] && display !== "none" ) {
+				jQuery._data( elem, "olddisplay", display );
+			}
+		}
+	}
+
+	// Set the display of most of the elements in a second loop
+	// to avoid the constant reflow
+	for ( index = 0; index < length; index++ ) {
+		elem = elements[ index ];
+		if ( !elem.style ) {
+			continue;
+		}
+		if ( !show || elem.style.display === "none" || elem.style.display === "" ) {
+			elem.style.display = show ? values[ index ] || "" : "none";
+		}
+	}
+
+	return elements;
+}
+
+jQuery.fn.extend({
+	css: function( name, value ) {
+		return jQuery.access( this, function( elem, name, value ) {
+			return value !== undefined ?
+				jQuery.style( elem, name, value ) :
+				jQuery.css( elem, name );
+		}, name, value, arguments.length > 1 );
+	},
+	show: function() {
+		return showHide( this, true );
+	},
+	hide: function() {
+		return showHide( this );
+	},
+	toggle: function( state, fn2 ) {
+		var bool = typeof state === "boolean";
+
+		if ( jQuery.isFunction( state ) && jQuery.isFunction( fn2 ) ) {
+			return eventsToggle.apply( this, arguments );
+		}
+
+		return this.each(function() {
+			if ( bool ? state : isHidden( this ) ) {
+				jQuery( this ).show();
+			} else {
+				jQuery( this ).hide();
+			}
+		});
+	}
+});
+
+jQuery.extend({
+	// Add in style property hooks for overriding the default
+	// behavior of getting and setting a style property
+	cssHooks: {
+		opacity: {
+			get: function( elem, computed ) {
+				if ( computed ) {
+					// We should always get a number back from opacity
+					var ret = curCSS( elem, "opacity" );
+					return ret === "" ? "1" : ret;
+
+				}
+			}
+		}
+	},
+
+	// Exclude the following css properties to add px
+	cssNumber: {
+		"fillOpacity": true,
+		"fontWeight": true,
+		"lineHeight": true,
+		"opacity": true,
+		"orphans": true,
+		"widows": true,
+		"zIndex": true,
+		"zoom": true
+	},
+
+	// Add in properties whose names you wish to fix before
+	// setting or getting the value
+	cssProps: {
+		// normalize float css property
+		"float": jQuery.support.cssFloat ? "cssFloat" : "styleFloat"
+	},
+
+	// Get and set the style property on a DOM Node
+	style: function( elem, name, value, extra ) {
+		// Don't set styles on text and comment nodes
+		if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) {
+			return;
+		}
+
+		// Make sure that we're working with the right name
+		var ret, type, hooks,
+			origName = jQuery.camelCase( name ),
+			style = elem.style;
+
+		name = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( style, origName ) );
+
+		// gets hook for the prefixed version
+		// followed by the unprefixed version
+		hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ];
+
+		// Check if we're setting a value
+		if ( value !== undefined ) {
+			type = typeof value;
+
+			// convert relative number strings (+= or -=) to relative numbers. #7345
+			if ( type === "string" && (ret = rrelNum.exec( value )) ) {
+				value = ( ret[1] + 1 ) * ret[2] + parseFloat( jQuery.css( elem, name ) );
+				// Fixes bug #9237
+				type = "number";
+			}
+
+			// Make sure that NaN and null values aren't set. See: #7116
+			if ( value == null || type === "number" && isNaN( value ) ) {
+				return;
+			}
+
+			// If a number was passed in, add 'px' to the (except for certain CSS properties)
+			if ( type === "number" && !jQuery.cssNumber[ origName ] ) {
+				value += "px";
+			}
+
+			// If a hook was provided, use that value, otherwise just set the specified value
+			if ( !hooks || !("set" in hooks) || (value = hooks.set( elem, value, extra )) !== undefined ) {
+				// Wrapped to prevent IE from throwing errors when 'invalid' values are provided
+				// Fixes bug #5509
+				try {
+					style[ name ] = value;
+				} catch(e) {}
+			}
+
+		} else {
+			// If a hook was provided get the non-computed value from there
+			if ( hooks && "get" in hooks && (ret = hooks.get( elem, false, extra )) !== undefined ) {
+				return ret;
+			}
+
+			// Otherwise just get the value from the style object
+			return style[ name ];
+		}
+	},
+
+	css: function( elem, name, numeric, extra ) {
+		var val, num, hooks,
+			origName = jQuery.camelCase( name );
+
+		// Make sure that we're working with the right name
+		name = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( elem.style, origName ) );
+
+		// gets hook for the prefixed version
+		// followed by the unprefixed version
+		hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ];
+
+		// If a hook was provided get the computed value from there
+		if ( hooks && "get" in hooks ) {
+			val = hooks.get( elem, true, extra );
+		}
+
+		// Otherwise, if a way to get the computed value exists, use that
+		if ( val === undefined ) {
+			val = curCSS( elem, name );
+		}
+
+		//convert "normal" to computed value
+		if ( val === "normal" && name in cssNormalTransform ) {
+			val = cssNormalTransform[ name ];
+		}
+
+		// Return, converting to number if forced or a qualifier was provided and val looks numeric
+		if ( numeric || extra !== undefined ) {
+			num = parseFloat( val );
+			return numeric || jQuery.isNumeric( num ) ? num || 0 : val;
+		}
+		return val;
+	},
+
+	// A method for quickly swapping in/out CSS properties to get correct calculations
+	swap: function( elem, options, callback ) {
+		var ret, name,
+			old = {};
+
+		// Remember the old values, and insert the new ones
+		for ( name in options ) {
+			old[ name ] = elem.style[ name ];
+			elem.style[ name ] = options[ name ];
+		}
+
+		ret = callback.call( elem );
+
+		// Revert the old values
+		for ( name in options ) {
+			elem.style[ name ] = old[ name ];
+		}
+
+		return ret;
+	}
+});
+
+// NOTE: To any future maintainer, we've window.getComputedStyle
+// because jsdom on node.js will break without it.
+if ( window.getComputedStyle ) {
+	curCSS = function( elem, name ) {
+		var ret, width, minWidth, maxWidth,
+			computed = window.getComputedStyle( elem, null ),
+			style = elem.style;
+
+		if ( computed ) {
+
+			// getPropertyValue is only needed for .css('filter') in IE9, see #12537
+			ret = computed.getPropertyValue( name ) || computed[ name ];
+
+			if ( ret === "" && !jQuery.contains( elem.ownerDocument, elem ) ) {
+				ret = jQuery.style( elem, name );
+			}
+
+			// A tribute to the "awesome hack by Dean Edwards"
+			// Chrome < 17 and Safari 5.0 uses "computed value" instead of "used value" for margin-right
+			// Safari 5.1.7 (at least) returns percentage for a larger set of values, but width seems to be reliably pixels
+			// this is against the CSSOM draft spec: http://dev.w3.org/csswg/cssom/#resolved-values
+			if ( rnumnonpx.test( ret ) && rmargin.test( name ) ) {
+				width = style.width;
+				minWidth = style.minWidth;
+				maxWidth = style.maxWidth;
+
+				style.minWidth = style.maxWidth = style.width = ret;
+				ret = computed.width;
+
+				style.width = width;
+				style.minWidth = minWidth;
+				style.maxWidth = maxWidth;
+			}
+		}
+
+		return ret;
+	};
+} else if ( document.documentElement.currentStyle ) {
+	curCSS = function( elem, name ) {
+		var left, rsLeft,
+			ret = elem.currentStyle && elem.currentStyle[ name ],
+			style = elem.style;
+
+		// Avoid setting ret to empty string here
+		// so we don't default to auto
+		if ( ret == null && style && style[ name ] ) {
+			ret = style[ name ];
+		}
+
+		// From the awesome hack by Dean Edwards
+		// http://erik.eae.net/archives/2007/07/27/18.54.15/#comment-102291
+
+		// If we're not dealing with a regular pixel number
+		// but a number that has a weird ending, we need to convert it to pixels
+		// but not position css attributes, as those are proportional to the parent element instead
+		// and we can't measure the parent instead because it might trigger a "stacking dolls" problem
+		if ( rnumnonpx.test( ret ) && !rposition.test( name ) ) {
+
+			// Remember the original values
+			left = style.left;
+			rsLeft = elem.runtimeStyle && elem.runtimeStyle.left;
+
+			// Put in the new values to get a computed value out
+			if ( rsLeft ) {
+				elem.runtimeStyle.left = elem.currentStyle.left;
+			}
+			style.left = name === "fontSize" ? "1em" : ret;
+			ret = style.pixelLeft + "px";
+
+			// Revert the changed values
+			style.left = left;
+			if ( rsLeft ) {
+				elem.runtimeStyle.left = rsLeft;
+			}
+		}
+
+		return ret === "" ? "auto" : ret;
+	};
+}
+
+function setPositiveNumber( elem, value, subtract ) {
+	var matches = rnumsplit.exec( value );
+	return matches ?
+			Math.max( 0, matches[ 1 ] - ( subtract || 0 ) ) + ( matches[ 2 ] || "px" ) :
+			value;
+}
+
+function augmentWidthOrHeight( elem, name, extra, isBorderBox ) {
+	var i = extra === ( isBorderBox ? "border" : "content" ) ?
+		// If we already have the right measurement, avoid augmentation
+		4 :
+		// Otherwise initialize for horizontal or vertical properties
+		name === "width" ? 1 : 0,
+
+		val = 0;
+
+	for ( ; i < 4; i += 2 ) {
+		// both box models exclude margin, so add it if we want it
+		if ( extra === "margin" ) {
+			// we use jQuery.css instead of curCSS here
+			// because of the reliableMarginRight CSS hook!
+			val += jQuery.css( elem, extra + cssExpand[ i ], true );
+		}
+
+		// From this point on we use curCSS for maximum performance (relevant in animations)
+		if ( isBorderBox ) {
+			// border-box includes padding, so remove it if we want content
+			if ( extra === "content" ) {
+				val -= parseFloat( curCSS( elem, "padding" + cssExpand[ i ] ) ) || 0;
+			}
+
+			// at this point, extra isn't border nor margin, so remove border
+			if ( extra !== "margin" ) {
+				val -= parseFloat( curCSS( elem, "border" + cssExpand[ i ] + "Width" ) ) || 0;
+			}
+		} else {
+			// at this point, extra isn't content, so add padding
+			val += parseFloat( curCSS( elem, "padding" + cssExpand[ i ] ) ) || 0;
+
+			// at this point, extra isn't content nor padding, so add border
+			if ( extra !== "padding" ) {
+				val += parseFloat( curCSS( elem, "border" + cssExpand[ i ] + "Width" ) ) || 0;
+			}
+		}
+	}
+
+	return val;
+}
+
+function getWidthOrHeight( elem, name, extra ) {
+
+	// Start with offset property, which is equivalent to the border-box value
+	var val = name === "width" ? elem.offsetWidth : elem.offsetHeight,
+		valueIsBorderBox = true,
+		isBorderBox = jQuery.support.boxSizing && jQuery.css( elem, "boxSizing" ) === "border-box";
+
+	// some non-html elements return undefined for offsetWidth, so check for null/undefined
+	// svg - https://bugzilla.mozilla.org/show_bug.cgi?id=649285
+	// MathML - https://bugzilla.mozilla.org/show_bug.cgi?id=491668
+	if ( val <= 0 || val == null ) {
+		// Fall back to computed then uncomputed css if necessary
+		val = curCSS( elem, name );
+		if ( val < 0 || val == null ) {
+			val = elem.style[ name ];
+		}
+
+		// Computed unit is not pixels. Stop here and return.
+		if ( rnumnonpx.test(val) ) {
+			return val;
+		}
+
+		// we need the check for style in case a browser which returns unreliable values
+		// for getComputedStyle silently falls back to the reliable elem.style
+		valueIsBorderBox = isBorderBox && ( jQuery.support.boxSizingReliable || val === elem.style[ name ] );
+
+		// Normalize "", auto, and prepare for extra
+		val = parseFloat( val ) || 0;
+	}
+
+	// use the active box-sizing model to add/subtract irrelevant styles
+	return ( val +
+		augmentWidthOrHeight(
+			elem,
+			name,
+			extra || ( isBorderBox ? "border" : "content" ),
+			valueIsBorderBox
+		)
+	) + "px";
+}
+
+
+// Try to determine the default display value of an element
+function css_defaultDisplay( nodeName ) {
+	if ( elemdisplay[ nodeName ] ) {
+		return elemdisplay[ nodeName ];
+	}
+
+	var elem = jQuery( "<" + nodeName + ">" ).appendTo( document.body ),
+		display = elem.css("display");
+	elem.remove();
+
+	// If the simple way fails,
+	// get element's real default display by attaching it to a temp iframe
+	if ( display === "none" || display === "" ) {
+		// Use the already-created iframe if possible
+		iframe = document.body.appendChild(
+			iframe || jQuery.extend( document.createElement("iframe"), {
+				frameBorder: 0,
+				width: 0,
+				height: 0
+			})
+		);
+
+		// Create a cacheable copy of the iframe document on first call.
+		// IE and Opera will allow us to reuse the iframeDoc without re-writing the fake HTML
+		// document to it; WebKit & Firefox won't allow reusing the iframe document.
+		if ( !iframeDoc || !iframe.createElement ) {
+			iframeDoc = ( iframe.contentWindow || iframe.contentDocument ).document;
+			iframeDoc.write("<!doctype html><html><body>");
+			iframeDoc.close();
+		}
+
+		elem = iframeDoc.body.appendChild( iframeDoc.createElement(nodeName) );
+
+		display = curCSS( elem, "display" );
+		document.body.removeChild( iframe );
+	}
+
+	// Store the correct default display
+	elemdisplay[ nodeName ] = display;
+
+	return display;
+}
+
+jQuery.each([ "height", "width" ], function( i, name ) {
+	jQuery.cssHooks[ name ] = {
+		get: function( elem, computed, extra ) {
+			if ( computed ) {
+				// certain elements can have dimension info if we invisibly show them
+				// however, it must have a current display style that would benefit from this
+				if ( elem.offsetWidth === 0 && rdisplayswap.test( curCSS( elem, "display" ) ) ) {
+					return jQuery.swap( elem, cssShow, function() {
+						return getWidthOrHeight( elem, name, extra );
+					});
+				} else {
+					return getWidthOrHeight( elem, name, extra );
+				}
+			}
+		},
+
+		set: function( elem, value, extra ) {
+			return setPositiveNumber( elem, value, extra ?
+				augmentWidthOrHeight(
+					elem,
+					name,
+					extra,
+					jQuery.support.boxSizing && jQuery.css( elem, "boxSizing" ) === "border-box"
+				) : 0
+			);
+		}
+	};
+});
+
+if ( !jQuery.support.opacity ) {
+	jQuery.cssHooks.opacity = {
+		get: function( elem, computed ) {
+			// IE uses filters for opacity
+			return ropacity.test( (computed && elem.currentStyle ? elem.currentStyle.filter : elem.style.filter) || "" ) ?
+				( 0.01 * parseFloat( RegExp.$1 ) ) + "" :
+				computed ? "1" : "";
+		},
+
+		set: function( elem, value ) {
+			var style = elem.style,
+				currentStyle = elem.currentStyle,
+				opacity = jQuery.isNumeric( value ) ? "alpha(opacity=" + value * 100 + ")" : "",
+				filter = currentStyle && currentStyle.filter || style.filter || "";
+
+			// IE has trouble with opacity if it does not have layout
+			// Force it by setting the zoom level
+			style.zoom = 1;
+
+			// if setting opacity to 1, and no other filters exist - attempt to remove filter attribute #6652
+			if ( value >= 1 && jQuery.trim( filter.replace( ralpha, "" ) ) === "" &&
+				style.removeAttribute ) {
+
+				// Setting style.filter to null, "" & " " still leave "filter:" in the cssText
+				// if "filter:" is present at all, clearType is disabled, we want to avoid this
+				// style.removeAttribute is IE Only, but so apparently is this code path...
+				style.removeAttribute( "filter" );
+
+				// if there there is no filter style applied in a css rule, we are done
+				if ( currentStyle && !currentStyle.filter ) {
+					return;
+				}
+			}
+
+			// otherwise, set new filter values
+			style.filter = ralpha.test( filter ) ?
+				filter.replace( ralpha, opacity ) :
+				filter + " " + opacity;
+		}
+	};
+}
+
+// These hooks cannot be added until DOM ready because the support test
+// for it is not run until after DOM ready
+jQuery(function() {
+	if ( !jQuery.support.reliableMarginRight ) {
+		jQuery.cssHooks.marginRight = {
+			get: function( elem, computed ) {
+				// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
+				// Work around by temporarily setting element display to inline-block
+				return jQuery.swap( elem, { "display": "inline-block" }, function() {
+					if ( computed ) {
+						return curCSS( elem, "marginRight" );
+					}
+				});
+			}
+		};
+	}
+
+	// Webkit bug: https://bugs.webkit.org/show_bug.cgi?id=29084
+	// getComputedStyle returns percent when specified for top/left/bottom/right
+	// rather than make the css module depend on the offset module, we just check for it here
+	if ( !jQuery.support.pixelPosition && jQuery.fn.position ) {
+		jQuery.each( [ "top", "left" ], function( i, prop ) {
+			jQuery.cssHooks[ prop ] = {
+				get: function( elem, computed ) {
+					if ( computed ) {
+						var ret = curCSS( elem, prop );
+						// if curCSS returns percentage, fallback to offset
+						return rnumnonpx.test( ret ) ? jQuery( elem ).position()[ prop ] + "px" : ret;
+					}
+				}
+			};
+		});
+	}
+
+});
+
+if ( jQuery.expr && jQuery.expr.filters ) {
+	jQuery.expr.filters.hidden = function( elem ) {
+		return ( elem.offsetWidth === 0 && elem.offsetHeight === 0 ) || (!jQuery.support.reliableHiddenOffsets && ((elem.style && elem.style.display) || curCSS( elem, "display" )) === "none");
+	};
+
+	jQuery.expr.filters.visible = function( elem ) {
+		return !jQuery.expr.filters.hidden( elem );
+	};
+}
+
+// These hooks are used by animate to expand properties
+jQuery.each({
+	margin: "",
+	padding: "",
+	border: "Width"
+}, function( prefix, suffix ) {
+	jQuery.cssHooks[ prefix + suffix ] = {
+		expand: function( value ) {
+			var i,
+
+				// assumes a single number if not a string
+				parts = typeof value === "string" ? value.split(" ") : [ value ],
+				expanded = {};
+
+			for ( i = 0; i < 4; i++ ) {
+				expanded[ prefix + cssExpand[ i ] + suffix ] =
+					parts[ i ] || parts[ i - 2 ] || parts[ 0 ];
+			}
+
+			return expanded;
+		}
+	};
+
+	if ( !rmargin.test( prefix ) ) {
+		jQuery.cssHooks[ prefix + suffix ].set = setPositiveNumber;
+	}
+});
+var r20 = /%20/g,
+	rbracket = /\[\]$/,
+	rCRLF = /\r?\n/g,
+	rinput = /^(?:color|date|datetime|datetime-local|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,
+	rselectTextarea = /^(?:select|textarea)/i;
+
+jQuery.fn.extend({
+	serialize: function() {
+		return jQuery.param( this.serializeArray() );
+	},
+	serializeArray: function() {
+		return this.map(function(){
+			return this.elements ? jQuery.makeArray( this.elements ) : this;
+		})
+		.filter(function(){
+			return this.name && !this.disabled &&
+				( this.checked || rselectTextarea.test( this.nodeName ) ||
+					rinput.test( this.type ) );
+		})
+		.map(function( i, elem ){
+			var val = jQuery( this ).val();
+
+			return val == null ?
+				null :
+				jQuery.isArray( val ) ?
+					jQuery.map( val, function( val, i ){
+						return { name: elem.name, value: val.replace( rCRLF, "\r\n" ) };
+					}) :
+					{ name: elem.name, value: val.replace( rCRLF, "\r\n" ) };
+		}).get();
+	}
+});
+
+//Serialize an array of form elements or a set of
+//key/values into a query string
+jQuery.param = function( a, traditional ) {
+	var prefix,
+		s = [],
+		add = function( key, value ) {
+			// If value is a function, invoke it and return its value
+			value = jQuery.isFunction( value ) ? value() : ( value == null ? "" : value );
+			s[ s.length ] = encodeURIComponent( key ) + "=" + encodeURIComponent( value );
+		};
+
+	// Set traditional to true for jQuery <= 1.3.2 behavior.
+	if ( traditional === undefined ) {
+		traditional = jQuery.ajaxSettings && jQuery.ajaxSettings.traditional;
+	}
+
+	// If an array was passed in, assume that it is an array of form elements.
+	if ( jQuery.isArray( a ) || ( a.jquery && !jQuery.isPlainObject( a ) ) ) {
+		// Serialize the form elements
+		jQuery.each( a, function() {
+			add( this.name, this.value );
+		});
+
+	} else {
+		// If traditional, encode the "old" way (the way 1.3.2 or older
+		// did it), otherwise encode params recursively.
+		for ( prefix in a ) {
+			buildParams( prefix, a[ prefix ], traditional, add );
+		}
+	}
+
+	// Return the resulting serialization
+	return s.join( "&" ).replace( r20, "+" );
+};
+
+function buildParams( prefix, obj, traditional, add ) {
+	var name;
+
+	if ( jQuery.isArray( obj ) ) {
+		// Serialize array item.
+		jQuery.each( obj, function( i, v ) {
+			if ( traditional || rbracket.test( prefix ) ) {
+				// Treat each array item as a scalar.
+				add( prefix, v );
+
+			} else {
+				// If array item is non-scalar (array or object), encode its
+				// numeric index to resolve deserialization ambiguity issues.
+				// Note that rack (as of 1.0.0) can't currently deserialize
+				// nested arrays properly, and attempting to do so may cause
+				// a server error. Possible fixes are to modify rack's
+				// deserialization algorithm or to provide an option or flag
+				// to force array serialization to be shallow.
+				buildParams( prefix + "[" + ( typeof v === "object" ? i : "" ) + "]", v, traditional, add );
+			}
+		});
+
+	} else if ( !traditional && jQuery.type( obj ) === "object" ) {
+		// Serialize object item.
+		for ( name in obj ) {
+			buildParams( prefix + "[" + name + "]", obj[ name ], traditional, add );
+		}
+
+	} else {
+		// Serialize scalar item.
+		add( prefix, obj );
+	}
+}
+var
+	// Document location
+	ajaxLocParts,
+	ajaxLocation,
+
+	rhash = /#.*$/,
+	rheaders = /^(.*?):[ \t]*([^\r\n]*)\r?$/mg, // IE leaves an \r character at EOL
+	// #7653, #8125, #8152: local protocol detection
+	rlocalProtocol = /^(?:about|app|app\-storage|.+\-extension|file|res|widget):$/,
+	rnoContent = /^(?:GET|HEAD)$/,
+	rprotocol = /^\/\//,
+	rquery = /\?/,
+	rscript = /<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,
+	rts = /([?&])_=[^&]*/,
+	rurl = /^([\w\+\.\-]+:)(?:\/\/([^\/?#:]*)(?::(\d+)|)|)/,
+
+	// Keep a copy of the old load method
+	_load = jQuery.fn.load,
+
+	/* Prefilters
+	 * 1) They are useful to introduce custom dataTypes (see ajax/jsonp.js for an example)
+	 * 2) These are called:
+	 *    - BEFORE asking for a transport
+	 *    - AFTER param serialization (s.data is a string if s.processData is true)
+	 * 3) key is the dataType
+	 * 4) the catchall symbol "*" can be used
+	 * 5) execution will start with transport dataType and THEN continue down to "*" if needed
+	 */
+	prefilters = {},
+
+	/* Transports bindings
+	 * 1) key is the dataType
+	 * 2) the catchall symbol "*" can be used
+	 * 3) selection will start with transport dataType and THEN go to "*" if needed
+	 */
+	transports = {},
+
+	// Avoid comment-prolog char sequence (#10098); must appease lint and evade compression
+	allTypes = ["*/"] + ["*"];
+
+// #8138, IE may throw an exception when accessing
+// a field from window.location if document.domain has been set
+try {
+	ajaxLocation = location.href;
+} catch( e ) {
+	// Use the href attribute of an A element
+	// since IE will modify it given document.location
+	ajaxLocation = document.createElement( "a" );
+	ajaxLocation.href = "";
+	ajaxLocation = ajaxLocation.href;
+}
+
+// Segment location into parts
+ajaxLocParts = rurl.exec( ajaxLocation.toLowerCase() ) || [];
+
+// Base "constructor" for jQuery.ajaxPrefilter and jQuery.ajaxTransport
+function addToPrefiltersOrTransports( structure ) {
+
+	// dataTypeExpression is optional and defaults to "*"
+	return function( dataTypeExpression, func ) {
+
+		if ( typeof dataTypeExpression !== "string" ) {
+			func = dataTypeExpression;
+			dataTypeExpression = "*";
+		}
+
+		var dataType, list, placeBefore,
+			dataTypes = dataTypeExpression.toLowerCase().split( core_rspace ),
+			i = 0,
+			length = dataTypes.length;
+
+		if ( jQuery.isFunction( func ) ) {
+			// For each dataType in the dataTypeExpression
+			for ( ; i < length; i++ ) {
+				dataType = dataTypes[ i ];
+				// We control if we're asked to add before
+				// any existing element
+				placeBefore = /^\+/.test( dataType );
+				if ( placeBefore ) {
+					dataType = dataType.substr( 1 ) || "*";
+				}
+				list = structure[ dataType ] = structure[ dataType ] || [];
+				// then we add to the structure accordingly
+				list[ placeBefore ? "unshift" : "push" ]( func );
+			}
+		}
+	};
+}
+
+// Base inspection function for prefilters and transports
+function inspectPrefiltersOrTransports( structure, options, originalOptions, jqXHR,
+		dataType /* internal */, inspected /* internal */ ) {
+
+	dataType = dataType || options.dataTypes[ 0 ];
+	inspected = inspected || {};
+
+	inspected[ dataType ] = true;
+
+	var selection,
+		list = structure[ dataType ],
+		i = 0,
+		length = list ? list.length : 0,
+		executeOnly = ( structure === prefilters );
+
+	for ( ; i < length && ( executeOnly || !selection ); i++ ) {
+		selection = list[ i ]( options, originalOptions, jqXHR );
+		// If we got redirected to another dataType
+		// we try there if executing only and not done already
+		if ( typeof selection === "string" ) {
+			if ( !executeOnly || inspected[ selection ] ) {
+				selection = undefined;
+			} else {
+				options.dataTypes.unshift( selection );
+				selection = inspectPrefiltersOrTransports(
+						structure, options, originalOptions, jqXHR, selection, inspected );
+			}
+		}
+	}
+	// If we're only executing or nothing was selected
+	// we try the catchall dataType if not done already
+	if ( ( executeOnly || !selection ) && !inspected[ "*" ] ) {
+		selection = inspectPrefiltersOrTransports(
+				structure, options, originalOptions, jqXHR, "*", inspected );
+	}
+	// unnecessary when only executing (prefilters)
+	// but it'll be ignored by the caller in that case
+	return selection;
+}
+
+// A special extend for ajax options
+// that takes "flat" options (not to be deep extended)
+// Fixes #9887
+function ajaxExtend( target, src ) {
+	var key, deep,
+		flatOptions = jQuery.ajaxSettings.flatOptions || {};
+	for ( key in src ) {
+		if ( src[ key ] !== undefined ) {
+			( flatOptions[ key ] ? target : ( deep || ( deep = {} ) ) )[ key ] = src[ key ];
+		}
+	}
+	if ( deep ) {
+		jQuery.extend( true, target, deep );
+	}
+}
+
+jQuery.fn.load = function( url, params, callback ) {
+	if ( typeof url !== "string" && _load ) {
+		return _load.apply( this, arguments );
+	}
+
+	// Don't do a request if no elements are being requested
+	if ( !this.length ) {
+		return this;
+	}
+
+	var selector, type, response,
+		self = this,
+		off = url.indexOf(" ");
+
+	if ( off >= 0 ) {
+		selector = url.slice( off, url.length );
+		url = url.slice( 0, off );
+	}
+
+	// If it's a function
+	if ( jQuery.isFunction( params ) ) {
+
+		// We assume that it's the callback
+		callback = params;
+		params = undefined;
+
+	// Otherwise, build a param string
+	} else if ( params && typeof params === "object" ) {
+		type = "POST";
+	}
+
+	// Request the remote document
+	jQuery.ajax({
+		url: url,
+
+		// if "type" variable is undefined, then "GET" method will be used
+		type: type,
+		dataType: "html",
+		data: params,
+		complete: function( jqXHR, status ) {
+			if ( callback ) {
+				self.each( callback, response || [ jqXHR.responseText, status, jqXHR ] );
+			}
+		}
+	}).done(function( responseText ) {
+
+		// Save response for use in complete callback
+		response = arguments;
+
+		// See if a selector was specified
+		self.html( selector ?
+
+			// Create a dummy div to hold the results
+			jQuery("<div>")
+
+				// inject the contents of the document in, removing the scripts
+				// to avoid any 'Permission Denied' errors in IE
+				.append( responseText.replace( rscript, "" ) )
+
+				// Locate the specified elements
+				.find( selector ) :
+
+			// If not, just inject the full result
+			responseText );
+
+	});
+
+	return this;
+};
+
+// Attach a bunch of functions for handling common AJAX events
+jQuery.each( "ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split( " " ), function( i, o ){
+	jQuery.fn[ o ] = function( f ){
+		return this.on( o, f );
+	};
+});
+
+jQuery.each( [ "get", "post" ], function( i, method ) {
+	jQuery[ method ] = function( url, data, callback, type ) {
+		// shift arguments if data argument was omitted
+		if ( jQuery.isFunction( data ) ) {
+			type = type || callback;
+			callback = data;
+			data = undefined;
+		}
+
+		return jQuery.ajax({
+			type: method,
+			url: url,
+			data: data,
+			success: callback,
+			dataType: type
+		});
+	};
+});
+
+jQuery.extend({
+
+	getScript: function( url, callback ) {
+		return jQuery.get( url, undefined, callback, "script" );
+	},
+
+	getJSON: function( url, data, callback ) {
+		return jQuery.get( url, data, callback, "json" );
+	},
+
+	// Creates a full fledged settings object into target
+	// with both ajaxSettings and settings fields.
+	// If target is omitted, writes into ajaxSettings.
+	ajaxSetup: function( target, settings ) {
+		if ( settings ) {
+			// Building a settings object
+			ajaxExtend( target, jQuery.ajaxSettings );
+		} else {
+			// Extending ajaxSettings
+			settings = target;
+			target = jQuery.ajaxSettings;
+		}
+		ajaxExtend( target, settings );
+		return target;
+	},
+
+	ajaxSettings: {
+		url: ajaxLocation,
+		isLocal: rlocalProtocol.test( ajaxLocParts[ 1 ] ),
+		global: true,
+		type: "GET",
+		contentType: "application/x-www-form-urlencoded; charset=UTF-8",
+		processData: true,
+		async: true,
+		/*
+		timeout: 0,
+		data: null,
+		dataType: null,
+		username: null,
+		password: null,
+		cache: null,
+		throws: false,
+		traditional: false,
+		headers: {},
+		*/
+
+		accepts: {
+			xml: "application/xml, text/xml",
+			html: "text/html",
+			text: "text/plain",
+			json: "application/json, text/javascript",
+			"*": allTypes
+		},
+
+		contents: {
+			xml: /xml/,
+			html: /html/,
+			json: /json/
+		},
+
+		responseFields: {
+			xml: "responseXML",
+			text: "responseText"
+		},
+
+		// List of data converters
+		// 1) key format is "source_type destination_type" (a single space in-between)
+		// 2) the catchall symbol "*" can be used for source_type
+		converters: {
+
+			// Convert anything to text
+			"* text": window.String,
+
+			// Text to html (true = no transformation)
+			"text html": true,
+
+			// Evaluate text as a json expression
+			"text json": jQuery.parseJSON,
+
+			// Parse text as xml
+			"text xml": jQuery.parseXML
+		},
+
+		// For options that shouldn't be deep extended:
+		// you can add your own custom options here if
+		// and when you create one that shouldn't be
+		// deep extended (see ajaxExtend)
+		flatOptions: {
+			context: true,
+			url: true
+		}
+	},
+
+	ajaxPrefilter: addToPrefiltersOrTransports( prefilters ),
+	ajaxTransport: addToPrefiltersOrTransports( transports ),
+
+	// Main method
+	ajax: function( url, options ) {
+
+		// If url is an object, simulate pre-1.5 signature
+		if ( typeof url === "object" ) {
+			options = url;
+			url = undefined;
+		}
+
+		// Force options to be an object
+		options = options || {};
+
+		var // ifModified key
+			ifModifiedKey,
+			// Response headers
+			responseHeadersString,
+			responseHeaders,
+			// transport
+			transport,
+			// timeout handle
+			timeoutTimer,
+			// Cross-domain detection vars
+			parts,
+			// To know if global events are to be dispatched
+			fireGlobals,
+			// Loop variable
+			i,
+			// Create the final options object
+			s = jQuery.ajaxSetup( {}, options ),
+			// Callbacks context
+			callbackContext = s.context || s,
+			// Context for global events
+			// It's the callbackContext if one was provided in the options
+			// and if it's a DOM node or a jQuery collection
+			globalEventContext = callbackContext !== s &&
+				( callbackContext.nodeType || callbackContext instanceof jQuery ) ?
+						jQuery( callbackContext ) : jQuery.event,
+			// Deferreds
+			deferred = jQuery.Deferred(),
+			completeDeferred = jQuery.Callbacks( "once memory" ),
+			// Status-dependent callbacks
+			statusCode = s.statusCode || {},
+			// Headers (they are sent all at once)
+			requestHeaders = {},
+			requestHeadersNames = {},
+			// The jqXHR state
+			state = 0,
+			// Default abort message
+			strAbort = "canceled",
+			// Fake xhr
+			jqXHR = {
+
+				readyState: 0,
+
+				// Caches the header
+				setRequestHeader: function( name, value ) {
+					if ( !state ) {
+						var lname = name.toLowerCase();
+						name = requestHeadersNames[ lname ] = requestHeadersNames[ lname ] || name;
+						requestHeaders[ name ] = value;
+					}
+					return this;
+				},
+
+				// Raw string
+				getAllResponseHeaders: function() {
+					return state === 2 ? responseHeadersString : null;
+				},
+
+				// Builds headers hashtable if needed
+				getResponseHeader: function( key ) {
+					var match;
+					if ( state === 2 ) {
+						if ( !responseHeaders ) {
+							responseHeaders = {};
+							while( ( match = rheaders.exec( responseHeadersString ) ) ) {
+								responseHeaders[ match[1].toLowerCase() ] = match[ 2 ];
+							}
+						}
+						match = responseHeaders[ key.toLowerCase() ];
+					}
+					return match === undefined ? null : match;
+				},
+
+				// Overrides response content-type header
+				overrideMimeType: function( type ) {
+					if ( !state ) {
+						s.mimeType = type;
+					}
+					return this;
+				},
+
+				// Cancel the request
+				abort: function( statusText ) {
+					statusText = statusText || strAbort;
+					if ( transport ) {
+						transport.abort( statusText );
+					}
+					done( 0, statusText );
+					return this;
+				}
+			};
+
+		// Callback for when everything is done
+		// It is defined here because jslint complains if it is declared
+		// at the end of the function (which would be more logical and readable)
+		function done( status, nativeStatusText, responses, headers ) {
+			var isSuccess, success, error, response, modified,
+				statusText = nativeStatusText;
+
+			// Called once
+			if ( state === 2 ) {
+				return;
+			}
+
+			// State is "done" now
+			state = 2;
+
+			// Clear timeout if it exists
+			if ( timeoutTimer ) {
+				clearTimeout( timeoutTimer );
+			}
+
+			// Dereference transport for early garbage collection
+			// (no matter how long the jqXHR object will be used)
+			transport = undefined;
+
+			// Cache response headers
+			responseHeadersString = headers || "";
+
+			// Set readyState
+			jqXHR.readyState = status > 0 ? 4 : 0;
+
+			// Get response data
+			if ( responses ) {
+				response = ajaxHandleResponses( s, jqXHR, responses );
+			}
+
+			// If successful, handle type chaining
+			if ( status >= 200 && status < 300 || status === 304 ) {
+
+				// Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode.
+				if ( s.ifModified ) {
+
+					modified = jqXHR.getResponseHeader("Last-Modified");
+					if ( modified ) {
+						jQuery.lastModified[ ifModifiedKey ] = modified;
+					}
+					modified = jqXHR.getResponseHeader("Etag");
+					if ( modified ) {
+						jQuery.etag[ ifModifiedKey ] = modified;
+					}
+				}
+
+				// If not modified
+				if ( status === 304 ) {
+
+					statusText = "notmodified";
+					isSuccess = true;
+
+				// If we have data
+				} else {
+
+					isSuccess = ajaxConvert( s, response );
+					statusText = isSuccess.state;
+					success = isSuccess.data;
+					error = isSuccess.error;
+					isSuccess = !error;
+				}
+			} else {
+				// We extract error from statusText
+				// then normalize statusText and status for non-aborts
+				error = statusText;
+				if ( !statusText || status ) {
+					statusText = "error";
+					if ( status < 0 ) {
+						status = 0;
+					}
+				}
+			}
+
+			// Set data for the fake xhr object
+			jqXHR.status = status;
+			jqXHR.statusText = ( nativeStatusText || statusText ) + "";
+
+			// Success/Error
+			if ( isSuccess ) {
+				deferred.resolveWith( callbackContext, [ success, statusText, jqXHR ] );
+			} else {
+				deferred.rejectWith( callbackContext, [ jqXHR, statusText, error ] );
+			}
+
+			// Status-dependent callbacks
+			jqXHR.statusCode( statusCode );
+			statusCode = undefined;
+
+			if ( fireGlobals ) {
+				globalEventContext.trigger( "ajax" + ( isSuccess ? "Success" : "Error" ),
+						[ jqXHR, s, isSuccess ? success : error ] );
+			}
+
+			// Complete
+			completeDeferred.fireWith( callbackContext, [ jqXHR, statusText ] );
+
+			if ( fireGlobals ) {
+				globalEventContext.trigger( "ajaxComplete", [ jqXHR, s ] );
+				// Handle the global AJAX counter
+				if ( !( --jQuery.active ) ) {
+					jQuery.event.trigger( "ajaxStop" );
+				}
+			}
+		}
+
+		// Attach deferreds
+		deferred.promise( jqXHR );
+		jqXHR.success = jqXHR.done;
+		jqXHR.error = jqXHR.fail;
+		jqXHR.complete = completeDeferred.add;
+
+		// Status-dependent callbacks
+		jqXHR.statusCode = function( map ) {
+			if ( map ) {
+				var tmp;
+				if ( state < 2 ) {
+					for ( tmp in map ) {
+						statusCode[ tmp ] = [ statusCode[tmp], map[tmp] ];
+					}
+				} else {
+					tmp = map[ jqXHR.status ];
+					jqXHR.always( tmp );
+				}
+			}
+			return this;
+		};
+
+		// Remove hash character (#7531: and string promotion)
+		// Add protocol if not provided (#5866: IE7 issue with protocol-less urls)
+		// We also use the url parameter if available
+		s.url = ( ( url || s.url ) + "" ).replace( rhash, "" ).replace( rprotocol, ajaxLocParts[ 1 ] + "//" );
+
+		// Extract dataTypes list
+		s.dataTypes = jQuery.trim( s.dataType || "*" ).toLowerCase().split( core_rspace );
+
+		// A cross-domain request is in order when we have a protocol:host:port mismatch
+		if ( s.crossDomain == null ) {
+			parts = rurl.exec( s.url.toLowerCase() );
+			s.crossDomain = !!( parts &&
+				( parts[ 1 ] !== ajaxLocParts[ 1 ] || parts[ 2 ] !== ajaxLocParts[ 2 ] ||
+					( parts[ 3 ] || ( parts[ 1 ] === "http:" ? 80 : 443 ) ) !=
+						( ajaxLocParts[ 3 ] || ( ajaxLocParts[ 1 ] === "http:" ? 80 : 443 ) ) )
+			);
+		}
+
+		// Convert data if not already a string
+		if ( s.data && s.processData && typeof s.data !== "string" ) {
+			s.data = jQuery.param( s.data, s.traditional );
+		}
+
+		// Apply prefilters
+		inspectPrefiltersOrTransports( prefilters, s, options, jqXHR );
+
+		// If request was aborted inside a prefilter, stop there
+		if ( state === 2 ) {
+			return jqXHR;
+		}
+
+		// We can fire global events as of now if asked to
+		fireGlobals = s.global;
+
+		// Uppercase the type
+		s.type = s.type.toUpperCase();
+
+		// Determine if request has content
+		s.hasContent = !rnoContent.test( s.type );
+
+		// Watch for a new set of requests
+		if ( fireGlobals && jQuery.active++ === 0 ) {
+			jQuery.event.trigger( "ajaxStart" );
+		}
+
+		// More options handling for requests with no content
+		if ( !s.hasContent ) {
+
+			// If data is available, append data to url
+			if ( s.data ) {
+				s.url += ( rquery.test( s.url ) ? "&" : "?" ) + s.data;
+				// #9682: remove data so that it's not used in an eventual retry
+				delete s.data;
+			}
+
+			// Get ifModifiedKey before adding the anti-cache parameter
+			ifModifiedKey = s.url;
+
+			// Add anti-cache in url if needed
+			if ( s.cache === false ) {
+
+				var ts = jQuery.now(),
+					// try replacing _= if it is there
+					ret = s.url.replace( rts, "$1_=" + ts );
+
+				// if nothing was replaced, add timestamp to the end
+				s.url = ret + ( ( ret === s.url ) ? ( rquery.test( s.url ) ? "&" : "?" ) + "_=" + ts : "" );
+			}
+		}
+
+		// Set the correct header, if data is being sent
+		if ( s.data && s.hasContent && s.contentType !== false || options.contentType ) {
+			jqXHR.setRequestHeader( "Content-Type", s.contentType );
+		}
+
+		// Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode.
+		if ( s.ifModified ) {
+			ifModifiedKey = ifModifiedKey || s.url;
+			if ( jQuery.lastModified[ ifModifiedKey ] ) {
+				jqXHR.setRequestHeader( "If-Modified-Since", jQuery.lastModified[ ifModifiedKey ] );
+			}
+			if ( jQuery.etag[ ifModifiedKey ] ) {
+				jqXHR.setRequestHeader( "If-None-Match", jQuery.etag[ ifModifiedKey ] );
+			}
+		}
+
+		// Set the Accepts header for the server, depending on the dataType
+		jqXHR.setRequestHeader(
+			"Accept",
+			s.dataTypes[ 0 ] && s.accepts[ s.dataTypes[0] ] ?
+				s.accepts[ s.dataTypes[0] ] + ( s.dataTypes[ 0 ] !== "*" ? ", " + allTypes + "; q=0.01" : "" ) :
+				s.accepts[ "*" ]
+		);
+
+		// Check for headers option
+		for ( i in s.headers ) {
+			jqXHR.setRequestHeader( i, s.headers[ i ] );
+		}
+
+		// Allow custom headers/mimetypes and early abort
+		if ( s.beforeSend && ( s.beforeSend.call( callbackContext, jqXHR, s ) === false || state === 2 ) ) {
+				// Abort if not done already and return
+				return jqXHR.abort();
+
+		}
+
+		// aborting is no longer a cancellation
+		strAbort = "abort";
+
+		// Install callbacks on deferreds
+		for ( i in { success: 1, error: 1, complete: 1 } ) {
+			jqXHR[ i ]( s[ i ] );
+		}
+
+		// Get transport
+		transport = inspectPrefiltersOrTransports( transports, s, options, jqXHR );
+
+		// If no transport, we auto-abort
+		if ( !transport ) {
+			done( -1, "No Transport" );
+		} else {
+			jqXHR.readyState = 1;
+			// Send global event
+			if ( fireGlobals ) {
+				globalEventContext.trigger( "ajaxSend", [ jqXHR, s ] );
+			}
+			// Timeout
+			if ( s.async && s.timeout > 0 ) {
+				timeoutTimer = setTimeout( function(){
+					jqXHR.abort( "timeout" );
+				}, s.timeout );
+			}
+
+			try {
+				state = 1;
+				transport.send( requestHeaders, done );
+			} catch (e) {
+				// Propagate exception as error if not done
+				if ( state < 2 ) {
+					done( -1, e );
+				// Simply rethrow otherwise
+				} else {
+					throw e;
+				}
+			}
+		}
+
+		return jqXHR;
+	},
+
+	// Counter for holding the number of active queries
+	active: 0,
+
+	// Last-Modified header cache for next request
+	lastModified: {},
+	etag: {}
+
+});
+
+/* Handles responses to an ajax request:
+ * - sets all responseXXX fields accordingly
+ * - finds the right dataType (mediates between content-type and expected dataType)
+ * - returns the corresponding response
+ */
+function ajaxHandleResponses( s, jqXHR, responses ) {
+
+	var ct, type, finalDataType, firstDataType,
+		contents = s.contents,
+		dataTypes = s.dataTypes,
+		responseFields = s.responseFields;
+
+	// Fill responseXXX fields
+	for ( type in responseFields ) {
+		if ( type in responses ) {
+			jqXHR[ responseFields[type] ] = responses[ type ];
+		}
+	}
+
+	// Remove auto dataType and get content-type in the process
+	while( dataTypes[ 0 ] === "*" ) {
+		dataTypes.shift();
+		if ( ct === undefined ) {
+			ct = s.mimeType || jqXHR.getResponseHeader( "content-type" );
+		}
+	}
+
+	// Check if we're dealing with a known content-type
+	if ( ct ) {
+		for ( type in contents ) {
+			if ( contents[ type ] && contents[ type ].test( ct ) ) {
+				dataTypes.unshift( type );
+				break;
+			}
+		}
+	}
+
+	// Check to see if we have a response for the expected dataType
+	if ( dataTypes[ 0 ] in responses ) {
+		finalDataType = dataTypes[ 0 ];
+	} else {
+		// Try convertible dataTypes
+		for ( type in responses ) {
+			if ( !dataTypes[ 0 ] || s.converters[ type + " " + dataTypes[0] ] ) {
+				finalDataType = type;
+				break;
+			}
+			if ( !firstDataType ) {
+				firstDataType = type;
+			}
+		}
+		// Or just use first one
+		finalDataType = finalDataType || firstDataType;
+	}
+
+	// If we found a dataType
+	// We add the dataType to the list if needed
+	// and return the corresponding response
+	if ( finalDataType ) {
+		if ( finalDataType !== dataTypes[ 0 ] ) {
+			dataTypes.unshift( finalDataType );
+		}
+		return responses[ finalDataType ];
+	}
+}
+
+// Chain conversions given the request and the original response
+function ajaxConvert( s, response ) {
+
+	var conv, conv2, current, tmp,
+		// Work with a copy of dataTypes in case we need to modify it for conversion
+		dataTypes = s.dataTypes.slice(),
+		prev = dataTypes[ 0 ],
+		converters = {},
+		i = 0;
+
+	// Apply the dataFilter if provided
+	if ( s.dataFilter ) {
+		response = s.dataFilter( response, s.dataType );
+	}
+
+	// Create converters map with lowercased keys
+	if ( dataTypes[ 1 ] ) {
+		for ( conv in s.converters ) {
+			converters[ conv.toLowerCase() ] = s.converters[ conv ];
+		}
+	}
+
+	// Convert to each sequential dataType, tolerating list modification
+	for ( ; (current = dataTypes[++i]); ) {
+
+		// There's only work to do if current dataType is non-auto
+		if ( current !== "*" ) {
+
+			// Convert response if prev dataType is non-auto and differs from current
+			if ( prev !== "*" && prev !== current ) {
+
+				// Seek a direct converter
+				conv = converters[ prev + " " + current ] || converters[ "* " + current ];
+
+				// If none found, seek a pair
+				if ( !conv ) {
+					for ( conv2 in converters ) {
+
+						// If conv2 outputs current
+						tmp = conv2.split(" ");
+						if ( tmp[ 1 ] === current ) {
+
+							// If prev can be converted to accepted input
+							conv = converters[ prev + " " + tmp[ 0 ] ] ||
+								converters[ "* " + tmp[ 0 ] ];
+							if ( conv ) {
+								// Condense equivalence converters
+								if ( conv === true ) {
+									conv = converters[ conv2 ];
+
+								// Otherwise, insert the intermediate dataType
+								} else if ( converters[ conv2 ] !== true ) {
+									current = tmp[ 0 ];
+									dataTypes.splice( i--, 0, current );
+								}
+
+								break;
+							}
+						}
+					}
+				}
+
+				// Apply converter (if not an equivalence)
+				if ( conv !== true ) {
+
+					// Unless errors are allowed to bubble, catch and return them
+					if ( conv && s["throws"] ) {
+						response = conv( response );
+					} else {
+						try {
+							response = conv( response );
+						} catch ( e ) {
+							return { state: "parsererror", error: conv ? e : "No conversion from " + prev + " to " + current };
+						}
+					}
+				}
+			}
+
+			// Update prev for next iteration
+			prev = current;
+		}
+	}
+
+	return { state: "success", data: response };
+}
+var oldCallbacks = [],
+	rquestion = /\?/,
+	rjsonp = /(=)\?(?=&|$)|\?\?/,
+	nonce = jQuery.now();
+
+// Default jsonp settings
+jQuery.ajaxSetup({
+	jsonp: "callback",
+	jsonpCallback: function() {
+		var callback = oldCallbacks.pop() || ( jQuery.expando + "_" + ( nonce++ ) );
+		this[ callback ] = true;
+		return callback;
+	}
+});
+
+// Detect, normalize options and install callbacks for jsonp requests
+jQuery.ajaxPrefilter( "json jsonp", function( s, originalSettings, jqXHR ) {
+
+	var callbackName, overwritten, responseContainer,
+		data = s.data,
+		url = s.url,
+		hasCallback = s.jsonp !== false,
+		replaceInUrl = hasCallback && rjsonp.test( url ),
+		replaceInData = hasCallback && !replaceInUrl && typeof data === "string" &&
+			!( s.contentType || "" ).indexOf("application/x-www-form-urlencoded") &&
+			rjsonp.test( data );
+
+	// Handle iff the expected data type is "jsonp" or we have a parameter to set
+	if ( s.dataTypes[ 0 ] === "jsonp" || replaceInUrl || replaceInData ) {
+
+		// Get callback name, remembering preexisting value associated with it
+		callbackName = s.jsonpCallback = jQuery.isFunction( s.jsonpCallback ) ?
+			s.jsonpCallback() :
+			s.jsonpCallback;
+		overwritten = window[ callbackName ];
+
+		// Insert callback into url or form data
+		if ( replaceInUrl ) {
+			s.url = url.replace( rjsonp, "$1" + callbackName );
+		} else if ( replaceInData ) {
+			s.data = data.replace( rjsonp, "$1" + callbackName );
+		} else if ( hasCallback ) {
+			s.url += ( rquestion.test( url ) ? "&" : "?" ) + s.jsonp + "=" + callbackName;
+		}
+
+		// Use data converter to retrieve json after script execution
+		s.converters["script json"] = function() {
+			if ( !responseContainer ) {
+				jQuery.error( callbackName + " was not called" );
+			}
+			return responseContainer[ 0 ];
+		};
+
+		// force json dataType
+		s.dataTypes[ 0 ] = "json";
+
+		// Install callback
+		window[ callbackName ] = function() {
+			responseContainer = arguments;
+		};
+
+		// Clean-up function (fires after converters)
+		jqXHR.always(function() {
+			// Restore preexisting value
+			window[ callbackName ] = overwritten;
+
+			// Save back as free
+			if ( s[ callbackName ] ) {
+				// make sure that re-using the options doesn't screw things around
+				s.jsonpCallback = originalSettings.jsonpCallback;
+
+				// save the callback name for future use
+				oldCallbacks.push( callbackName );
+			}
+
+			// Call if it was a function and we have a response
+			if ( responseContainer && jQuery.isFunction( overwritten ) ) {
+				overwritten( responseContainer[ 0 ] );
+			}
+
+			responseContainer = overwritten = undefined;
+		});
+
+		// Delegate to script
+		return "script";
+	}
+});
+// Install script dataType
+jQuery.ajaxSetup({
+	accepts: {
+		script: "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"
+	},
+	contents: {
+		script: /javascript|ecmascript/
+	},
+	converters: {
+		"text script": function( text ) {
+			jQuery.globalEval( text );
+			return text;
+		}
+	}
+});
+
+// Handle cache's special case and global
+jQuery.ajaxPrefilter( "script", function( s ) {
+	if ( s.cache === undefined ) {
+		s.cache = false;
+	}
+	if ( s.crossDomain ) {
+		s.type = "GET";
+		s.global = false;
+	}
+});
+
+// Bind script tag hack transport
+jQuery.ajaxTransport( "script", function(s) {
+
+	// This transport only deals with cross domain requests
+	if ( s.crossDomain ) {
+
+		var script,
+			head = document.head || document.getElementsByTagName( "head" )[0] || document.documentElement;
+
+		return {
+
+			send: function( _, callback ) {
+
+				script = document.createElement( "script" );
+
+				script.async = "async";
+
+				if ( s.scriptCharset ) {
+					script.charset = s.scriptCharset;
+				}
+
+				script.src = s.url;
+
+				// Attach handlers for all browsers
+				script.onload = script.onreadystatechange = function( _, isAbort ) {
+
+					if ( isAbort || !script.readyState || /loaded|complete/.test( script.readyState ) ) {
+
+						// Handle memory leak in IE
+						script.onload = script.onreadystatechange = null;
+
+						// Remove the script
+						if ( head && script.parentNode ) {
+							head.removeChild( script );
+						}
+
+						// Dereference the script
+						script = undefined;
+
+						// Callback if not abort
+						if ( !isAbort ) {
+							callback( 200, "success" );
+						}
+					}
+				};
+				// Use insertBefore instead of appendChild  to circumvent an IE6 bug.
+				// This arises when a base node is used (#2709 and #4378).
+				head.insertBefore( script, head.firstChild );
+			},
+
+			abort: function() {
+				if ( script ) {
+					script.onload( 0, 1 );
+				}
+			}
+		};
+	}
+});
+var xhrCallbacks,
+	// #5280: Internet Explorer will keep connections alive if we don't abort on unload
+	xhrOnUnloadAbort = window.ActiveXObject ? function() {
+		// Abort all pending requests
+		for ( var key in xhrCallbacks ) {
+			xhrCallbacks[ key ]( 0, 1 );
+		}
+	} : false,
+	xhrId = 0;
+
+// Functions to create xhrs
+function createStandardXHR() {
+	try {
+		return new window.XMLHttpRequest();
+	} catch( e ) {}
+}
+
+function createActiveXHR() {
+	try {
+		return new window.ActiveXObject( "Microsoft.XMLHTTP" );
+	} catch( e ) {}
+}
+
+// Create the request object
+// (This is still attached to ajaxSettings for backward compatibility)
+jQuery.ajaxSettings.xhr = window.ActiveXObject ?
+	/* Microsoft failed to properly
+	 * implement the XMLHttpRequest in IE7 (can't request local files),
+	 * so we use the ActiveXObject when it is available
+	 * Additionally XMLHttpRequest can be disabled in IE7/IE8 so
+	 * we need a fallback.
+	 */
+	function() {
+		return !this.isLocal && createStandardXHR() || createActiveXHR();
+	} :
+	// For all other browsers, use the standard XMLHttpRequest object
+	createStandardXHR;
+
+// Determine support properties
+(function( xhr ) {
+	jQuery.extend( jQuery.support, {
+		ajax: !!xhr,
+		cors: !!xhr && ( "withCredentials" in xhr )
+	});
+})( jQuery.ajaxSettings.xhr() );
+
+// Create transport if the browser can provide an xhr
+if ( jQuery.support.ajax ) {
+
+	jQuery.ajaxTransport(function( s ) {
+		// Cross domain only allowed if supported through XMLHttpRequest
+		if ( !s.crossDomain || jQuery.support.cors ) {
+
+			var callback;
+
+			return {
+				send: function( headers, complete ) {
+
+					// Get a new xhr
+					var handle, i,
+						xhr = s.xhr();
+
+					// Open the socket
+					// Passing null username, generates a login popup on Opera (#2865)
+					if ( s.username ) {
+						xhr.open( s.type, s.url, s.async, s.username, s.password );
+					} else {
+						xhr.open( s.type, s.url, s.async );
+					}
+
+					// Apply custom fields if provided
+					if ( s.xhrFields ) {
+						for ( i in s.xhrFields ) {
+							xhr[ i ] = s.xhrFields[ i ];
+						}
+					}
+
+					// Override mime type if needed
+					if ( s.mimeType && xhr.overrideMimeType ) {
+						xhr.overrideMimeType( s.mimeType );
+					}
+
+					// X-Requested-With header
+					// For cross-domain requests, seeing as conditions for a preflight are
+					// akin to a jigsaw puzzle, we simply never set it to be sure.
+					// (it can always be set on a per-request basis or even using ajaxSetup)
+					// For same-domain requests, won't change header if already provided.
+					if ( !s.crossDomain && !headers["X-Requested-With"] ) {
+						headers[ "X-Requested-With" ] = "XMLHttpRequest";
+					}
+
+					// Need an extra try/catch for cross domain requests in Firefox 3
+					try {
+						for ( i in headers ) {
+							xhr.setRequestHeader( i, headers[ i ] );
+						}
+					} catch( _ ) {}
+
+					// Do send the request
+					// This may raise an exception which is actually
+					// handled in jQuery.ajax (so no try/catch here)
+					xhr.send( ( s.hasContent && s.data ) || null );
+
+					// Listener
+					callback = function( _, isAbort ) {
+
+						var status,
+							statusText,
+							responseHeaders,
+							responses,
+							xml;
+
+						// Firefox throws exceptions when accessing properties
+						// of an xhr when a network error occurred
+						// http://helpful.knobs-dials.com/index.php/Component_returned_failure_code:_0x80040111_(NS_ERROR_NOT_AVAILABLE)
+						try {
+
+							// Was never called and is aborted or complete
+							if ( callback && ( isAbort || xhr.readyState === 4 ) ) {
+
+								// Only called once
+								callback = undefined;
+
+								// Do not keep as active anymore
+								if ( handle ) {
+									xhr.onreadystatechange = jQuery.noop;
+									if ( xhrOnUnloadAbort ) {
+										delete xhrCallbacks[ handle ];
+									}
+								}
+
+								// If it's an abort
+								if ( isAbort ) {
+									// Abort it manually if needed
+									if ( xhr.readyState !== 4 ) {
+										xhr.abort();
+									}
+								} else {
+									status = xhr.status;
+									responseHeaders = xhr.getAllResponseHeaders();
+									responses = {};
+									xml = xhr.responseXML;
+
+									// Construct response list
+									if ( xml && xml.documentElement /* #4958 */ ) {
+										responses.xml = xml;
+									}
+
+									// When requesting binary data, IE6-9 will throw an exception
+									// on any attempt to access responseText (#11426)
+									try {
+										responses.text = xhr.responseText;
+									} catch( e ) {
+									}
+
+									// Firefox throws an exception when accessing
+									// statusText for faulty cross-domain requests
+									try {
+										statusText = xhr.statusText;
+									} catch( e ) {
+										// We normalize with Webkit giving an empty statusText
+										statusText = "";
+									}
+
+									// Filter status for non standard behaviors
+
+									// If the request is local and we have data: assume a success
+									// (success with no data won't get notified, that's the best we
+									// can do given current implementations)
+									if ( !status && s.isLocal && !s.crossDomain ) {
+										status = responses.text ? 200 : 404;
+									// IE - #1450: sometimes returns 1223 when it should be 204
+									} else if ( status === 1223 ) {
+										status = 204;
+									}
+								}
+							}
+						} catch( firefoxAccessException ) {
+							if ( !isAbort ) {
+								complete( -1, firefoxAccessException );
+							}
+						}
+
+						// Call complete if needed
+						if ( responses ) {
+							complete( status, statusText, responses, responseHeaders );
+						}
+					};
+
+					if ( !s.async ) {
+						// if we're in sync mode we fire the callback
+						callback();
+					} else if ( xhr.readyState === 4 ) {
+						// (IE6 & IE7) if it's in cache and has been
+						// retrieved directly we need to fire the callback
+						setTimeout( callback, 0 );
+					} else {
+						handle = ++xhrId;
+						if ( xhrOnUnloadAbort ) {
+							// Create the active xhrs callbacks list if needed
+							// and attach the unload handler
+							if ( !xhrCallbacks ) {
+								xhrCallbacks = {};
+								jQuery( window ).unload( xhrOnUnloadAbort );
+							}
+							// Add to list of active xhrs callbacks
+							xhrCallbacks[ handle ] = callback;
+						}
+						xhr.onreadystatechange = callback;
+					}
+				},
+
+				abort: function() {
+					if ( callback ) {
+						callback(0,1);
+					}
+				}
+			};
+		}
+	});
+}
+var fxNow, timerId,
+	rfxtypes = /^(?:toggle|show|hide)$/,
+	rfxnum = new RegExp( "^(?:([-+])=|)(" + core_pnum + ")([a-z%]*)$", "i" ),
+	rrun = /queueHooks$/,
+	animationPrefilters = [ defaultPrefilter ],
+	tweeners = {
+		"*": [function( prop, value ) {
+			var end, unit,
+				tween = this.createTween( prop, value ),
+				parts = rfxnum.exec( value ),
+				target = tween.cur(),
+				start = +target || 0,
+				scale = 1,
+				maxIterations = 20;
+
+			if ( parts ) {
+				end = +parts[2];
+				unit = parts[3] || ( jQuery.cssNumber[ prop ] ? "" : "px" );
+
+				// We need to compute starting value
+				if ( unit !== "px" && start ) {
+					// Iteratively approximate from a nonzero starting point
+					// Prefer the current property, because this process will be trivial if it uses the same units
+					// Fallback to end or a simple constant
+					start = jQuery.css( tween.elem, prop, true ) || end || 1;
+
+					do {
+						// If previous iteration zeroed out, double until we get *something*
+						// Use a string for doubling factor so we don't accidentally see scale as unchanged below
+						scale = scale || ".5";
+
+						// Adjust and apply
+						start = start / scale;
+						jQuery.style( tween.elem, prop, start + unit );
+
+					// Update scale, tolerating zero or NaN from tween.cur()
+					// And breaking the loop if scale is unchanged or perfect, or if we've just had enough
+					} while ( scale !== (scale = tween.cur() / target) && scale !== 1 && --maxIterations );
+				}
+
+				tween.unit = unit;
+				tween.start = start;
+				// If a +=/-= token was provided, we're doing a relative animation
+				tween.end = parts[1] ? start + ( parts[1] + 1 ) * end : end;
+			}
+			return tween;
+		}]
+	};
+
+// Animations created synchronously will run synchronously
+function createFxNow() {
+	setTimeout(function() {
+		fxNow = undefined;
+	}, 0 );
+	return ( fxNow = jQuery.now() );
+}
+
+function createTweens( animation, props ) {
+	jQuery.each( props, function( prop, value ) {
+		var collection = ( tweeners[ prop ] || [] ).concat( tweeners[ "*" ] ),
+			index = 0,
+			length = collection.length;
+		for ( ; index < length; index++ ) {
+			if ( collection[ index ].call( animation, prop, value ) ) {
+
+				// we're done with this property
+				return;
+			}
+		}
+	});
+}
+
+function Animation( elem, properties, options ) {
+	var result,
+		index = 0,
+		tweenerIndex = 0,
+		length = animationPrefilters.length,
+		deferred = jQuery.Deferred().always( function() {
+			// don't match elem in the :animated selector
+			delete tick.elem;
+		}),
+		tick = function() {
+			var currentTime = fxNow || createFxNow(),
+				remaining = Math.max( 0, animation.startTime + animation.duration - currentTime ),
+				// archaic crash bug won't allow us to use 1 - ( 0.5 || 0 ) (#12497)
+				temp = remaining / animation.duration || 0,
+				percent = 1 - temp,
+				index = 0,
+				length = animation.tweens.length;
+
+			for ( ; index < length ; index++ ) {
+				animation.tweens[ index ].run( percent );
+			}
+
+			deferred.notifyWith( elem, [ animation, percent, remaining ]);
+
+			if ( percent < 1 && length ) {
+				return remaining;
+			} else {
+				deferred.resolveWith( elem, [ animation ] );
+				return false;
+			}
+		},
+		animation = deferred.promise({
+			elem: elem,
+			props: jQuery.extend( {}, properties ),
+			opts: jQuery.extend( true, { specialEasing: {} }, options ),
+			originalProperties: properties,
+			originalOptions: options,
+			startTime: fxNow || createFxNow(),
+			duration: options.duration,
+			tweens: [],
+			createTween: function( prop, end, easing ) {
+				var tween = jQuery.Tween( elem, animation.opts, prop, end,
+						animation.opts.specialEasing[ prop ] || animation.opts.easing );
+				animation.tweens.push( tween );
+				return tween;
+			},
+			stop: function( gotoEnd ) {
+				var index = 0,
+					// if we are going to the end, we want to run all the tweens
+					// otherwise we skip this part
+					length = gotoEnd ? animation.tweens.length : 0;
+
+				for ( ; index < length ; index++ ) {
+					animation.tweens[ index ].run( 1 );
+				}
+
+				// resolve when we played the last frame
+				// otherwise, reject
+				if ( gotoEnd ) {
+					deferred.resolveWith( elem, [ animation, gotoEnd ] );
+				} else {
+					deferred.rejectWith( elem, [ animation, gotoEnd ] );
+				}
+				return this;
+			}
+		}),
+		props = animation.props;
+
+	propFilter( props, animation.opts.specialEasing );
+
+	for ( ; index < length ; index++ ) {
+		result = animationPrefilters[ index ].call( animation, elem, props, animation.opts );
+		if ( result ) {
+			return result;
+		}
+	}
+
+	createTweens( animation, props );
+
+	if ( jQuery.isFunction( animation.opts.start ) ) {
+		animation.opts.start.call( elem, animation );
+	}
+
+	jQuery.fx.timer(
+		jQuery.extend( tick, {
+			anim: animation,
+			queue: animation.opts.queue,
+			elem: elem
+		})
+	);
+
+	// attach callbacks from options
+	return animation.progress( animation.opts.progress )
+		.done( animation.opts.done, animation.opts.complete )
+		.fail( animation.opts.fail )
+		.always( animation.opts.always );
+}
+
+function propFilter( props, specialEasing ) {
+	var index, name, easing, value, hooks;
+
+	// camelCase, specialEasing and expand cssHook pass
+	for ( index in props ) {
+		name = jQuery.camelCase( index );
+		easing = specialEasing[ name ];
+		value = props[ index ];
+		if ( jQuery.isArray( value ) ) {
+			easing = value[ 1 ];
+			value = props[ index ] = value[ 0 ];
+		}
+
+		if ( index !== name ) {
+			props[ name ] = value;
+			delete props[ index ];
+		}
+
+		hooks = jQuery.cssHooks[ name ];
+		if ( hooks && "expand" in hooks ) {
+			value = hooks.expand( value );
+			delete props[ name ];
+
+			// not quite $.extend, this wont overwrite keys already present.
+			// also - reusing 'index' from above because we have the correct "name"
+			for ( index in value ) {
+				if ( !( index in props ) ) {
+					props[ index ] = value[ index ];
+					specialEasing[ index ] = easing;
+				}
+			}
+		} else {
+			specialEasing[ name ] = easing;
+		}
+	}
+}
+
+jQuery.Animation = jQuery.extend( Animation, {
+
+	tweener: function( props, callback ) {
+		if ( jQuery.isFunction( props ) ) {
+			callback = props;
+			props = [ "*" ];
+		} else {
+			props = props.split(" ");
+		}
+
+		var prop,
+			index = 0,
+			length = props.length;
+
+		for ( ; index < length ; index++ ) {
+			prop = props[ index ];
+			tweeners[ prop ] = tweeners[ prop ] || [];
+			tweeners[ prop ].unshift( callback );
+		}
+	},
+
+	prefilter: function( callback, prepend ) {
+		if ( prepend ) {
+			animationPrefilters.unshift( callback );
+		} else {
+			animationPrefilters.push( callback );
+		}
+	}
+});
+
+function defaultPrefilter( elem, props, opts ) {
+	var index, prop, value, length, dataShow, toggle, tween, hooks, oldfire,
+		anim = this,
+		style = elem.style,
+		orig = {},
+		handled = [],
+		hidden = elem.nodeType && isHidden( elem );
+
+	// handle queue: false promises
+	if ( !opts.queue ) {
+		hooks = jQuery._queueHooks( elem, "fx" );
+		if ( hooks.unqueued == null ) {
+			hooks.unqueued = 0;
+			oldfire = hooks.empty.fire;
+			hooks.empty.fire = function() {
+				if ( !hooks.unqueued ) {
+					oldfire();
+				}
+			};
+		}
+		hooks.unqueued++;
+
+		anim.always(function() {
+			// doing this makes sure that the complete handler will be called
+			// before this completes
+			anim.always(function() {
+				hooks.unqueued--;
+				if ( !jQuery.queue( elem, "fx" ).length ) {
+					hooks.empty.fire();
+				}
+			});
+		});
+	}
+
+	// height/width overflow pass
+	if ( elem.nodeType === 1 && ( "height" in props || "width" in props ) ) {
+		// Make sure that nothing sneaks out
+		// Record all 3 overflow attributes because IE does not
+		// change the overflow attribute when overflowX and
+		// overflowY are set to the same value
+		opts.overflow = [ style.overflow, style.overflowX, style.overflowY ];
+
+		// Set display property to inline-block for height/width
+		// animations on inline elements that are having width/height animated
+		if ( jQuery.css( elem, "display" ) === "inline" &&
+				jQuery.css( elem, "float" ) === "none" ) {
+
+			// inline-level elements accept inline-block;
+			// block-level elements need to be inline with layout
+			if ( !jQuery.support.inlineBlockNeedsLayout || css_defaultDisplay( elem.nodeName ) === "inline" ) {
+				style.display = "inline-block";
+
+			} else {
+				style.zoom = 1;
+			}
+		}
+	}
+
+	if ( opts.overflow ) {
+		style.overflow = "hidden";
+		if ( !jQuery.support.shrinkWrapBlocks ) {
+			anim.done(function() {
+				style.overflow = opts.overflow[ 0 ];
+				style.overflowX = opts.overflow[ 1 ];
+				style.overflowY = opts.overflow[ 2 ];
+			});
+		}
+	}
+
+
+	// show/hide pass
+	for ( index in props ) {
+		value = props[ index ];
+		if ( rfxtypes.exec( value ) ) {
+			delete props[ index ];
+			toggle = toggle || value === "toggle";
+			if ( value === ( hidden ? "hide" : "show" ) ) {
+				continue;
+			}
+			handled.push( index );
+		}
+	}
+
+	length = handled.length;
+	if ( length ) {
+		dataShow = jQuery._data( elem, "fxshow" ) || jQuery._data( elem, "fxshow", {} );
+		if ( "hidden" in dataShow ) {
+			hidden = dataShow.hidden;
+		}
+
+		// store state if its toggle - enables .stop().toggle() to "reverse"
+		if ( toggle ) {
+			dataShow.hidden = !hidden;
+		}
+		if ( hidden ) {
+			jQuery( elem ).show();
+		} else {
+			anim.done(function() {
+				jQuery( elem ).hide();
+			});
+		}
+		anim.done(function() {
+			var prop;
+			jQuery.removeData( elem, "fxshow", true );
+			for ( prop in orig ) {
+				jQuery.style( elem, prop, orig[ prop ] );
+			}
+		});
+		for ( index = 0 ; index < length ; index++ ) {
+			prop = handled[ index ];
+			tween = anim.createTween( prop, hidden ? dataShow[ prop ] : 0 );
+			orig[ prop ] = dataShow[ prop ] || jQuery.style( elem, prop );
+
+			if ( !( prop in dataShow ) ) {
+				dataShow[ prop ] = tween.start;
+				if ( hidden ) {
+					tween.end = tween.start;
+					tween.start = prop === "width" || prop === "height" ? 1 : 0;
+				}
+			}
+		}
+	}
+}
+
+function Tween( elem, options, prop, end, easing ) {
+	return new Tween.prototype.init( elem, options, prop, end, easing );
+}
+jQuery.Tween = Tween;
+
+Tween.prototype = {
+	constructor: Tween,
+	init: function( elem, options, prop, end, easing, unit ) {
+		this.elem = elem;
+		this.prop = prop;
+		this.easing = easing || "swing";
+		this.options = options;
+		this.start = this.now = this.cur();
+		this.end = end;
+		this.unit = unit || ( jQuery.cssNumber[ prop ] ? "" : "px" );
+	},
+	cur: function() {
+		var hooks = Tween.propHooks[ this.prop ];
+
+		return hooks && hooks.get ?
+			hooks.get( this ) :
+			Tween.propHooks._default.get( this );
+	},
+	run: function( percent ) {
+		var eased,
+			hooks = Tween.propHooks[ this.prop ];
+
+		if ( this.options.duration ) {
+			this.pos = eased = jQuery.easing[ this.easing ](
+				percent, this.options.duration * percent, 0, 1, this.options.duration
+			);
+		} else {
+			this.pos = eased = percent;
+		}
+		this.now = ( this.end - this.start ) * eased + this.start;
+
+		if ( this.options.step ) {
+			this.options.step.call( this.elem, this.now, this );
+		}
+
+		if ( hooks && hooks.set ) {
+			hooks.set( this );
+		} else {
+			Tween.propHooks._default.set( this );
+		}
+		return this;
+	}
+};
+
+Tween.prototype.init.prototype = Tween.prototype;
+
+Tween.propHooks = {
+	_default: {
+		get: function( tween ) {
+			var result;
+
+			if ( tween.elem[ tween.prop ] != null &&
+				(!tween.elem.style || tween.elem.style[ tween.prop ] == null) ) {
+				return tween.elem[ tween.prop ];
+			}
+
+			// passing any value as a 4th parameter to .css will automatically
+			// attempt a parseFloat and fallback to a string if the parse fails
+			// so, simple values such as "10px" are parsed to Float.
+			// complex values such as "rotate(1rad)" are returned as is.
+			result = jQuery.css( tween.elem, tween.prop, false, "" );
+			// Empty strings, null, undefined and "auto" are converted to 0.
+			return !result || result === "auto" ? 0 : result;
+		},
+		set: function( tween ) {
+			// use step hook for back compat - use cssHook if its there - use .style if its
+			// available and use plain properties where available
+			if ( jQuery.fx.step[ tween.prop ] ) {
+				jQuery.fx.step[ tween.prop ]( tween );
+			} else if ( tween.elem.style && ( tween.elem.style[ jQuery.cssProps[ tween.prop ] ] != null || jQuery.cssHooks[ tween.prop ] ) ) {
+				jQuery.style( tween.elem, tween.prop, tween.now + tween.unit );
+			} else {
+				tween.elem[ tween.prop ] = tween.now;
+			}
+		}
+	}
+};
+
+// Remove in 2.0 - this supports IE8's panic based approach
+// to setting things on disconnected nodes
+
+Tween.propHooks.scrollTop = Tween.propHooks.scrollLeft = {
+	set: function( tween ) {
+		if ( tween.elem.nodeType && tween.elem.parentNode ) {
+			tween.elem[ tween.prop ] = tween.now;
+		}
+	}
+};
+
+jQuery.each([ "toggle", "show", "hide" ], function( i, name ) {
+	var cssFn = jQuery.fn[ name ];
+	jQuery.fn[ name ] = function( speed, easing, callback ) {
+		return speed == null || typeof speed === "boolean" ||
+			// special check for .toggle( handler, handler, ... )
+			( !i && jQuery.isFunction( speed ) && jQuery.isFunction( easing ) ) ?
+			cssFn.apply( this, arguments ) :
+			this.animate( genFx( name, true ), speed, easing, callback );
+	};
+});
+
+jQuery.fn.extend({
+	fadeTo: function( speed, to, easing, callback ) {
+
+		// show any hidden elements after setting opacity to 0
+		return this.filter( isHidden ).css( "opacity", 0 ).show()
+
+			// animate to the value specified
+			.end().animate({ opacity: to }, speed, easing, callback );
+	},
+	animate: function( prop, speed, easing, callback ) {
+		var empty = jQuery.isEmptyObject( prop ),
+			optall = jQuery.speed( speed, easing, callback ),
+			doAnimation = function() {
+				// Operate on a copy of prop so per-property easing won't be lost
+				var anim = Animation( this, jQuery.extend( {}, prop ), optall );
+
+				// Empty animations resolve immediately
+				if ( empty ) {
+					anim.stop( true );
+				}
+			};
+
+		return empty || optall.queue === false ?
+			this.each( doAnimation ) :
+			this.queue( optall.queue, doAnimation );
+	},
+	stop: function( type, clearQueue, gotoEnd ) {
+		var stopQueue = function( hooks ) {
+			var stop = hooks.stop;
+			delete hooks.stop;
+			stop( gotoEnd );
+		};
+
+		if ( typeof type !== "string" ) {
+			gotoEnd = clearQueue;
+			clearQueue = type;
+			type = undefined;
+		}
+		if ( clearQueue && type !== false ) {
+			this.queue( type || "fx", [] );
+		}
+
+		return this.each(function() {
+			var dequeue = true,
+				index = type != null && type + "queueHooks",
+				timers = jQuery.timers,
+				data = jQuery._data( this );
+
+			if ( index ) {
+				if ( data[ index ] && data[ index ].stop ) {
+					stopQueue( data[ index ] );
+				}
+			} else {
+				for ( index in data ) {
+					if ( data[ index ] && data[ index ].stop && rrun.test( index ) ) {
+						stopQueue( data[ index ] );
+					}
+				}
+			}
+
+			for ( index = timers.length; index--; ) {
+				if ( timers[ index ].elem === this && (type == null || timers[ index ].queue === type) ) {
+					timers[ index ].anim.stop( gotoEnd );
+					dequeue = false;
+					timers.splice( index, 1 );
+				}
+			}
+
+			// start the next in the queue if the last step wasn't forced
+			// timers currently will call their complete callbacks, which will dequeue
+			// but only if they were gotoEnd
+			if ( dequeue || !gotoEnd ) {
+				jQuery.dequeue( this, type );
+			}
+		});
+	}
+});
+
+// Generate parameters to create a standard animation
+function genFx( type, includeWidth ) {
+	var which,
+		attrs = { height: type },
+		i = 0;
+
+	// if we include width, step value is 1 to do all cssExpand values,
+	// if we don't include width, step value is 2 to skip over Left and Right
+	includeWidth = includeWidth? 1 : 0;
+	for( ; i < 4 ; i += 2 - includeWidth ) {
+		which = cssExpand[ i ];
+		attrs[ "margin" + which ] = attrs[ "padding" + which ] = type;
+	}
+
+	if ( includeWidth ) {
+		attrs.opacity = attrs.width = type;
+	}
+
+	return attrs;
+}
+
+// Generate shortcuts for custom animations
+jQuery.each({
+	slideDown: genFx("show"),
+	slideUp: genFx("hide"),
+	slideToggle: genFx("toggle"),
+	fadeIn: { opacity: "show" },
+	fadeOut: { opacity: "hide" },
+	fadeToggle: { opacity: "toggle" }
+}, function( name, props ) {
+	jQuery.fn[ name ] = function( speed, easing, callback ) {
+		return this.animate( props, speed, easing, callback );
+	};
+});
+
+jQuery.speed = function( speed, easing, fn ) {
+	var opt = speed && typeof speed === "object" ? jQuery.extend( {}, speed ) : {
+		complete: fn || !fn && easing ||
+			jQuery.isFunction( speed ) && speed,
+		duration: speed,
+		easing: fn && easing || easing && !jQuery.isFunction( easing ) && easing
+	};
+
+	opt.duration = jQuery.fx.off ? 0 : typeof opt.duration === "number" ? opt.duration :
+		opt.duration in jQuery.fx.speeds ? jQuery.fx.speeds[ opt.duration ] : jQuery.fx.speeds._default;
+
+	// normalize opt.queue - true/undefined/null -> "fx"
+	if ( opt.queue == null || opt.queue === true ) {
+		opt.queue = "fx";
+	}
+
+	// Queueing
+	opt.old = opt.complete;
+
+	opt.complete = function() {
+		if ( jQuery.isFunction( opt.old ) ) {
+			opt.old.call( this );
+		}
+
+		if ( opt.queue ) {
+			jQuery.dequeue( this, opt.queue );
+		}
+	};
+
+	return opt;
+};
+
+jQuery.easing = {
+	linear: function( p ) {
+		return p;
+	},
+	swing: function( p ) {
+		return 0.5 - Math.cos( p*Math.PI ) / 2;
+	}
+};
+
+jQuery.timers = [];
+jQuery.fx = Tween.prototype.init;
+jQuery.fx.tick = function() {
+	var timer,
+		timers = jQuery.timers,
+		i = 0;
+
+	fxNow = jQuery.now();
+
+	for ( ; i < timers.length; i++ ) {
+		timer = timers[ i ];
+		// Checks the timer has not already been removed
+		if ( !timer() && timers[ i ] === timer ) {
+			timers.splice( i--, 1 );
+		}
+	}
+
+	if ( !timers.length ) {
+		jQuery.fx.stop();
+	}
+	fxNow = undefined;
+};
+
+jQuery.fx.timer = function( timer ) {
+	if ( timer() && jQuery.timers.push( timer ) && !timerId ) {
+		timerId = setInterval( jQuery.fx.tick, jQuery.fx.interval );
+	}
+};
+
+jQuery.fx.interval = 13;
+
+jQuery.fx.stop = function() {
+	clearInterval( timerId );
+	timerId = null;
+};
+
+jQuery.fx.speeds = {
+	slow: 600,
+	fast: 200,
+	// Default speed
+	_default: 400
+};
+
+// Back Compat <1.8 extension point
+jQuery.fx.step = {};
+
+if ( jQuery.expr && jQuery.expr.filters ) {
+	jQuery.expr.filters.animated = function( elem ) {
+		return jQuery.grep(jQuery.timers, function( fn ) {
+			return elem === fn.elem;
+		}).length;
+	};
+}
+var rroot = /^(?:body|html)$/i;
+
+jQuery.fn.offset = function( options ) {
+	if ( arguments.length ) {
+		return options === undefined ?
+			this :
+			this.each(function( i ) {
+				jQuery.offset.setOffset( this, options, i );
+			});
+	}
+
+	var docElem, body, win, clientTop, clientLeft, scrollTop, scrollLeft,
+		box = { top: 0, left: 0 },
+		elem = this[ 0 ],
+		doc = elem && elem.ownerDocument;
+
+	if ( !doc ) {
+		return;
+	}
+
+	if ( (body = doc.body) === elem ) {
+		return jQuery.offset.bodyOffset( elem );
+	}
+
+	docElem = doc.documentElement;
+
+	// Make sure it's not a disconnected DOM node
+	if ( !jQuery.contains( docElem, elem ) ) {
+		return box;
+	}
+
+	// If we don't have gBCR, just use 0,0 rather than error
+	// BlackBerry 5, iOS 3 (original iPhone)
+	if ( typeof elem.getBoundingClientRect !== "undefined" ) {
+		box = elem.getBoundingClientRect();
+	}
+	win = getWindow( doc );
+	clientTop  = docElem.clientTop  || body.clientTop  || 0;
+	clientLeft = docElem.clientLeft || body.clientLeft || 0;
+	scrollTop  = win.pageYOffset || docElem.scrollTop;
+	scrollLeft = win.pageXOffset || docElem.scrollLeft;
+	return {
+		top: box.top  + scrollTop  - clientTop,
+		left: box.left + scrollLeft - clientLeft
+	};
+};
+
+jQuery.offset = {
+
+	bodyOffset: function( body ) {
+		var top = body.offsetTop,
+			left = body.offsetLeft;
+
+		if ( jQuery.support.doesNotIncludeMarginInBodyOffset ) {
+			top  += parseFloat( jQuery.css(body, "marginTop") ) || 0;
+			left += parseFloat( jQuery.css(body, "marginLeft") ) || 0;
+		}
+
+		return { top: top, left: left };
+	},
+
+	setOffset: function( elem, options, i ) {
+		var position = jQuery.css( elem, "position" );
+
+		// set position first, in-case top/left are set even on static elem
+		if ( position === "static" ) {
+			elem.style.position = "relative";
+		}
+
+		var curElem = jQuery( elem ),
+			curOffset = curElem.offset(),
+			curCSSTop = jQuery.css( elem, "top" ),
+			curCSSLeft = jQuery.css( elem, "left" ),
+			calculatePosition = ( position === "absolute" || position === "fixed" ) && jQuery.inArray("auto", [curCSSTop, curCSSLeft]) > -1,
+			props = {}, curPosition = {}, curTop, curLeft;
+
+		// need to be able to calculate position if either top or left is auto and position is either absolute or fixed
+		if ( calculatePosition ) {
+			curPosition = curElem.position();
+			curTop = curPosition.top;
+			curLeft = curPosition.left;
+		} else {
+			curTop = parseFloat( curCSSTop ) || 0;
+			curLeft = parseFloat( curCSSLeft ) || 0;
+		}
+
+		if ( jQuery.isFunction( options ) ) {
+			options = options.call( elem, i, curOffset );
+		}
+
+		if ( options.top != null ) {
+			props.top = ( options.top - curOffset.top ) + curTop;
+		}
+		if ( options.left != null ) {
+			props.left = ( options.left - curOffset.left ) + curLeft;
+		}
+
+		if ( "using" in options ) {
+			options.using.call( elem, props );
+		} else {
+			curElem.css( props );
+		}
+	}
+};
+
+
+jQuery.fn.extend({
+
+	position: function() {
+		if ( !this[0] ) {
+			return;
+		}
+
+		var elem = this[0],
+
+		// Get *real* offsetParent
+		offsetParent = this.offsetParent(),
+
+		// Get correct offsets
+		offset       = this.offset(),
+		parentOffset = rroot.test(offsetParent[0].nodeName) ? { top: 0, left: 0 } : offsetParent.offset();
+
+		// Subtract element margins
+		// note: when an element has margin: auto the offsetLeft and marginLeft
+		// are the same in Safari causing offset.left to incorrectly be 0
+		offset.top  -= parseFloat( jQuery.css(elem, "marginTop") ) || 0;
+		offset.left -= parseFloat( jQuery.css(elem, "marginLeft") ) || 0;
+
+		// Add offsetParent borders
+		parentOffset.top  += parseFloat( jQuery.css(offsetParent[0], "borderTopWidth") ) || 0;
+		parentOffset.left += parseFloat( jQuery.css(offsetParent[0], "borderLeftWidth") ) || 0;
+
+		// Subtract the two offsets
+		return {
+			top:  offset.top  - parentOffset.top,
+			left: offset.left - parentOffset.left
+		};
+	},
+
+	offsetParent: function() {
+		return this.map(function() {
+			var offsetParent = this.offsetParent || document.body;
+			while ( offsetParent && (!rroot.test(offsetParent.nodeName) && jQuery.css(offsetParent, "position") === "static") ) {
+				offsetParent = offsetParent.offsetParent;
+			}
+			return offsetParent || document.body;
+		});
+	}
+});
+
+
+// Create scrollLeft and scrollTop methods
+jQuery.each( {scrollLeft: "pageXOffset", scrollTop: "pageYOffset"}, function( method, prop ) {
+	var top = /Y/.test( prop );
+
+	jQuery.fn[ method ] = function( val ) {
+		return jQuery.access( this, function( elem, method, val ) {
+			var win = getWindow( elem );
+
+			if ( val === undefined ) {
+				return win ? (prop in win) ? win[ prop ] :
+					win.document.documentElement[ method ] :
+					elem[ method ];
+			}
+
+			if ( win ) {
+				win.scrollTo(
+					!top ? val : jQuery( win ).scrollLeft(),
+					 top ? val : jQuery( win ).scrollTop()
+				);
+
+			} else {
+				elem[ method ] = val;
+			}
+		}, method, val, arguments.length, null );
+	};
+});
+
+function getWindow( elem ) {
+	return jQuery.isWindow( elem ) ?
+		elem :
+		elem.nodeType === 9 ?
+			elem.defaultView || elem.parentWindow :
+			false;
+}
+// Create innerHeight, innerWidth, height, width, outerHeight and outerWidth methods
+jQuery.each( { Height: "height", Width: "width" }, function( name, type ) {
+	jQuery.each( { padding: "inner" + name, content: type, "": "outer" + name }, function( defaultExtra, funcName ) {
+		// margin is only for outerHeight, outerWidth
+		jQuery.fn[ funcName ] = function( margin, value ) {
+			var chainable = arguments.length && ( defaultExtra || typeof margin !== "boolean" ),
+				extra = defaultExtra || ( margin === true || value === true ? "margin" : "border" );
+
+			return jQuery.access( this, function( elem, type, value ) {
+				var doc;
+
+				if ( jQuery.isWindow( elem ) ) {
+					// As of 5/8/2012 this will yield incorrect results for Mobile Safari, but there
+					// isn't a whole lot we can do. See pull request at this URL for discussion:
+					// https://github.com/jquery/jquery/pull/764
+					return elem.document.documentElement[ "client" + name ];
+				}
+
+				// Get document width or height
+				if ( elem.nodeType === 9 ) {
+					doc = elem.documentElement;
+
+					// Either scroll[Width/Height] or offset[Width/Height] or client[Width/Height], whichever is greatest
+					// unfortunately, this causes bug #3838 in IE6/8 only, but there is currently no good, small way to fix it.
+					return Math.max(
+						elem.body[ "scroll" + name ], doc[ "scroll" + name ],
+						elem.body[ "offset" + name ], doc[ "offset" + name ],
+						doc[ "client" + name ]
+					);
+				}
+
+				return value === undefined ?
+					// Get width or height on the element, requesting but not forcing parseFloat
+					jQuery.css( elem, type, value, extra ) :
+
+					// Set width or height on the element
+					jQuery.style( elem, type, value, extra );
+			}, type, chainable ? margin : undefined, chainable, null );
+		};
+	});
+});
+// Expose jQuery to the global object
+window.jQuery = window.$ = jQuery;
+
+// Expose jQuery as an AMD module, but only for AMD loaders that
+// understand the issues with loading multiple versions of jQuery
+// in a page that all might call define(). The loader will indicate
+// they have special allowances for multiple jQuery versions by
+// specifying define.amd.jQuery = true. Register as a named module,
+// since jQuery can be concatenated with other files that may use define,
+// but not use a proper concatenation script that understands anonymous
+// AMD modules. A named AMD is safest and most robust way to register.
+// Lowercase jquery is used because AMD module names are derived from
+// file names, and jQuery is normally delivered in a lowercase file name.
+// Do this after creating the global so that if an AMD module wants to call
+// noConflict to hide this version of jQuery, it will work.
+if ( typeof define === "function" && define.amd && define.amd.jQuery ) {
+	define( "jquery", [], function () { return jQuery; } );
+}
+
+})( window );
diff --git a/qa/workunits/erasure-code/plot.js b/qa/workunits/erasure-code/plot.js
new file mode 100644
index 00000000..bd2bba5b
--- /dev/null
+++ b/qa/workunits/erasure-code/plot.js
@@ -0,0 +1,82 @@
+$(function() {
+    encode = [];
+    if (typeof encode_vandermonde_isa != 'undefined') {
+        encode.push({
+	    data: encode_vandermonde_isa,
+            label: "ISA, Vandermonde",
+	    points: { show: true },
+	    lines: { show: true },
+	});
+    }
+    if (typeof encode_vandermonde_jerasure != 'undefined') {
+        encode.push({
+	    data: encode_vandermonde_jerasure,
+            label: "Jerasure Generic, Vandermonde",
+	    points: { show: true },
+	    lines: { show: true },
+	});
+    }
+    if (typeof encode_cauchy_isa != 'undefined') {
+        encode.push({
+	    data: encode_cauchy_isa,
+            label: "ISA, Cauchy",
+	    points: { show: true },
+	    lines: { show: true },
+	});
+    }
+    if (typeof encode_cauchy_jerasure != 'undefined') {
+        encode.push({
+	    data: encode_cauchy_jerasure,
+            label: "Jerasure, Cauchy",
+	    points: { show: true },
+	    lines: { show: true },
+	});
+    }
+    $.plot("#encode", encode, {
+	xaxis: {
+	    mode: "categories",
+	    tickLength: 0
+	},
+    });
+
+    decode = [];
+    if (typeof decode_vandermonde_isa != 'undefined') {
+        decode.push({
+	    data: decode_vandermonde_isa,
+            label: "ISA, Vandermonde",
+	    points: { show: true },
+	    lines: { show: true },
+	});
+    }
+    if (typeof decode_vandermonde_jerasure != 'undefined') {
+        decode.push({
+	    data: decode_vandermonde_jerasure,
+            label: "Jerasure Generic, Vandermonde",
+	    points: { show: true },
+	    lines: { show: true },
+	});
+    }
+    if (typeof decode_cauchy_isa != 'undefined') {
+        decode.push({
+	    data: decode_cauchy_isa,
+            label: "ISA, Cauchy",
+	    points: { show: true },
+	    lines: { show: true },
+	});
+    }
+    if (typeof decode_cauchy_jerasure != 'undefined') {
+        decode.push({
+	    data: decode_cauchy_jerasure,
+            label: "Jerasure, Cauchy",
+	    points: { show: true },
+	    lines: { show: true },
+	});
+    }
+    $.plot("#decode", decode, {
+	xaxis: {
+	    mode: "categories",
+	    tickLength: 0
+	},
+    });
+
+});
diff --git a/qa/workunits/false.sh b/qa/workunits/false.sh
new file mode 100644
index 00000000..8a961b32
--- /dev/null
+++ b/qa/workunits/false.sh
@@ -0,0 +1,3 @@
+#!/bin/sh -ex
+
+false
\ No newline at end of file
diff --git a/qa/workunits/fs/.gitignore b/qa/workunits/fs/.gitignore
new file mode 100644
index 00000000..f7f7a061
--- /dev/null
+++ b/qa/workunits/fs/.gitignore
@@ -0,0 +1 @@
+test_o_trunc
diff --git a/qa/workunits/fs/Makefile b/qa/workunits/fs/Makefile
new file mode 100644
index 00000000..c9934254
--- /dev/null
+++ b/qa/workunits/fs/Makefile
@@ -0,0 +1,11 @@
+CFLAGS = -Wall -Wextra -D_GNU_SOURCE
+
+TARGETS = test_o_trunc
+
+.c:
+	$(CC) $(CFLAGS) $@.c -o $@
+
+all:	$(TARGETS)
+
+clean:
+	rm $(TARGETS)
diff --git a/qa/workunits/fs/misc/acl.sh b/qa/workunits/fs/misc/acl.sh
new file mode 100755
index 00000000..198b0567
--- /dev/null
+++ b/qa/workunits/fs/misc/acl.sh
@@ -0,0 +1,50 @@
+#!/bin/sh -x
+
+set -e
+mkdir -p testdir
+cd testdir
+
+set +e
+setfacl -d -m u:nobody:rw .
+if test $? != 0; then
+	echo "Filesystem does not support ACL"
+	exit 0
+fi
+
+expect_failure() {
+	if "$@"; then return 1; else return 0; fi
+}
+
+set -e
+c=0
+while [ $c -lt 100  ]
+do
+	c=`expr $c + 1`
+	# inherited ACL from parent directory's default ACL
+	mkdir d1
+	c1=`getfacl d1 | grep -c "nobody:rw"`
+	echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null
+	c2=`getfacl d1 | grep -c "nobody:rw"`
+	rmdir d1
+	if [ $c1 -ne 2 ] || [ $c2 -ne 2 ]
+	then
+		echo "ERROR: incorrect ACLs"
+		exit 1
+	fi
+done
+
+mkdir d1
+
+# The ACL xattr only contains ACL header. ACL should be removed
+# in this case.
+setfattr -n system.posix_acl_access -v 0x02000000 d1
+setfattr -n system.posix_acl_default -v 0x02000000 .
+
+expect_failure getfattr -n system.posix_acl_access d1
+expect_failure getfattr -n system.posix_acl_default .
+
+
+rmdir d1
+cd ..
+rmdir testdir
+echo OK
diff --git a/qa/workunits/fs/misc/chmod.sh b/qa/workunits/fs/misc/chmod.sh
new file mode 100755
index 00000000..de66776f
--- /dev/null
+++ b/qa/workunits/fs/misc/chmod.sh
@@ -0,0 +1,60 @@
+#!/bin/sh -x
+
+set -e
+
+check_perms() {
+
+	file=$1
+	r=$(ls -la ${file})
+	if test $? != 0; then
+		echo "ERROR: File listing/stat failed"
+		exit 1
+	fi
+
+	perms=$2
+	if test "${perms}" != $(echo ${r} | awk '{print $1}') && \
+           test "${perms}." != $(echo ${r} | awk '{print $1}') && \
+           test "${perms}+" != $(echo ${r} | awk '{print $1}'); then
+		echo "ERROR: Permissions should be ${perms}"
+		exit 1
+	fi
+}
+
+file=test_chmod.$$
+
+echo "foo" > ${file}
+if test $? != 0; then
+	echo "ERROR: Failed to create file ${file}"
+	exit 1
+fi
+
+chmod 400 ${file}
+if test $? != 0; then
+	echo "ERROR: Failed to change mode of ${file}"
+	exit 1
+fi
+
+check_perms ${file} "-r--------"
+
+set +e
+echo "bar" >> ${file}
+if test $? = 0; then
+	echo "ERROR: Write to read-only file should Fail"
+	exit 1
+fi
+
+set -e
+chmod 600 ${file}
+echo "bar" >> ${file}
+if test $? != 0; then
+	echo "ERROR: Write to writeable file failed"
+	exit 1
+fi
+
+check_perms ${file} "-rw-------"
+
+echo "foo" >> ${file}
+if test $? != 0; then
+	echo "ERROR: Failed to write to file"
+	exit 1
+fi
diff --git a/qa/workunits/fs/misc/direct_io.py b/qa/workunits/fs/misc/direct_io.py
new file mode 100755
index 00000000..b5c42265
--- /dev/null
+++ b/qa/workunits/fs/misc/direct_io.py
@@ -0,0 +1,50 @@
+#!/usr/bin/python
+
+import json
+import mmap
+import os
+import subprocess
+
+
+def get_data_pool():
+    cmd = ['ceph', 'fs', 'ls', '--format=json-pretty']
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    out = proc.communicate()[0]
+    return json.loads(out)[0]['data_pools'][0]
+
+
+def main():
+    fd = os.open("testfile", os.O_RDWR | os.O_CREAT | os.O_TRUNC | os.O_DIRECT, 0o644)
+
+    ino = os.fstat(fd).st_ino
+    obj_name = "{ino:x}.00000000".format(ino=ino)
+    pool_name = get_data_pool()
+
+    buf = mmap.mmap(-1, 1)
+    buf.write('1')
+    os.write(fd, buf)
+
+    proc = subprocess.Popen(['rados', '-p', pool_name, 'get', obj_name, 'tmpfile'])
+    proc.wait()
+
+    with open('tmpfile', 'r') as tmpf:
+        out = tmpf.read()
+        if out != '1':
+            raise RuntimeError("data were not written to object store directly")
+
+    with open('tmpfile', 'w') as tmpf:
+        tmpf.write('2')
+
+    proc = subprocess.Popen(['rados', '-p', pool_name, 'put', obj_name, 'tmpfile'])
+    proc.wait()
+
+    os.lseek(fd, 0, os.SEEK_SET)
+    out = os.read(fd, 1)
+    if out != '2':
+        raise RuntimeError("data were not directly read from object store")
+
+    os.close(fd)
+    print('ok')
+
+
+main()
diff --git a/qa/workunits/fs/misc/dirfrag.sh b/qa/workunits/fs/misc/dirfrag.sh
new file mode 100755
index 00000000..eea0ec3b
--- /dev/null
+++ b/qa/workunits/fs/misc/dirfrag.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -e
+
+DEPTH=5
+COUNT=10000
+
+kill_jobs() {
+  jobs -p | xargs kill
+}
+trap kill_jobs INT
+
+create_files() {
+  for i in `seq 1 $COUNT`
+  do
+    touch file$i
+  done
+}
+
+delete_files() {
+  for i in `ls -f`
+  do
+    if [[ ${i}a = file*a ]]
+    then
+      rm -f $i
+    fi
+  done
+}
+
+rm -rf testdir
+mkdir testdir
+cd testdir
+
+echo "creating folder hierarchy"
+for i in `seq 1 $DEPTH`; do
+  mkdir dir$i
+  cd dir$i
+  create_files &
+done
+wait
+
+echo "created hierarchy, now cleaning up"
+
+for i in `seq 1 $DEPTH`; do
+  delete_files &
+  cd ..
+done
+wait
+
+echo "cleaned up hierarchy"
+cd ..
+rm -rf testdir
diff --git a/qa/workunits/fs/misc/filelock_deadlock.py b/qa/workunits/fs/misc/filelock_deadlock.py
new file mode 100755
index 00000000..3ebc9777
--- /dev/null
+++ b/qa/workunits/fs/misc/filelock_deadlock.py
@@ -0,0 +1,72 @@
+#!/usr/bin/python
+
+import errno
+import fcntl
+import os
+import signal
+import struct
+import time
+
+
+def handler(signum, frame):
+    pass
+
+
+def lock_two(f1, f2):
+    lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 10, 0, 0)
+    fcntl.fcntl(f1, fcntl.F_SETLKW, lockdata)
+    time.sleep(10)
+
+    # don't wait forever
+    signal.signal(signal.SIGALRM, handler)
+    signal.alarm(10)
+    exitcode = 0
+    try:
+        fcntl.fcntl(f2, fcntl.F_SETLKW, lockdata)
+    except IOError as e:
+        if e.errno == errno.EDEADLK:
+            exitcode = 1
+        elif e.errno == errno.EINTR:
+            exitcode = 2
+        else:
+            exitcode = 3
+    os._exit(exitcode)
+
+
+def main():
+    pid1 = os.fork()
+    if pid1 == 0:
+        f1 = open("testfile1", 'w')
+        f2 = open("testfile2", 'w')
+        lock_two(f1, f2)
+
+    pid2 = os.fork()
+    if pid2 == 0:
+        f1 = open("testfile2", 'w')
+        f2 = open("testfile3", 'w')
+        lock_two(f1, f2)
+
+    pid3 = os.fork()
+    if pid3 == 0:
+        f1 = open("testfile3", 'w')
+        f2 = open("testfile1", 'w')
+        lock_two(f1, f2)
+
+    deadlk_count = 0
+    i = 0
+    while i < 3:
+        pid, status = os.wait()
+        exitcode = status >> 8
+        if exitcode == 1:
+            deadlk_count += 1
+        elif exitcode != 0:
+            raise RuntimeError("unexpect exit code of child")
+        i += 1
+
+    if deadlk_count != 1:
+        raise RuntimeError("unexpect count of EDEADLK")
+
+    print('ok')
+
+
+main()
diff --git a/qa/workunits/fs/misc/filelock_interrupt.py b/qa/workunits/fs/misc/filelock_interrupt.py
new file mode 100755
index 00000000..7b5b3e7d
--- /dev/null
+++ b/qa/workunits/fs/misc/filelock_interrupt.py
@@ -0,0 +1,87 @@
+#!/usr/bin/python
+
+import errno
+import fcntl
+import signal
+import struct
+
+"""
+introduced by Linux 3.15
+"""
+fcntl.F_OFD_GETLK = 36
+fcntl.F_OFD_SETLK = 37
+fcntl.F_OFD_SETLKW = 38
+
+
+def handler(signum, frame):
+    pass
+
+
+def main():
+    f1 = open("testfile", 'w')
+    f2 = open("testfile", 'w')
+
+    fcntl.flock(f1, fcntl.LOCK_SH | fcntl.LOCK_NB)
+
+    """
+    is flock interruptible?
+    """
+    signal.signal(signal.SIGALRM, handler)
+    signal.alarm(5)
+    try:
+        fcntl.flock(f2, fcntl.LOCK_EX)
+    except IOError as e:
+        if e.errno != errno.EINTR:
+            raise
+    else:
+        raise RuntimeError("expect flock to block")
+
+    fcntl.flock(f1, fcntl.LOCK_UN)
+
+    lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 10, 0, 0)
+    try:
+        fcntl.fcntl(f1, fcntl.F_OFD_SETLK, lockdata)
+    except IOError as e:
+        if e.errno != errno.EINVAL:
+            raise
+        else:
+            print('kernel does not support fcntl.F_OFD_SETLK')
+            return
+
+    lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 10, 10, 0, 0)
+    fcntl.fcntl(f2, fcntl.F_OFD_SETLK, lockdata)
+
+    """
+    is posix lock interruptible?
+    """
+    signal.signal(signal.SIGALRM, handler)
+    signal.alarm(5)
+    try:
+        lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
+        fcntl.fcntl(f2, fcntl.F_OFD_SETLKW, lockdata)
+    except IOError as e:
+        if e.errno != errno.EINTR:
+            raise
+    else:
+        raise RuntimeError("expect posix lock to block")
+
+    """
+    file handler 2 should still hold lock on 10~10
+    """
+    try:
+        lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 10, 10, 0, 0)
+        fcntl.fcntl(f1, fcntl.F_OFD_SETLK, lockdata)
+    except IOError as e:
+        if e.errno == errno.EAGAIN:
+            pass
+    else:
+        raise RuntimeError("expect file handler 2 to hold lock on 10~10")
+
+    lockdata = struct.pack('hhllhh', fcntl.F_UNLCK, 0, 0, 0, 0, 0)
+    fcntl.fcntl(f1, fcntl.F_OFD_SETLK, lockdata)
+    fcntl.fcntl(f2, fcntl.F_OFD_SETLK, lockdata)
+
+    print('ok')
+
+
+main()
diff --git a/qa/workunits/fs/misc/i_complete_vs_rename.sh b/qa/workunits/fs/misc/i_complete_vs_rename.sh
new file mode 100755
index 00000000..a9b98271
--- /dev/null
+++ b/qa/workunits/fs/misc/i_complete_vs_rename.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -e
+
+mkdir x
+cd x
+touch a
+touch b
+touch c
+touch d
+ls
+chmod 777 .
+stat e || true
+touch f
+touch g
+
+# over existing file
+echo attempting rename over existing file...
+touch ../xx
+mv ../xx f
+ls | grep f || false
+echo rename over existing file is okay
+
+# over negative dentry
+echo attempting rename over negative dentry...
+touch ../xx
+mv ../xx e
+ls | grep e || false
+echo rename over negative dentry is ok
+
+echo OK
diff --git a/qa/workunits/fs/misc/layout_vxattrs.sh b/qa/workunits/fs/misc/layout_vxattrs.sh
new file mode 100755
index 00000000..81133627
--- /dev/null
+++ b/qa/workunits/fs/misc/layout_vxattrs.sh
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# detect data pool
+datapool=
+dir=.
+while true ; do
+    echo $dir
+    datapool=$(getfattr -n ceph.dir.layout.pool $dir --only-values) && break
+    dir=$dir/..
+done
+
+# file
+rm -f file file2
+touch file file2
+
+getfattr -n ceph.file.layout file
+getfattr -n ceph.file.layout file | grep -q object_size=
+getfattr -n ceph.file.layout file | grep -q stripe_count=
+getfattr -n ceph.file.layout file | grep -q stripe_unit=
+getfattr -n ceph.file.layout file | grep -q pool=
+getfattr -n ceph.file.layout.pool file
+getfattr -n ceph.file.layout.pool_namespace file
+getfattr -n ceph.file.layout.stripe_unit file
+getfattr -n ceph.file.layout.stripe_count file
+getfattr -n ceph.file.layout.object_size file
+
+getfattr -n ceph.file.layout.bogus file   2>&1 | grep -q 'No such attribute'
+getfattr -n ceph.dir.layout file    2>&1 | grep -q 'No such attribute'
+
+setfattr -n ceph.file.layout.stripe_unit -v 1048576 file2
+setfattr -n ceph.file.layout.stripe_count -v 8 file2
+setfattr -n ceph.file.layout.object_size -v 10485760 file2
+
+setfattr -n ceph.file.layout.pool -v $datapool file2
+getfattr -n ceph.file.layout.pool file2 | grep -q $datapool
+setfattr -n ceph.file.layout.pool_namespace -v foons file2
+getfattr -n ceph.file.layout.pool_namespace file2 | grep -q foons
+setfattr -x ceph.file.layout.pool_namespace file2
+getfattr -n ceph.file.layout.pool_namespace file2 | grep -q -v foons
+
+getfattr -n ceph.file.layout.stripe_unit file2 | grep -q 1048576
+getfattr -n ceph.file.layout.stripe_count file2 | grep -q 8
+getfattr -n ceph.file.layout.object_size file2 | grep -q 10485760
+
+setfattr -n ceph.file.layout -v "stripe_unit=4194304 stripe_count=16 object_size=41943040 pool=$datapool pool_namespace=foons" file2
+getfattr -n ceph.file.layout.stripe_unit file2 | grep -q 4194304
+getfattr -n ceph.file.layout.stripe_count file2 | grep -q 16
+getfattr -n ceph.file.layout.object_size file2 | grep -q 41943040
+getfattr -n ceph.file.layout.pool file2 | grep -q $datapool
+getfattr -n ceph.file.layout.pool_namespace file2 | grep -q foons
+
+setfattr -n ceph.file.layout -v "stripe_unit=1048576" file2
+getfattr -n ceph.file.layout.stripe_unit file2 | grep -q 1048576
+getfattr -n ceph.file.layout.stripe_count file2 | grep -q 16
+getfattr -n ceph.file.layout.object_size file2 | grep -q 41943040
+getfattr -n ceph.file.layout.pool file2 | grep -q $datapool
+getfattr -n ceph.file.layout.pool_namespace file2 | grep -q foons
+
+setfattr -n ceph.file.layout -v "stripe_unit=2097152 stripe_count=4 object_size=2097152 pool=$datapool pool_namespace=barns" file2
+getfattr -n ceph.file.layout.stripe_unit file2 | grep -q 2097152
+getfattr -n ceph.file.layout.stripe_count file2 | grep -q 4
+getfattr -n ceph.file.layout.object_size file2 | grep -q 2097152
+getfattr -n ceph.file.layout.pool file2 | grep -q $datapool
+getfattr -n ceph.file.layout.pool_namespace file2 | grep -q barns
+
+# dir
+rm -f dir/file || true
+rmdir dir || true
+mkdir -p dir
+
+getfattr -d -m - dir | grep -q ceph.dir.layout       && exit 1 || true
+getfattr -d -m - dir | grep -q ceph.file.layout      && exit 1 || true
+getfattr -n ceph.dir.layout dir                      && exit 1 || true
+
+setfattr -n ceph.dir.layout.stripe_unit -v 1048576 dir
+setfattr -n ceph.dir.layout.stripe_count -v 8 dir
+setfattr -n ceph.dir.layout.object_size -v 10485760 dir
+setfattr -n ceph.dir.layout.pool -v $datapool dir
+setfattr -n ceph.dir.layout.pool_namespace -v dirns dir
+
+getfattr -n ceph.dir.layout dir
+getfattr -n ceph.dir.layout dir | grep -q object_size=10485760
+getfattr -n ceph.dir.layout dir | grep -q stripe_count=8
+getfattr -n ceph.dir.layout dir | grep -q stripe_unit=1048576
+getfattr -n ceph.dir.layout dir | grep -q pool=$datapool
+getfattr -n ceph.dir.layout dir | grep -q pool_namespace=dirns
+getfattr -n ceph.dir.layout.pool dir | grep -q $datapool
+getfattr -n ceph.dir.layout.stripe_unit dir | grep -q 1048576
+getfattr -n ceph.dir.layout.stripe_count dir | grep -q 8
+getfattr -n ceph.dir.layout.object_size dir | grep -q 10485760
+getfattr -n ceph.dir.layout.pool_namespace dir | grep -q dirns
+
+
+setfattr -n ceph.file.layout -v "stripe_count=16" file2
+getfattr -n ceph.file.layout.stripe_count file2 | grep -q 16
+setfattr -n ceph.file.layout -v "object_size=10485760 stripe_count=8 stripe_unit=1048576 pool=$datapool pool_namespace=dirns" file2
+getfattr -n ceph.file.layout.stripe_count file2 | grep -q 8
+
+touch dir/file
+getfattr -n ceph.file.layout.pool dir/file | grep -q $datapool
+getfattr -n ceph.file.layout.stripe_unit dir/file | grep -q 1048576
+getfattr -n ceph.file.layout.stripe_count dir/file | grep -q 8
+getfattr -n ceph.file.layout.object_size dir/file | grep -q 10485760
+getfattr -n ceph.file.layout.pool_namespace dir/file | grep -q dirns
+
+setfattr -x ceph.dir.layout.pool_namespace dir
+getfattr -n ceph.dir.layout dir | grep -q -v pool_namespace=dirns
+
+setfattr -x ceph.dir.layout dir
+getfattr -n ceph.dir.layout dir     2>&1 | grep -q 'No such attribute'
+
+echo OK
+
diff --git a/qa/workunits/fs/misc/mkpool_layout_vxattrs.sh b/qa/workunits/fs/misc/mkpool_layout_vxattrs.sh
new file mode 100755
index 00000000..6b2fecbc
--- /dev/null
+++ b/qa/workunits/fs/misc/mkpool_layout_vxattrs.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+
+touch foo.$$
+ceph osd pool create foo.$$ 8
+ceph fs add_data_pool cephfs foo.$$
+setfattr -n ceph.file.layout.pool -v foo.$$ foo.$$
+
+# cleanup
+rm foo.$$
+ceph fs rm_data_pool cephfs foo.$$
+ceph osd pool rm foo.$$ foo.$$ --yes-i-really-really-mean-it
+
+echo OK
diff --git a/qa/workunits/fs/misc/multiple_rsync.sh b/qa/workunits/fs/misc/multiple_rsync.sh
new file mode 100755
index 00000000..4397c1e7
--- /dev/null
+++ b/qa/workunits/fs/misc/multiple_rsync.sh
@@ -0,0 +1,25 @@
+#!/bin/sh -ex
+
+
+# Populate with some arbitrary files from the local system.  Take
+# a copy to protect against false fails from system updates during test.
+export PAYLOAD=/tmp/multiple_rsync_payload.$$
+sudo cp -r /usr/lib/ $PAYLOAD
+
+set -e
+
+sudo rsync -av $PAYLOAD payload.1
+sudo rsync -av $PAYLOAD payload.2
+
+# this shouldn't transfer any additional files
+echo we should get 4 here if no additional files are transferred
+sudo rsync -auv $PAYLOAD payload.1 | tee /tmp/$$
+hexdump -C /tmp/$$
+wc -l /tmp/$$ | grep 4
+sudo rsync -auv $PAYLOAD payload.2 | tee /tmp/$$
+hexdump -C /tmp/$$
+wc -l /tmp/$$ | grep 4
+echo OK
+
+rm /tmp/$$
+sudo rm -rf $PAYLOAD
diff --git a/qa/workunits/fs/misc/rstats.sh b/qa/workunits/fs/misc/rstats.sh
new file mode 100755
index 00000000..4c32edb2
--- /dev/null
+++ b/qa/workunits/fs/misc/rstats.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+set -x
+
+timeout=30
+old_value=""
+new_value=""
+
+wait_until_changed() {
+	name=$1
+	wait=0
+	while [ $wait -lt $timeout ]; do
+		new_value=`getfattr --only-value -n ceph.dir.$name .`
+		[ $new_value == $old_value ] || return 0
+		sleep 1
+		wait=$(($wait + 1))
+	done
+	return 1
+}
+
+check_rctime() {
+	old_sec=$(echo $old_value | cut -d. -f1)
+	old_nsec=$(echo $old_value | cut -d. -f2)
+	new_sec=$(echo $new_value | cut -d. -f1)
+	new_nsec=$(echo $new_value | cut -d. -f2)
+	[ "$old_sec" -lt "$new_sec" ] && return 0
+	[ "$old_sec" -gt "$new_sec" ] && return 1
+	[ "$old_nsec" -lt "$new_nsec" ] && return 0
+	return 1
+}
+
+# sync(3) does not make ceph-fuse flush dirty caps, because fuse kernel module
+# does not notify ceph-fuse about it. Use fsync(3) instead.
+fsync_path() {
+	cmd="import os; fd=os.open(\"$1\", os.O_RDONLY); os.fsync(fd); os.close(fd)"
+	python -c "$cmd"
+}
+
+set -e
+
+mkdir -p rstats_testdir/d1/d2
+cd rstats_testdir
+
+# rfiles
+old_value=`getfattr --only-value -n ceph.dir.rfiles .`
+[ $old_value == 0 ] || false
+touch d1/d2/f1
+wait_until_changed rfiles
+[ $new_value == $(($old_value + 1)) ] || false
+
+# rsubdirs
+old_value=`getfattr --only-value -n ceph.dir.rsubdirs .`
+[ $old_value == 3 ] || false
+mkdir d1/d2/d3
+wait_until_changed rsubdirs
+[ $new_value == $(($old_value + 1)) ] || false
+
+# rbytes
+old_value=`getfattr --only-value -n ceph.dir.rbytes .`
+[ $old_value == 0 ] || false
+echo hello > d1/d2/f2
+fsync_path d1/d2/f2
+wait_until_changed rbytes
+[ $new_value == $(($old_value + 6)) ] || false
+
+#rctime
+old_value=`getfattr --only-value -n ceph.dir.rctime .`
+touch d1/d2/d3 # touch existing file
+fsync_path d1/d2/d3
+wait_until_changed rctime
+check_rctime
+
+old_value=`getfattr --only-value -n ceph.dir.rctime .`
+touch d1/d2/f3 # create new file
+wait_until_changed rctime
+check_rctime
+
+cd ..
+rm -rf rstats_testdir
+echo OK
diff --git a/qa/workunits/fs/misc/subvolume.sh b/qa/workunits/fs/misc/subvolume.sh
new file mode 100755
index 00000000..75716a6c
--- /dev/null
+++ b/qa/workunits/fs/misc/subvolume.sh
@@ -0,0 +1,63 @@
+#!/bin/sh -x
+
+expect_failure() {
+	if "$@"; then return 1; else return 0; fi
+}
+
+set -e
+
+mkdir group
+mkdir group/subvol1
+
+setfattr -n ceph.dir.subvolume -v 1 group/subvol1
+
+# rename subvolume
+mv group/subvol1 group/subvol2
+
+# move file out of the subvolume
+touch group/subvol2/file1
+expect_failure python3 -c "import os; os.rename('group/subvol2/file1', 'group/file1')"
+# move file into the subvolume
+touch group/file2
+expect_failure python3 -c "import os; os.rename('group/file2', 'group/subvol2/file2')"
+
+# create hardlink within subvolume
+ln group/subvol2/file1 group/subvol2/file1_
+
+# create hardlink out of subvolume
+expect_failure ln group/subvol2/file1  group/file1_
+expect_failure ln group/file2 group/subvol1/file2_
+
+# create snapshot at subvolume root
+mkdir group/subvol2/.snap/s1
+
+# create snapshot at descendent dir of subvolume
+mkdir group/subvol2/dir
+expect_failure mkdir group/subvol2/dir/.snap/s2
+
+mkdir group/subvol3
+setfattr -n ceph.dir.subvolume -v 1 group/subvol3
+
+# move file across subvolumes
+expect_failure python3 -c "import os; os.rename('group/subvol2/file1', 'group/subvol3/file1')"
+
+# create hardlink across subvolumes
+expect_failure ln group/subvol2/file1 group/subvol3/file1
+
+# create subvolume inside existing subvolume
+expect_failure setfattr -n ceph.dir.subvolume -v 1 group/subvol2/dir
+
+# clear subvolume flag
+setfattr -n ceph.dir.subvolume -v 0 group/subvol2
+mkdir group/subvol2/dir/.snap/s2
+
+# parent subvolume override child subvolume
+setfattr -n ceph.dir.subvolume -v 1 group/subvol2/dir
+setfattr -n ceph.dir.subvolume -v 1 group/subvol2
+expect_failure mkdir group/subvol2/dir/.snap/s3
+
+rmdir group/subvol2/.snap/s1
+rmdir group/subvol2/dir/.snap/s2
+rm -rf group
+
+echo OK
diff --git a/qa/workunits/fs/misc/trivial_sync.sh b/qa/workunits/fs/misc/trivial_sync.sh
new file mode 100755
index 00000000..7c8c4e2b
--- /dev/null
+++ b/qa/workunits/fs/misc/trivial_sync.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+
+mkdir foo
+echo foo > bar
+sync
diff --git a/qa/workunits/fs/misc/xattrs.sh b/qa/workunits/fs/misc/xattrs.sh
new file mode 100755
index 00000000..fcd94d22
--- /dev/null
+++ b/qa/workunits/fs/misc/xattrs.sh
@@ -0,0 +1,14 @@
+#!/bin/sh -x
+
+set -e
+
+touch file
+
+setfattr -n user.foo -v foo file
+setfattr -n user.bar -v bar file
+setfattr -n user.empty file
+getfattr -d file | grep foo
+getfattr -d file | grep bar
+getfattr -d file | grep empty
+
+echo OK.
diff --git a/qa/workunits/fs/multiclient_sync_read_eof.py b/qa/workunits/fs/multiclient_sync_read_eof.py
new file mode 100755
index 00000000..1d5bb650
--- /dev/null
+++ b/qa/workunits/fs/multiclient_sync_read_eof.py
@@ -0,0 +1,42 @@
+#!/usr/bin/python
+
+import argparse
+import os
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('mnt1')
+    parser.add_argument('mnt2')
+    parser.add_argument('fn')
+    args = parser.parse_args()
+
+    open(os.path.join(args.mnt1, args.fn), 'w')
+    f1 = open(os.path.join(args.mnt1, args.fn), 'r+')
+    f2 = open(os.path.join(args.mnt2, args.fn), 'r+')
+
+    f1.write('foo')
+    f1.flush()
+    a = f2.read(3)
+    print('got "%s"' % a)
+    assert a == 'foo'
+    f2.write('bar')
+    f2.flush()
+    a = f1.read(3)
+    print('got "%s"' % a)
+    assert a == 'bar'
+
+    ## test short reads
+    f1.write('short')
+    f1.flush()
+    a = f2.read(100)
+    print('got "%s"' % a)
+    assert a == 'short'
+    f2.write('longer')
+    f2.flush()
+    a = f1.read(1000)
+    print('got "%s"' % a)
+    assert a == 'longer'
+
+    print('ok')
+
+main()
diff --git a/qa/workunits/fs/norstats/kernel_untar_tar.sh b/qa/workunits/fs/norstats/kernel_untar_tar.sh
new file mode 100755
index 00000000..6a175dcd
--- /dev/null
+++ b/qa/workunits/fs/norstats/kernel_untar_tar.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# check if there is file changed while being archived
+
+set -e
+
+KERNEL=linux-4.0.5
+
+wget -q http://download.ceph.com/qa/$KERNEL.tar.xz
+
+mkdir untar_tar
+cd untar_tar
+
+tar Jxvf ../$KERNEL.tar.xz $KERNEL/Documentation/
+tar cf doc.tar $KERNEL
+
+tar xf doc.tar
+sync
+tar c $KERNEL >/dev/null
+
+rm -rf $KERNEL
+
+tar xf doc.tar
+sync
+tar c $KERNEL >/dev/null
+
+echo Ok
diff --git a/qa/workunits/fs/quota/quota.sh b/qa/workunits/fs/quota/quota.sh
new file mode 100755
index 00000000..1315be6d
--- /dev/null
+++ b/qa/workunits/fs/quota/quota.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+set -ex
+
+function expect_false()
+{
+	set -x
+	if "$@"; then return 1; else return 0; fi
+}
+
+function write_file()
+{
+	set +x
+	for ((i=1;i<=$2;i++))
+	do
+		dd if=/dev/zero of=$1 bs=1M count=1 conv=notrunc oflag=append 2>/dev/null >/dev/null
+		if [ $? != 0 ]; then
+			echo Try to write $(($i * 1048576))
+			set -x
+			return 1
+		fi
+		sleep 0.05
+	done
+	set -x
+	return 0
+}
+
+mkdir quota-test
+cd quota-test
+
+# bytes
+setfattr . -n ceph.quota.max_bytes -v 100000000  # 100m
+expect_false write_file big 1000     # 1g
+expect_false write_file second 10
+setfattr . -n ceph.quota.max_bytes -v 0
+dd if=/dev/zero of=third bs=1M count=10
+dd if=/dev/zero of=big2 bs=1M count=100
+
+
+rm -rf *
+
+# files
+setfattr . -n ceph.quota.max_files -v 5
+mkdir ok
+touch ok/1
+touch ok/2
+touch 3
+expect_false touch shouldbefail     #  5 files will include the "."
+expect_false touch ok/shouldbefail  #  5 files will include the "."
+setfattr . -n ceph.quota.max_files -v 0
+touch shouldbecreated
+touch shouldbecreated2
+
+
+rm -rf *
+
+# mix
+mkdir bytes bytes/files
+
+setfattr bytes -n ceph.quota.max_bytes -v 10000000   #10m
+setfattr bytes/files -n ceph.quota.max_files -v 5
+dd if=/dev/zero of=bytes/files/1 bs=1M count=4
+dd if=/dev/zero of=bytes/files/2 bs=1M count=4
+expect_false write_file bytes/files/3 1000
+expect_false write_file bytes/files/4 1000
+expect_false write_file bytes/files/5 1000
+stat --printf="%n %s\n" bytes/files/1 #4M
+stat --printf="%n %s\n" bytes/files/2 #4M
+stat --printf="%n %s\n" bytes/files/3 #bigger than 2M
+stat --printf="%n %s\n" bytes/files/4 #should be zero
+expect_false stat bytes/files/5       #shouldn't be exist
+
+
+
+
+rm -rf *
+
+#mv
+mkdir files limit
+truncate files/file -s 10G
+setfattr limit -n ceph.quota.max_bytes -v 1000000 #1m
+expect_false mv files limit/
+
+
+
+rm -rf *
+
+#limit by ancestor
+
+mkdir -p ancestor/p1/p2/parent/p3
+setfattr ancestor -n ceph.quota.max_bytes -v 1000000
+setfattr ancestor/p1/p2/parent -n ceph.quota.max_bytes -v 1000000000 #1g
+expect_false write_file ancestor/p1/p2/parent/p3/file1 900 #900m
+stat --printf="%n %s\n" ancestor/p1/p2/parent/p3/file1
+
+
+#get/set attribute
+
+setfattr -n ceph.quota.max_bytes -v 0 .
+setfattr -n ceph.quota.max_bytes -v 1 .
+setfattr -n ceph.quota.max_bytes -v 9223372036854775807 .
+expect_false setfattr -n ceph.quota.max_bytes -v 9223372036854775808 .
+expect_false setfattr -n ceph.quota.max_bytes -v -1 .
+expect_false setfattr -n ceph.quota.max_bytes -v -9223372036854775808 .
+expect_false setfattr -n ceph.quota.max_bytes -v -9223372036854775809 .
+
+setfattr -n ceph.quota.max_files -v 0 .
+setfattr -n ceph.quota.max_files -v 1 .
+setfattr -n ceph.quota.max_files -v 9223372036854775807 .
+expect_false setfattr -n ceph.quota.max_files -v 9223372036854775808 .
+expect_false setfattr -n ceph.quota.max_files -v -1 .
+expect_false setfattr -n ceph.quota.max_files -v -9223372036854775808 .
+expect_false setfattr -n ceph.quota.max_files -v -9223372036854775809 .
+
+setfattr -n ceph.quota -v "max_bytes=0 max_files=0" .
+setfattr -n ceph.quota -v "max_bytes=1 max_files=0" .
+setfattr -n ceph.quota -v "max_bytes=0 max_files=1" .
+setfattr -n ceph.quota -v "max_bytes=1 max_files=1" .
+expect_false setfattr -n ceph.quota -v "max_bytes=-1 max_files=0" .
+expect_false setfattr -n ceph.quota -v "max_bytes=0 max_files=-1" .
+expect_false setfattr -n ceph.quota -v "max_bytes=-1 max_files=-1" .
+
+#addme
+
+cd ..
+rm -rf quota-test
+
+echo OK
diff --git a/qa/workunits/fs/snap-hierarchy.sh b/qa/workunits/fs/snap-hierarchy.sh
new file mode 100755
index 00000000..67f0e014
--- /dev/null
+++ b/qa/workunits/fs/snap-hierarchy.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -ex
+
+if [ -d "$1" ]; then
+  mkdir -p -- "$1" && cd "$1"
+fi
+
+[ "$VERIFY" != verify ] && mkdir 1
+[ "$VERIFY" != verify ] && mkdir 1/.snap/first
+stat 1/.snap/first
+[ "$VERIFY" != verify ] && mkdir 1/2
+stat 1/.snap/first/2 && exit 1
+[ "$VERIFY" != verify ] && mkdir 1/2/.snap/second
+stat 1/2/.snap/second
+[ "$VERIFY" != verify ] && touch 1/foo
+stat 1/.snap/first/foo && exit 1
+[ "$VERIFY" != verify ] && mkdir 1/.snap/third
+stat 1/.snap/third/foo || exit 1
+[ "$VERIFY" != verify ] && mkdir 1/2/3
+[ "$VERIFY" != verify ] && mkdir 1/2/.snap/fourth
+stat 1/2/.snap/fourth/3
+
+exit 0
diff --git a/qa/workunits/fs/snaps/snap-rm-diff.sh b/qa/workunits/fs/snaps/snap-rm-diff.sh
new file mode 100755
index 00000000..63f64287
--- /dev/null
+++ b/qa/workunits/fs/snaps/snap-rm-diff.sh
@@ -0,0 +1,11 @@
+#!/bin/sh -ex
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+wget -q http://download.ceph.com/qa/linux-2.6.33.tar.bz2
+mkdir foo
+cp linux* foo
+mkdir foo/.snap/barsnap
+rm foo/linux*
+diff -q foo/.snap/barsnap/linux* linux* && echo "passed: files are identical"
+rmdir foo/.snap/barsnap
+echo OK
diff --git a/qa/workunits/fs/snaps/snaptest-0.sh b/qa/workunits/fs/snaps/snaptest-0.sh
new file mode 100755
index 00000000..791caf9e
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-0.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -x
+
+expect_failure() {
+	if "$@"; then return 1; else return 0; fi
+}
+set -e
+
+ceph fs set cephfs allow_new_snaps false
+expect_failure mkdir .snap/foo
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+echo asdf > foo
+mkdir .snap/foo
+grep asdf .snap/foo/foo
+rmdir .snap/foo
+
+echo asdf > bar
+mkdir .snap/bar
+rm bar
+grep asdf .snap/bar/bar
+rmdir .snap/bar
+rm foo
+
+ceph fs set cephfs allow_new_snaps false
+expect_failure mkdir .snap/baz
+
+echo OK
diff --git a/qa/workunits/fs/snaps/snaptest-1.sh b/qa/workunits/fs/snaps/snaptest-1.sh
new file mode 100755
index 00000000..476531fc
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-1.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -ex
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+echo 1 > file1
+echo 2 > file2
+echo 3 > file3
+[ -e file4 ] && rm file4
+mkdir .snap/snap1
+echo 4 > file4
+now=`ls`
+then=`ls .snap/snap1`
+rmdir .snap/snap1
+if [ "$now" = "$then" ]; then
+    echo live and snap contents are identical?
+    false
+fi
+
+# do it again
+echo 1 > file1
+echo 2 > file2
+echo 3 > file3
+mkdir .snap/snap1
+echo 4 > file4
+rmdir .snap/snap1
+
+rm file?
+
+echo OK
diff --git a/qa/workunits/fs/snaps/snaptest-2.sh b/qa/workunits/fs/snaps/snaptest-2.sh
new file mode 100755
index 00000000..6ded7b66
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-2.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+echo "Create dir 100 to 199 ..."
+for i in $(seq 100 199); do
+	echo "    create dir $i"
+	mkdir "$i"
+	for y in $(seq 10 20); do
+		echo "This is a test file before any snapshot was taken." >"$i/$y"
+	done
+done
+
+echo "Take first snapshot .snap/test1"
+mkdir .snap/test1
+
+echo "Create dir 200 to 299 ..."
+for i in $(seq 200 299); do
+	echo "    create dir $i"
+        mkdir $i
+        for y in $(seq 20 29); do
+                echo "This is a test file. Created after .snap/test1" >"$i/$y"
+        done
+done
+
+echo "Create a snapshot in every first level dir ..."
+for dir in $(ls); do
+	echo "    create $dir/.snap/snap-subdir-test"
+	mkdir "$dir/.snap/snap-subdir-test"
+	for y in $(seq 30 39); do
+		echo "        create $dir/$y file after the snapshot"
+                echo "This is a test file. Created after $dir/.snap/snap-subdir-test" >"$dir/$y"
+        done
+done
+
+echo "Take second snapshot .snap/test2"
+mkdir .snap/test2
+
+echo "Copy content of .snap/test1 to copyofsnap1 ..."
+mkdir copyofsnap1
+cp -Rv .snap/test1 copyofsnap1/
+
+
+echo "Take third snapshot .snap/test3"
+mkdir .snap/test3
+
+echo "Delete the snapshots..."
+
+find ./ -type d -print | \
+        xargs -I% -n1 find %/.snap -mindepth 1 -maxdepth 1 \
+                         \( ! -name "_*" \) -print 2>/dev/null
+
+find ./ -type d -print | \
+	xargs -I% -n1 find %/.snap -mindepth 1 -maxdepth 1 \
+                         \( ! -name "_*" \) -print 2>/dev/null | \
+	xargs -n1 rmdir
+
+echo "Delete all the files and directories ..."
+rm -Rfv ./*
+
+echo OK
diff --git a/qa/workunits/fs/snaps/snaptest-authwb.sh b/qa/workunits/fs/snaps/snaptest-authwb.sh
new file mode 100755
index 00000000..2c53e2a6
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-authwb.sh
@@ -0,0 +1,14 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+touch foo
+chmod +x foo
+mkdir .snap/s
+find .snap/s/foo -executable | grep foo
+rmdir .snap/s
+rm foo
+
+echo OK
\ No newline at end of file
diff --git a/qa/workunits/fs/snaps/snaptest-capwb.sh b/qa/workunits/fs/snaps/snaptest-capwb.sh
new file mode 100755
index 00000000..f36d38ab
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-capwb.sh
@@ -0,0 +1,35 @@
+#!/bin/sh -x
+
+set -e
+
+mkdir foo
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+# make sure mds handles it when the client does not send flushsnap
+echo x > foo/x
+sync
+mkdir foo/.snap/ss
+ln foo/x foo/xx
+cat foo/.snap/ss/x
+rmdir foo/.snap/ss
+
+#
+echo a > foo/a
+echo b > foo/b
+mkdir foo/.snap/s
+r=`cat foo/.snap/s/a`
+[ -z "$r" ] && echo "a appears empty in snapshot" && false
+
+ln foo/b foo/b2
+cat foo/.snap/s/b
+
+echo "this used to hang:"
+echo more >> foo/b2
+echo "oh, it didn't hang! good job."
+cat foo/b
+rmdir foo/.snap/s
+
+rm -r foo
+
+echo OK
\ No newline at end of file
diff --git a/qa/workunits/fs/snaps/snaptest-dir-rename.sh b/qa/workunits/fs/snaps/snaptest-dir-rename.sh
new file mode 100755
index 00000000..85b929a2
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-dir-rename.sh
@@ -0,0 +1,19 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+#
+# make sure we keep an existing dn's seq
+#
+
+mkdir a
+mkdir .snap/bar
+mkdir a/.snap/foo
+rmdir a/.snap/foo
+rmdir a
+stat .snap/bar/a
+rmdir .snap/bar
+
+echo OK
\ No newline at end of file
diff --git a/qa/workunits/fs/snaps/snaptest-double-null.sh b/qa/workunits/fs/snaps/snaptest-double-null.sh
new file mode 100755
index 00000000..49a1b271
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-double-null.sh
@@ -0,0 +1,25 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+# multiple intervening snapshots with no modifications, and thus no
+# snapflush client_caps messages.  make sure the mds can handle this.
+
+for f in `seq 1 20` ; do
+
+mkdir a
+cat > a/foo &
+mkdir a/.snap/one
+mkdir a/.snap/two
+chmod 777 a/foo
+sync   # this might crash the mds
+ps
+rmdir a/.snap/*
+rm a/foo
+rmdir a
+
+done
+
+echo OK
diff --git a/qa/workunits/fs/snaps/snaptest-estale.sh b/qa/workunits/fs/snaps/snaptest-estale.sh
new file mode 100755
index 00000000..e005b9a8
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-estale.sh
@@ -0,0 +1,15 @@
+#!/bin/sh -x
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+mkdir .snap/foo
+
+echo "We want ENOENT, not ESTALE, here."
+for f in `seq 1 100`
+do
+    stat .snap/foo/$f 2>&1 | grep 'No such file'
+done
+
+rmdir .snap/foo
+
+echo "OK"
diff --git a/qa/workunits/fs/snaps/snaptest-git-ceph.sh b/qa/workunits/fs/snaps/snaptest-git-ceph.sh
new file mode 100755
index 00000000..50b854a5
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-git-ceph.sh
@@ -0,0 +1,35 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+git clone git://git.ceph.com/ceph.git
+cd ceph
+
+versions=`seq 1 21`
+
+for v in $versions
+do
+    ver="v0.$v"
+    echo $ver
+    git reset --hard $ver
+    mkdir .snap/$ver
+done
+
+for v in $versions
+do
+    ver="v0.$v"
+    echo checking $ver
+    cd .snap/$ver
+    git diff --exit-code
+    cd ../..
+done
+
+for v in $versions
+do
+    ver="v0.$v"
+    rmdir .snap/$ver
+done
+
+echo OK
diff --git a/qa/workunits/fs/snaps/snaptest-hardlink.sh b/qa/workunits/fs/snaps/snaptest-hardlink.sh
new file mode 100755
index 00000000..9848a019
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-hardlink.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+mkdir 1 2
+echo asdf >1/file1
+echo asdf >1/file2
+
+ln 1/file1 2/file1
+ln 1/file2 2/file2
+
+mkdir 2/.snap/s1
+
+echo qwer >1/file1
+grep asdf 2/.snap/s1/file1
+
+rm -f 1/file2
+grep asdf 2/.snap/s1/file2
+rm -f 2/file2
+grep asdf 2/.snap/s1/file2
+
+rmdir 2/.snap/s1
+rm -rf 1 2
+
+echo OK
diff --git a/qa/workunits/fs/snaps/snaptest-intodir.sh b/qa/workunits/fs/snaps/snaptest-intodir.sh
new file mode 100755
index 00000000..94af4422
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-intodir.sh
@@ -0,0 +1,24 @@
+#!/bin/sh -ex
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+# this tests fix for #1399
+mkdir foo
+mkdir foo/.snap/one
+touch bar
+mv bar foo
+sync
+# should not crash :)
+
+mkdir baz
+mkdir baz/.snap/two
+mv baz foo
+sync
+# should not crash :)
+
+# clean up.
+rmdir foo/baz/.snap/two
+rmdir foo/.snap/one
+rm -r foo
+
+echo OK
\ No newline at end of file
diff --git a/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh b/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh
new file mode 100755
index 00000000..56ceaa8a
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh
@@ -0,0 +1,44 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+echo asdf > a
+mkdir .snap/1
+chmod 777 a
+mkdir .snap/2
+echo qwer > a
+mkdir .snap/3
+chmod 666 a
+mkdir .snap/4
+echo zxcv > a
+mkdir .snap/5
+
+ls -al .snap/?/a
+
+grep asdf .snap/1/a
+stat .snap/1/a | grep 'Size: 5'
+
+grep asdf .snap/2/a
+stat .snap/2/a | grep 'Size: 5'
+stat .snap/2/a | grep -- '-rwxrwxrwx'
+
+grep qwer .snap/3/a
+stat .snap/3/a | grep 'Size: 5'
+stat .snap/3/a | grep -- '-rwxrwxrwx'
+
+grep qwer .snap/4/a
+stat .snap/4/a | grep 'Size: 5'
+stat .snap/4/a | grep -- '-rw-rw-rw-'
+
+grep zxcv .snap/5/a
+stat .snap/5/a | grep 'Size: 5'
+stat .snap/5/a | grep -- '-rw-rw-rw-'
+
+rmdir .snap/[12345]
+
+echo "OK"
+
+
+
diff --git a/qa/workunits/fs/snaps/snaptest-parents.sh b/qa/workunits/fs/snaps/snaptest-parents.sh
new file mode 100755
index 00000000..a66a977f
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-parents.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+echo "making directory tree and files"
+mkdir -p 1/a/b/c/
+echo "i'm file1" > 1/a/file1
+echo "i'm file2" > 1/a/b/file2
+echo "i'm file3" > 1/a/b/c/file3
+echo "snapshotting"
+mkdir 1/.snap/foosnap1
+mkdir 2
+echo "moving tree"
+mv 1/a 2
+echo "checking snapshot contains tree..."
+dir1=`find 1/.snap/foosnap1 | wc -w`
+dir2=`find 2/ | wc -w`
+#diff $dir1 $dir2 && echo "Success!"
+test $dir1==$dir2 && echo "Success!"
+echo "adding folder and file to tree..."
+mkdir 2/a/b/c/d
+echo "i'm file 4!" > 2/a/b/c/d/file4
+echo "snapshotting tree 2"
+mkdir 2/.snap/barsnap2
+echo "comparing snapshots"
+dir1=`find 1/.snap/foosnap1/ -maxdepth 2 | wc -w`
+dir2=`find 2/.snap/barsnap2/ -maxdepth 2 | wc -w`
+#diff $dir1 $dir2 && echo "Success!"
+test $dir1==$dir2 && echo "Success!"
+echo "moving subtree to first folder"
+mv 2/a/b/c 1
+echo "comparing snapshots and new tree"
+dir1=`find 1/ | wc -w`
+dir2=`find 2/.snap/barsnap2/a/b/c | wc -w`
+#diff $dir1 $dir2 && echo "Success!"
+test $dir1==$dir2 && echo "Success!"
+rmdir 1/.snap/*
+rmdir 2/.snap/*
+echo "OK"
diff --git a/qa/workunits/fs/snaps/snaptest-realm-split.sh b/qa/workunits/fs/snaps/snaptest-realm-split.sh
new file mode 100755
index 00000000..3f01fd54
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-realm-split.sh
@@ -0,0 +1,33 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+mkdir -p 1/a
+exec 3<> 1/a/file1
+
+echo -n a >&3
+
+mkdir 1/.snap/s1
+
+echo -n b >&3
+
+mkdir 2
+# create new snaprealm at dir a, file1's cap should be attached to the new snaprealm
+mv 1/a 2
+
+mkdir 2/.snap/s2
+
+echo -n c >&3
+
+exec 3>&-
+
+grep '^a$' 1/.snap/s1/a/file1
+grep '^ab$' 2/.snap/s2/a/file1
+grep '^abc$' 2/a/file1
+
+rmdir 1/.snap/s1
+rmdir 2/.snap/s2
+rm -rf 1 2
+echo OK
diff --git a/qa/workunits/fs/snaps/snaptest-snap-rename.sh b/qa/workunits/fs/snaps/snaptest-snap-rename.sh
new file mode 100755
index 00000000..414ba0e3
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-snap-rename.sh
@@ -0,0 +1,35 @@
+#!/bin/sh -x
+
+expect_failure() {
+	if "$@"; then return 1; else return 0; fi
+}
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+mkdir -p d1/d2
+mkdir -p d1/d3
+mkdir d1/.snap/foo
+mkdir d1/d2/.snap/foo
+mkdir d1/d3/.snap/foo
+mkdir d1/d3/.snap/bar
+mv d1/d2/.snap/foo d1/d2/.snap/bar
+# snapshot name can't start with _
+expect_failure mv d1/d2/.snap/bar d1/d2/.snap/_bar
+# can't rename parent snapshot
+expect_failure mv d1/d2/.snap/_foo_* d1/d2/.snap/foo
+expect_failure mv d1/d2/.snap/_foo_* d1/d2/.snap/_foo_1
+# can't rename snapshot to different directroy
+expect_failure mv d1/d2/.snap/bar d1/.snap/
+# can't overwrite existing snapshot
+expect_failure python -c "import os; os.rename('d1/d3/.snap/foo', 'd1/d3/.snap/bar')"
+# can't move snaphost out of snapdir
+expect_failure python -c "import os; os.rename('d1/.snap/foo', 'd1/foo')"
+
+rmdir d1/.snap/foo
+rmdir d1/d2/.snap/bar
+rmdir d1/d3/.snap/foo
+rmdir d1/d3/.snap/bar
+rm -rf d1
+
+echo OK
diff --git a/qa/workunits/fs/snaps/snaptest-snap-rm-cmp.sh b/qa/workunits/fs/snaps/snaptest-snap-rm-cmp.sh
new file mode 100755
index 00000000..c5bd65e9
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-snap-rm-cmp.sh
@@ -0,0 +1,26 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+file=linux-2.6.33.tar.bz2
+wget -q http://download.ceph.com/qa/$file
+
+real=`md5sum $file | awk '{print $1}'`
+
+for f in `seq 1 20`
+do
+    echo $f
+    cp $file a
+    mkdir .snap/s
+    rm a
+    cp .snap/s/a /tmp/a
+    cur=`md5sum /tmp/a | awk '{print $1}'`
+    if [ "$cur" != "$real" ]; then
+	echo "FAIL: bad match, /tmp/a $cur != real $real"
+	false
+    fi
+    rmdir .snap/s
+done
+rm $file
diff --git a/qa/workunits/fs/snaps/snaptest-upchildrealms.sh b/qa/workunits/fs/snaps/snaptest-upchildrealms.sh
new file mode 100755
index 00000000..a4cc9ab3
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-upchildrealms.sh
@@ -0,0 +1,30 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+#
+# verify that a snap update on a parent realm will induce
+# snap cap writeback for inodes child realms
+#
+
+mkdir a
+mkdir a/b
+mkdir a/.snap/a1
+mkdir a/b/.snap/b1
+echo asdf > a/b/foo
+mkdir a/.snap/a2
+# client _should_ have just queued a capsnap for writeback
+ln a/b/foo a/b/bar       # make the server cow the inode
+
+echo "this should not hang..."
+cat a/b/.snap/_a2_*/foo
+echo "good, it did not hang."
+
+rmdir a/b/.snap/b1
+rmdir a/.snap/a1
+rmdir a/.snap/a2
+rm -r a
+
+echo "OK"
\ No newline at end of file
diff --git a/qa/workunits/fs/snaps/snaptest-xattrwb.sh b/qa/workunits/fs/snaps/snaptest-xattrwb.sh
new file mode 100755
index 00000000..09398878
--- /dev/null
+++ b/qa/workunits/fs/snaps/snaptest-xattrwb.sh
@@ -0,0 +1,31 @@
+#!/bin/sh -x
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+echo "testing simple xattr wb"
+touch x
+setfattr -n user.foo x
+mkdir .snap/s1
+getfattr -n user.foo .snap/s1/x | grep user.foo
+rm x
+rmdir .snap/s1
+
+echo "testing wb with pre-wb server cow"
+mkdir a
+mkdir a/b
+mkdir a/b/c
+# b now has As but not Ax
+setfattr -n user.foo a/b
+mkdir a/.snap/s
+mkdir a/b/cc
+# b now has been cowed on the server, but we still have dirty xattr caps
+getfattr -n user.foo a/b          # there they are...
+getfattr -n user.foo a/.snap/s/b | grep user.foo  # should be there, too!
+
+# ok, clean up
+rmdir a/.snap/s
+rm -r a
+
+echo OK
\ No newline at end of file
diff --git a/qa/workunits/fs/snaps/untar_snap_rm.sh b/qa/workunits/fs/snaps/untar_snap_rm.sh
new file mode 100755
index 00000000..928e8911
--- /dev/null
+++ b/qa/workunits/fs/snaps/untar_snap_rm.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+
+ceph fs set cephfs allow_new_snaps true --yes-i-really-mean-it
+
+do_tarball() {
+    wget http://download.ceph.com/qa/$1
+    tar xvf$2 $1
+    mkdir .snap/k
+    sync
+    rm -rv $3
+    cp -av .snap/k .
+    rmdir .snap/k
+    rm -rv k
+    rm $1
+}
+
+do_tarball coreutils_8.5.orig.tar.gz z coreutils-8.5
+do_tarball linux-2.6.33.tar.bz2 j linux-2.6.33
diff --git a/qa/workunits/fs/test_o_trunc.c b/qa/workunits/fs/test_o_trunc.c
new file mode 100644
index 00000000..1ce19e4b
--- /dev/null
+++ b/qa/workunits/fs/test_o_trunc.c
@@ -0,0 +1,45 @@
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main(int argc, char *argv[])
+{
+	char obuf[32], ibuf[1024];
+	int n, max = 0;
+	
+	if (argc > 2)
+		max = atoi(argv[2]);
+	if (!max)
+		max = 600;
+	
+	memset(obuf, 0xff, sizeof(obuf));
+	
+	for (n = 1; n <= max; ++n) {
+		int fd, ret;
+		fd = open(argv[1], O_RDWR | O_CREAT | O_TRUNC, 0644);
+		printf("%d/%d: open fd = %d\n", n, max, fd);
+		
+		ret = write(fd, obuf, sizeof(obuf));
+		printf("write ret = %d\n", ret);
+		
+		sleep(1);
+		
+		ret = write(fd, obuf, sizeof(obuf));
+		printf("write ret = %d\n", ret);
+		
+		ret = pread(fd, ibuf, sizeof(ibuf), 0);
+		printf("pread ret = %d\n", ret);
+		
+		if (memcmp(obuf, ibuf, sizeof(obuf))) {
+			printf("mismatch\n");
+			close(fd);
+			break;
+		}
+		close(fd);
+	}
+	return 0;
+}
diff --git a/qa/workunits/fs/test_o_trunc.sh b/qa/workunits/fs/test_o_trunc.sh
new file mode 100755
index 00000000..90a72600
--- /dev/null
+++ b/qa/workunits/fs/test_o_trunc.sh
@@ -0,0 +1,7 @@
+#!/bin/sh -ex
+
+mydir=`dirname $0`
+$mydir/test_o_trunc trunc.foo 600
+
+echo OK
+
diff --git a/qa/workunits/fs/test_python.sh b/qa/workunits/fs/test_python.sh
new file mode 100755
index 00000000..656d89f0
--- /dev/null
+++ b/qa/workunits/fs/test_python.sh
@@ -0,0 +1,6 @@
+#!/bin/sh -ex
+
+# Running as root because the filesystem root directory will be
+# owned by uid 0, and that's where we're writing.
+sudo nosetests -v $(dirname $0)/../../../src/test/pybind/test_cephfs.py
+exit 0
diff --git a/qa/workunits/fs/upgrade/volume_client b/qa/workunits/fs/upgrade/volume_client
new file mode 100755
index 00000000..37ee954c
--- /dev/null
+++ b/qa/workunits/fs/upgrade/volume_client
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+set -ex
+
+PYTHON="python2"
+
+function run_payload {
+    local payload="$1"
+    sudo "$PYTHON" <<EOF
+from __future__ import print_function
+from ceph_volume_client import CephFSVolumeClient, VolumePath
+from sys import version_info as sys_version_info
+from rados import OSError as rados_OSError
+import logging
+log = logging.getLogger("ceph_volume_client")
+log.addHandler(logging.StreamHandler())
+log.setLevel(logging.DEBUG)
+vc = CephFSVolumeClient("manila", "/etc/ceph/ceph.conf", "ceph")
+vc.connect()
+${payload}
+vc.disconnect()
+EOF
+}
+
+function import_key {
+  local client="$1"
+  if [ -n "$2" ]; then
+    local keyring="$2"
+  else
+    local keyring="/etc/ceph/ceph.client.${client}.keyring"
+  fi
+  local T=$(mktemp)
+  tee "$T" >&2
+  sudo touch -- "$keyring"
+  sudo ceph-authtool "$keyring" --import-keyring "$T"
+  rm -f -- "$T"
+}
+
+function conf_keys {
+  local client="$1"
+  ls /etc/ceph >&2
+  ceph auth get-or-create "client.manila" mds 'allow *' osd 'allow rw' mon 'allow *' | import_key "$client" /etc/ceph/ceph.keyring
+}
+
+function create_data_isolated {
+  local PAYLOAD='
+vp = VolumePath(None, "vol_isolated")
+vc.create_volume(vp, (1<<33), data_isolated=True)
+auth_result = vc.authorize(vp, "vol_data_isolated", tenant_id="test")
+print("[client.vol_data_isolated]\n\tkey = ", auth_result["auth_key"])
+'
+
+  run_payload "$PAYLOAD" | import_key "vol_data_isolated"
+}
+
+function create_default {
+  local PAYLOAD='
+vp = VolumePath(None, "vol_default")
+vc.create_volume(vp, (1<<33))
+auth_result = vc.authorize(vp, "vol_default", tenant_id="test")
+print("[client.vol_default]\n\tkey = ", auth_result["auth_key"])
+'
+  run_payload "$PAYLOAD" | import_key "vol_default"
+}
+
+function create {
+  create_data_isolated
+  create_default
+}
+
+function populate {
+  pwd
+  df -h .
+  ls -l
+  cp -a /usr/bin .
+}
+
+function verify_data_isolated {
+  ceph fs subvolume getpath cephfs vol_isolated
+  stat bin
+  ls bin | tail
+}
+
+function verify_default {
+  ceph fs subvolume getpath cephfs vol_default
+  stat bin
+  ls bin | tail
+}
+
+function verify {
+  diff <(ceph fs subvolume ls cephfs | jq -cS 'sort_by(.name)' | tee /dev/stderr) <(printf '[{"name":"vol_isolated"},{"name":"vol_default"}]' | jq -cS 'sort_by(.name)')
+  verify_data_isolated
+  verify_default
+}
+
+function main {
+  if [ "$1" = create ]; then
+    conf_keys
+    create
+  elif [ "$1" = populate ]; then
+    populate
+  elif [ "$1" = verify ]; then
+    # verify (sub)volumes still exist and are configured correctly
+    verify
+  else
+    exit 1
+  fi
+}
+
+main "$ACTION"
diff --git a/qa/workunits/hadoop/repl.sh b/qa/workunits/hadoop/repl.sh
new file mode 100755
index 00000000..84f6150a
--- /dev/null
+++ b/qa/workunits/hadoop/repl.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+# bail if $TESTDIR is not set as this test will fail in that scenario
+[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; }
+
+# if HADOOP_PREFIX is not set, use default
+[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/hadoop; }
+
+# create pools with different replication factors
+for repl in 2 3 7 8 9; do
+  name=hadoop.$repl
+  ceph osd pool create $name 8 8
+  ceph osd pool set $name size $repl
+
+  id=`ceph osd dump | sed -n "s/^pool \([0-9]*\) '$name'.*/\1/p"`
+  ceph fs add_data_pool cephfs $id
+done
+
+# create a file in each of the pools
+for repl in 2 3 7 8 9; do
+  name=hadoop.$repl
+  $HADOOP_PREFIX/bin/hadoop fs -rm -f /$name.dat
+  dd if=/dev/zero bs=1048576 count=1 | \
+    $HADOOP_PREFIX/bin/hadoop fs -Dceph.data.pools="$name" \
+    -put - /$name.dat
+done
+
+# check that hadoop reports replication matching
+# that of the pool the file was written into
+for repl in 2 3 7 8 9; do
+  name=hadoop.$repl
+  repl2=$($HADOOP_PREFIX/bin/hadoop fs -ls /$name.dat | awk '{print $2}')
+  if [ $repl -ne $repl2 ]; then
+    echo "replication factors didn't match!"
+    exit 1
+  fi
+done
+
+exit 0
diff --git a/qa/workunits/hadoop/terasort.sh b/qa/workunits/hadoop/terasort.sh
new file mode 100755
index 00000000..3d6988a2
--- /dev/null
+++ b/qa/workunits/hadoop/terasort.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+INPUT=/terasort-input
+OUTPUT=/terasort-output
+REPORT=/tersort-report
+
+num_records=100000
+[ ! -z $NUM_RECORDS ] && num_records=$NUM_RECORDS
+
+# bail if $TESTDIR is not set as this test will fail in that scenario
+[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; }
+
+# if HADOOP_PREFIX is not set, use default
+[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/hadoop; }
+
+# Nuke hadoop directories
+$HADOOP_PREFIX/bin/hadoop fs -rm -r $INPUT $OUTPUT $REPORT || true
+
+# Generate terasort data
+#
+#-Ddfs.blocksize=512M \
+#-Dio.file.buffer.size=131072 \
+#-Dmapreduce.map.java.opts=-Xmx1536m \
+#-Dmapreduce.map.memory.mb=2048 \
+#-Dmapreduce.task.io.sort.mb=256 \
+#-Dyarn.app.mapreduce.am.resource.mb=1024 \
+#-Dmapred.map.tasks=64 \
+$HADOOP_PREFIX/bin/hadoop jar \
+  $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \
+  teragen \
+  -Dmapred.map.tasks=9 \
+  $num_records \
+  $INPUT
+
+# Run the sort job
+#
+#-Ddfs.blocksize=512M \
+#-Dio.file.buffer.size=131072 \
+#-Dmapreduce.map.java.opts=-Xmx1536m \
+#-Dmapreduce.map.memory.mb=2048 \
+#-Dmapreduce.map.output.compress=true \
+#-Dmapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.Lz4Codec \
+#-Dmapreduce.reduce.java.opts=-Xmx1536m \
+#-Dmapreduce.reduce.memory.mb=2048 \
+#-Dmapreduce.task.io.sort.factor=100 \
+#-Dmapreduce.task.io.sort.mb=768 \
+#-Dyarn.app.mapreduce.am.resource.mb=1024 \
+#-Dmapred.reduce.tasks=100 \
+#-Dmapreduce.terasort.output.replication=1 \
+$HADOOP_PREFIX/bin/hadoop jar \
+  $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \
+  terasort \
+  -Dmapred.reduce.tasks=10 \
+  $INPUT $OUTPUT
+
+# Validate the sorted data
+#
+#-Ddfs.blocksize=512M \
+#-Dio.file.buffer.size=131072 \
+#-Dmapreduce.map.java.opts=-Xmx1536m \
+#-Dmapreduce.map.memory.mb=2048 \
+#-Dmapreduce.reduce.java.opts=-Xmx1536m \
+#-Dmapreduce.reduce.memory.mb=2048 \
+#-Dmapreduce.task.io.sort.mb=256 \
+#-Dyarn.app.mapreduce.am.resource.mb=1024 \
+#-Dmapred.reduce.tasks=1 \
+$HADOOP_PREFIX/bin/hadoop jar \
+  $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \
+  teravalidate \
+  -Dmapred.reduce.tasks=1 \
+  $OUTPUT $REPORT
+
+exit 0
diff --git a/qa/workunits/hadoop/wordcount.sh b/qa/workunits/hadoop/wordcount.sh
new file mode 100755
index 00000000..616b08af
--- /dev/null
+++ b/qa/workunits/hadoop/wordcount.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+WC_INPUT=/wc_input
+WC_OUTPUT=/wc_output
+DATA_INPUT=$(mktemp -d)
+
+echo "starting hadoop-wordcount test"
+
+# bail if $TESTDIR is not set as this test will fail in that scenario
+[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; }
+
+# if HADOOP_PREFIX is not set, use default
+[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/hadoop; }
+
+# Nuke hadoop directories
+$HADOOP_PREFIX/bin/hadoop fs -rm -r $WC_INPUT $WC_OUTPUT || true
+
+# Fetch and import testing data set
+curl http://download.ceph.com/qa/hadoop_input_files.tar | tar xf - -C $DATA_INPUT
+$HADOOP_PREFIX/bin/hadoop fs -copyFromLocal $DATA_INPUT $WC_INPUT
+rm -rf $DATA_INPUT
+
+# Run the job
+$HADOOP_PREFIX/bin/hadoop jar \
+  $HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \
+  wordcount $WC_INPUT $WC_OUTPUT
+
+# Cleanup
+$HADOOP_PREFIX/bin/hadoop fs -rm -r $WC_INPUT $WC_OUTPUT || true
+
+echo "completed hadoop-wordcount test"
+exit 0
diff --git a/qa/workunits/kernel_untar_build.sh b/qa/workunits/kernel_untar_build.sh
new file mode 100755
index 00000000..fbab4aae
--- /dev/null
+++ b/qa/workunits/kernel_untar_build.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -e
+
+wget -O linux.tar.gz http://download.ceph.com/qa/linux-4.17.tar.gz
+
+mkdir t
+cd t
+tar xzf ../linux.tar.gz
+cd linux*
+make defconfig
+make -j`grep -c processor /proc/cpuinfo`
+cd ..
+if ! rm -rv linux* ; then
+    echo "uh oh rm -r failed, it left behind:"
+    find .
+    exit 1
+fi
+cd ..
+rm -rv t linux*
diff --git a/qa/workunits/libcephfs/test.sh b/qa/workunits/libcephfs/test.sh
new file mode 100755
index 00000000..9d3656be
--- /dev/null
+++ b/qa/workunits/libcephfs/test.sh
@@ -0,0 +1,8 @@
+#!/bin/sh -e
+
+ceph_test_libcephfs
+ceph_test_libcephfs_access
+ceph_test_libcephfs_reclaim
+ceph_test_libcephfs_lazyio
+
+exit 0
diff --git a/qa/workunits/mgr/test_localpool.sh b/qa/workunits/mgr/test_localpool.sh
new file mode 100755
index 00000000..40a749e8
--- /dev/null
+++ b/qa/workunits/mgr/test_localpool.sh
@@ -0,0 +1,21 @@
+#!/bin/sh -ex
+
+ceph config set mgr mgr/localpool/subtree host
+ceph config set mgr mgr/localpool/failure_domain osd
+ceph mgr module enable localpool
+
+while ! ceph osd pool ls | grep '^by-host-'
+do
+    sleep 5
+done
+
+ceph mgr module disable localpool
+for p in `ceph osd pool ls | grep '^by-host-'`
+do
+    ceph osd pool rm $p $p --yes-i-really-really-mean-it
+done
+
+ceph config rm mgr mgr/localpool/subtree
+ceph config rm mgr mgr/localpool/failure_domain
+
+echo OK
diff --git a/qa/workunits/mon/auth_caps.sh b/qa/workunits/mon/auth_caps.sh
new file mode 100755
index 00000000..1f59ae1f
--- /dev/null
+++ b/qa/workunits/mon/auth_caps.sh
@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+declare -A keymap
+
+combinations="r w x rw rx wx rwx"
+
+for i in ${combinations}; do
+  k="foo_$i"
+  k=`ceph auth get-or-create-key client.$i mon "allow $i"` || exit 1
+  keymap["$i"]=$k
+done
+
+# add special caps
+keymap["all"]=`ceph auth get-or-create-key client.all mon 'allow *'` || exit 1
+
+tmp=`mktemp`
+ceph auth export > $tmp
+
+trap "rm $tmp" INT ERR EXIT QUIT 0
+
+expect() {
+
+  set +e
+
+  local expected_ret=$1
+  local ret
+
+  shift
+  cmd=$@
+
+  eval $cmd
+  ret=$?
+
+  set -e
+
+  if [[ $ret -ne $expected_ret ]]; then
+    echo "ERROR: running \'$cmd\': expected $expected_ret got $ret"
+    return 1
+  fi
+
+  return 0
+}
+
+read_ops() {
+  local caps=$1
+  local has_read=1 has_exec=1
+  local ret
+  local args
+
+  ( echo $caps | grep 'r' ) || has_read=0
+  ( echo $caps | grep 'x' ) || has_exec=0
+  
+  if [[ "$caps" == "all" ]]; then
+    has_read=1
+    has_exec=1
+  fi
+
+  ret=13
+  if [[ $has_read -gt 0 && $has_exec -gt 0 ]]; then
+    ret=0
+  fi
+
+  args="--id $caps --key ${keymap[$caps]}"
+ 
+  expect $ret ceph auth get client.admin $args
+  expect $ret ceph auth get-key client.admin $args
+  expect $ret ceph auth export $args
+  expect $ret ceph auth export client.admin $args
+  expect $ret ceph auth ls $args
+  expect $ret ceph auth print-key client.admin $args
+  expect $ret ceph auth print_key client.admin $args
+}
+
+write_ops() {
+
+  local caps=$1
+  local has_read=1 has_write=1 has_exec=1
+  local ret
+  local args
+
+  ( echo $caps | grep 'r' ) || has_read=0
+  ( echo $caps | grep 'w' ) || has_write=0
+  ( echo $caps | grep 'x' ) || has_exec=0
+
+  if [[ "$caps" == "all" ]]; then
+    has_read=1
+    has_write=1
+    has_exec=1
+  fi
+
+  ret=13
+  if [[ $has_read -gt 0 && $has_write -gt 0 && $has_exec -gt 0 ]]; then
+    ret=0
+  fi
+
+  args="--id $caps --key ${keymap[$caps]}"
+
+  expect $ret ceph auth add client.foo $args
+  expect $ret "ceph auth caps client.foo mon 'allow *' $args"
+  expect $ret ceph auth get-or-create client.admin $args
+  expect $ret ceph auth get-or-create-key client.admin $args
+  expect $ret ceph auth get-or-create-key client.baz $args
+  expect $ret ceph auth del client.foo $args
+  expect $ret ceph auth del client.baz $args
+  expect $ret ceph auth import -i $tmp $args
+}
+
+echo "running combinations: ${!keymap[@]}"
+
+subcmd=$1
+
+for i in ${!keymap[@]}; do
+  echo "caps: $i"
+  if [[ -z "$subcmd" || "$subcmd" == "read" || "$subcmd" == "all" ]]; then
+    read_ops $i
+  fi
+
+  if [[ -z "$subcmd" || "$subcmd" == "write" || "$subcmd" == "all" ]]; then
+    write_ops $i
+  fi
+done
+
+# cleanup
+for i in ${combinations} all; do
+  ceph auth del client.$i || exit 1
+done
+
+echo "OK"
diff --git a/qa/workunits/mon/caps.py b/qa/workunits/mon/caps.py
new file mode 100644
index 00000000..2634f776
--- /dev/null
+++ b/qa/workunits/mon/caps.py
@@ -0,0 +1,362 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+
+import subprocess
+import shlex
+import errno
+import sys
+import os
+import io
+import re
+
+import six
+
+from ceph_argparse import * # noqa
+
+keyring_base = '/tmp/cephtest-caps.keyring'
+
+class UnexpectedReturn(Exception):
+  def __init__(self, cmd, ret, expected, msg):
+    if isinstance(cmd, list):
+      self.cmd = ' '.join(cmd)
+    else:
+      assert isinstance(cmd, str) or isinstance(cmd, six.text_type), \
+          'cmd needs to be either a list or a str'
+      self.cmd = cmd
+    self.cmd = str(self.cmd)
+    self.ret = int(ret)
+    self.expected = int(expected)
+    self.msg = str(msg)
+
+  def __str__(self):
+    return repr('{c}: expected return {e}, got {r} ({o})'.format(
+        c=self.cmd, e=self.expected, r=self.ret, o=self.msg))
+
+def call(cmd):
+  if isinstance(cmd, list):
+    args = cmd
+  elif isinstance(cmd, str) or isinstance(cmd, six.text_type):
+    args = shlex.split(cmd)
+  else:
+    assert False, 'cmd is not a string/unicode nor a list!'
+
+  print('call: {0}'.format(args))
+  proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  ret = proc.wait()
+
+  return (ret, proc)
+
+def expect(cmd, expected_ret):
+
+  try:
+    (r, p) = call(cmd)
+  except ValueError as e:
+    print('unable to run {c}: {err}'.format(c=repr(cmd), err=e.message),
+          file=sys.stderr)
+    return errno.EINVAL
+
+  assert r == p.returncode, \
+      'wth? r was supposed to match returncode!'
+
+  if r != expected_ret:
+    raise UnexpectedReturn(repr(cmd), r, expected_ret, str(p.stderr.read()))
+
+  return p
+
+def expect_to_file(cmd, expected_ret, out_file, mode='a'):
+
+  # Let the exception be propagated to the caller
+  p = expect(cmd, expected_ret)
+  assert p.returncode == expected_ret, \
+      'expected result doesn\'t match and no exception was thrown!'
+
+  with io.open(out_file, mode) as file:
+    file.write(six.text_type(p.stdout.read()))
+
+  return p
+
+class Command:
+  def __init__(self, cid, j):
+    self.cid = cid[3:]
+    self.perms = j['perm']
+    self.module = j['module']
+
+    self.sig = ''
+    self.args = []
+    for s in j['sig']:
+      if not isinstance(s, dict):
+        assert isinstance(s, str) or isinstance(s,six.text_type), \
+            'malformatted signature cid {0}: {1}\n{2}'.format(cid,s,j)
+        if len(self.sig) > 0:
+          self.sig += ' '
+        self.sig += s
+      else:
+        self.args.append(s)
+
+  def __str__(self):
+    return repr('command {0}: {1} (requires \'{2}\')'.format(self.cid,\
+          self.sig, self.perms))
+
+
+def destroy_keyring(path):
+  if not os.path.exists(path):
+    raise Exception('oops! cannot remove inexistent keyring {0}'.format(path))
+
+  # grab all client entities from the keyring
+  entities = [m.group(1) for m in [re.match(r'\[client\.(.*)\]', l)
+                for l in [str(line.strip())
+                  for line in io.open(path,'r')]] if m is not None]
+
+  # clean up and make sure each entity is gone
+  for e in entities:
+    expect('ceph auth del client.{0}'.format(e), 0)
+    expect('ceph auth get client.{0}'.format(e), errno.ENOENT)
+
+  # remove keyring
+  os.unlink(path)
+
+  return True
+
+def test_basic_auth():
+  # make sure we can successfully add/del entities, change their caps
+  # and import/export keyrings.
+
+  expect('ceph auth add client.basicauth', 0)
+  expect('ceph auth caps client.basicauth mon \'allow *\'', 0)
+  # entity exists and caps do not match
+  expect('ceph auth add client.basicauth', errno.EINVAL)
+  # this command attempts to change an existing state and will fail
+  expect('ceph auth add client.basicauth mon \'allow w\'', errno.EINVAL)
+  expect('ceph auth get-or-create client.basicauth', 0)
+  expect('ceph auth get-key client.basicauth', 0)
+  expect('ceph auth get-or-create client.basicauth2', 0)
+  # cleanup
+  expect('ceph auth del client.basicauth', 0)
+  expect('ceph auth del client.basicauth2', 0)
+
+  return True
+
+def gen_module_keyring(module):
+  module_caps = [
+      ('all', '{t} \'allow service {s} rwx\'', 0),
+      ('none', '', errno.EACCES),
+      ('wrong', '{t} \'allow service foobar rwx\'', errno.EACCES),
+      ('right', '{t} \'allow service {s} {p}\'', 0),
+      ('no-execute', '{t} \'allow service {s} x\'', errno.EACCES)
+      ]
+
+  keyring = '{0}.service-{1}'.format(keyring_base,module)
+  for perms in 'r rw x'.split():
+    for (n,p,r) in module_caps:
+      c = p.format(t='mon', s=module, p=perms)
+      expect_to_file(
+          'ceph auth get-or-create client.{cn}-{cp} {caps}'.format(
+          cn=n,cp=perms,caps=c), 0, keyring)
+
+  return keyring
+
+
+def test_all():
+
+
+  perms = {
+      'good': {
+        'broad':[
+          ('rwx', 'allow *'),
+          ('r', 'allow r'),
+          ('rw', 'allow rw'),
+          ('x', 'allow x'),
+          ],
+        'service':[
+          ('rwx', 'allow service {s} rwx'),
+          ('r', 'allow service {s} r'),
+          ('rw', 'allow service {s} rw'),
+          ('x', 'allow service {s} x'),
+          ],
+        'command':[
+          ('rwx', 'allow command "{c}"'),
+          ],
+        'command-with':[
+          ('rwx', 'allow command "{c}" with {kv}')
+          ],
+        'command-with-prefix':[
+          ('rwx', 'allow command "{c}" with {key} prefix {val}')
+          ]
+        },
+      'bad': {
+        'broad':[
+          ('none', ''),
+          ],
+        'service':[
+          ('none1', 'allow service foo rwx'),
+          ('none2', 'allow service foo r'),
+          ('none3', 'allow service foo rw'),
+          ('none4', 'allow service foo x'),
+          ],
+        'command':[
+          ('none', 'allow command foo'),
+          ],
+        'command-with':[
+          ('none', 'allow command "{c}" with foo=bar'),
+          ],
+        'command-with-prefix':[
+          ('none', 'allow command "{c}" with foo prefix bar'),
+          ],
+        }
+      }
+
+  cmds = {
+      '':[
+        {
+          'cmd':('status', '', 'r')
+          },
+        {
+          'pre':'heap start_profiler',
+          'cmd':('heap', 'heapcmd=stats', 'rw'),
+          'post':'heap stop_profiler'
+          }
+        ],
+      'auth':[
+        {
+          'pre':'',
+          'cmd':('auth ls', '', 'r'),
+          'post':''
+          },
+        {
+          'pre':'auth get-or-create client.foo mon \'allow *\'',
+          'cmd':('auth caps', 'entity="client.foo"', 'rw'),
+          'post':'auth del client.foo'
+          }
+        ],
+      'pg':[
+        {
+          'cmd':('pg getmap', '', 'r'),
+          },
+        ],
+      'mds':[
+        {
+          'cmd':('mds getmap', '', 'r'),
+          },
+        ],
+      'mon':[
+        {
+          'cmd':('mon getmap', '', 'r')
+          },
+        {
+          'cmd':('mon remove', 'name=a', 'rw')
+          }
+        ],
+      'osd':[
+        {
+          'cmd':('osd getmap', '', 'r'),
+          },
+        {
+          'cmd':('osd pause', '', 'rw'),
+          'post':'osd unpause'
+          },
+        {
+          'cmd':('osd crush dump', '', 'r')
+          },
+        ],
+      'config-key':[
+          {
+            'pre':'config-key set foo bar',
+            'cmd':('config-key get', 'key=foo', 'r')
+            },
+          {
+            'pre':'config-key set foo bar',
+            'cmd':('config-key del', 'key=foo', 'rw')
+            }
+          ]
+      }
+
+  for (module,cmd_lst) in cmds.items():
+    k = keyring_base + '.' + module
+    for cmd in cmd_lst:
+
+      (cmd_cmd, cmd_args, cmd_perm) = cmd['cmd']
+      cmd_args_key = ''
+      cmd_args_val = ''
+      if len(cmd_args) > 0:
+        (cmd_args_key, cmd_args_val) = cmd_args.split('=')
+
+      print('generating keyring for {m}/{c}'.format(m=module,c=cmd_cmd))
+      # gen keyring
+      for (good_or_bad,kind_map) in perms.items():
+        for (kind,lst) in kind_map.items():
+          for (perm, cap) in lst:
+            cap_formatted = cap.format(
+                s=module,
+                c=cmd_cmd,
+                kv=cmd_args,
+                key=cmd_args_key,
+                val=cmd_args_val)
+
+            if len(cap_formatted) == 0:
+              run_cap = ''
+            else:
+              run_cap = 'mon \'{fc}\''.format(fc=cap_formatted)
+
+            cname = 'client.{gb}-{kind}-{p}'.format(
+                gb=good_or_bad,kind=kind,p=perm)
+            expect_to_file(
+                'ceph auth get-or-create {n} {c}'.format(
+                  n=cname,c=run_cap), 0, k)
+      # keyring generated
+      print('testing {m}/{c}'.format(m=module,c=cmd_cmd))
+
+      # test
+      for good_bad in perms.keys():
+        for (kind,lst) in perms[good_bad].items():
+          for (perm,_) in lst:
+            cname = 'client.{gb}-{k}-{p}'.format(gb=good_bad,k=kind,p=perm)
+
+          if good_bad == 'good':
+            expect_ret = 0
+          else:
+            expect_ret = errno.EACCES
+
+          if ( cmd_perm not in perm ):
+            expect_ret = errno.EACCES
+          if 'with' in kind and len(cmd_args) == 0:
+            expect_ret = errno.EACCES
+          if 'service' in kind and len(module) == 0:
+            expect_ret = errno.EACCES
+
+          if 'pre' in cmd and len(cmd['pre']) > 0:
+            expect('ceph {0}'.format(cmd['pre']), 0)
+          expect('ceph -n {cn} -k {k} {c} {arg_val}'.format(
+            cn=cname,k=k,c=cmd_cmd,arg_val=cmd_args_val), expect_ret)
+          if 'post' in cmd and len(cmd['post']) > 0:
+            expect('ceph {0}'.format(cmd['post']), 0)
+      # finish testing
+      destroy_keyring(k)
+
+
+  return True
+
+
+def test_misc():
+
+  k = keyring_base + '.misc'
+  expect_to_file(
+      'ceph auth get-or-create client.caps mon \'allow command "auth caps"' \
+          ' with entity="client.caps"\'', 0, k)
+  expect('ceph -n client.caps -k {kf} mon_status'.format(kf=k), errno.EACCES)
+  expect('ceph -n client.caps -k {kf} auth caps client.caps mon \'allow *\''.format(kf=k), 0)
+  expect('ceph -n client.caps -k {kf} mon_status'.format(kf=k), 0)
+  destroy_keyring(k)
+
+def main():
+
+  test_basic_auth()
+  test_all()
+  test_misc()
+
+  print('OK')
+
+  return 0
+
+if __name__ == '__main__':
+  main()
diff --git a/qa/workunits/mon/caps.sh b/qa/workunits/mon/caps.sh
new file mode 100755
index 00000000..c5db5650
--- /dev/null
+++ b/qa/workunits/mon/caps.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+set -x
+
+tmp=/tmp/cephtest-mon-caps-madness
+
+exit_on_error=1
+
+[[ ! -z $TEST_EXIT_ON_ERROR ]] && exit_on_error=$TEST_EXIT_ON_ERROR
+
+if [ `uname` = FreeBSD ]; then
+    ETIMEDOUT=60
+else
+    ETIMEDOUT=110
+fi
+
+expect()
+{
+  cmd=$1
+  expected_ret=$2
+
+  echo $cmd
+  eval $cmd >&/dev/null
+  ret=$?
+
+  if [[ $ret -ne $expected_ret ]]; then
+    echo "Error: Expected return $expected_ret, got $ret"
+    [[ $exit_on_error -eq 1 ]] && exit 1
+    return 1
+  fi
+
+  return 0
+}
+
+expect "ceph auth get-or-create client.bazar > $tmp.bazar.keyring" 0
+expect "ceph -k $tmp.bazar.keyring --user bazar mon_status" 13
+ceph auth del client.bazar
+
+c="'allow command \"auth ls\", allow command mon_status'"
+expect "ceph auth get-or-create client.foo mon $c > $tmp.foo.keyring" 0
+expect "ceph -k $tmp.foo.keyring --user foo mon_status" 0
+expect "ceph -k $tmp.foo.keyring --user foo auth ls" 0
+expect "ceph -k $tmp.foo.keyring --user foo auth export" 13
+expect "ceph -k $tmp.foo.keyring --user foo auth del client.bazar" 13
+expect "ceph -k $tmp.foo.keyring --user foo osd dump" 13
+
+# monitor drops the subscribe message from client if it does not have enough caps
+# for read from mon. in that case, the client will be waiting for mgrmap in vain,
+# if it is instructed to send a command to mgr. "pg dump" is served by mgr. so,
+# we need to set a timeout for testing this scenario.
+#
+# leave plenty of time here because the mons might be thrashing.
+export CEPH_ARGS='--rados-mon-op-timeout=300'
+expect "ceph -k $tmp.foo.keyring --user foo pg dump" $ETIMEDOUT
+export CEPH_ARGS=''
+
+expect "ceph -k $tmp.foo.keyring --user foo quorum_status" 13
+ceph auth del client.foo
+
+c="'allow command service with prefix=list, allow command mon_status'"
+expect "ceph auth get-or-create client.bar mon $c > $tmp.bar.keyring" 0
+expect "ceph -k $tmp.bar.keyring --user bar mon_status" 0
+expect "ceph -k $tmp.bar.keyring --user bar auth ls" 13
+expect "ceph -k $tmp.bar.keyring --user bar auth export" 13
+expect "ceph -k $tmp.bar.keyring --user bar auth del client.foo" 13
+expect "ceph -k $tmp.bar.keyring --user bar osd dump" 13
+
+# again, we'll need to timeout.
+export CEPH_ARGS='--rados-mon-op-timeout=300'
+expect "ceph -k $tmp.bar.keyring --user bar pg dump" $ETIMEDOUT
+export CEPH_ARGS=''
+
+expect "ceph -k $tmp.bar.keyring --user bar quorum_status" 13
+ceph auth del client.bar
+
+rm $tmp.bazar.keyring $tmp.foo.keyring $tmp.bar.keyring
+
+# invalid caps health warning
+cat <<EOF | ceph auth import -i -
+[client.bad]
+  caps mon = this is wrong
+  caps osd = does not parse
+  caps mds = also does not parse
+EOF
+ceph health | grep AUTH_BAD_CAP
+ceph health detail | grep client.bad
+ceph auth rm client.bad
+expect "ceph auth health | grep AUTH_BAD_CAP" 1
+
+echo OK
diff --git a/qa/workunits/mon/config.sh b/qa/workunits/mon/config.sh
new file mode 100755
index 00000000..4f7bdac2
--- /dev/null
+++ b/qa/workunits/mon/config.sh
@@ -0,0 +1,110 @@
+#!/bin/bash -ex
+
+function expect_false()
+{
+	set -x
+	if "$@"; then return 1; else return 0; fi
+}
+
+ceph config dump
+
+# value validation
+ceph config set mon.a debug_xio 22
+ceph config set mon.a debug_xio 22/33
+ceph config get mon.a debug_xio | grep 22
+ceph config set mon.a debug_xio 1/2
+expect_false bin/ceph config set mon.a debug_xio foo
+expect_false bin/ceph config set mon.a debug_xio -10
+ceph config rm mon.a debug_xio
+
+ceph config set global log_graylog_port 123
+expect_false ceph config set global log_graylog_port asdf
+ceph config rm global log_graylog_port
+
+ceph config set mon mon_cluster_log_to_stderr true
+ceph config get mon.a mon_cluster_log_to_stderr | grep true
+ceph config set mon mon_cluster_log_to_stderr 2
+ceph config get mon.a mon_cluster_log_to_stderr | grep true
+ceph config set mon mon_cluster_log_to_stderr 1
+ceph config get mon.a mon_cluster_log_to_stderr | grep true
+ceph config set mon mon_cluster_log_to_stderr false
+ceph config get mon.a mon_cluster_log_to_stderr | grep false
+ceph config set mon mon_cluster_log_to_stderr 0
+ceph config get mon.a mon_cluster_log_to_stderr | grep false
+expect_false ceph config set mon mon_cluster_log_to_stderr fiddle
+expect_false ceph config set mon mon_cluster_log_to_stderr ''
+ceph config rm mon mon_cluster_log_to_stderr
+
+expect_false ceph config set mon.a osd_pool_default_type foo
+ceph config set mon.a osd_pool_default_type replicated
+ceph config rm mon.a osd_pool_default_type
+
+# scoping
+ceph config set global debug_xio 33
+ceph config get mon.a debug_xio | grep 33
+ceph config set mon debug_xio 11
+ceph config get mon.a debug_xio | grep 11
+ceph config set mon.a debug_xio 22
+ceph config get mon.a debug_xio | grep 22
+ceph config rm mon.a debug_xio
+ceph config get mon.a debug_xio | grep 11
+ceph config rm mon debug_xio
+ceph config get mon.a debug_xio | grep 33
+ceph config rm global debug_xio
+
+# help
+ceph config help debug_xio | grep debug_xio
+
+# show
+ceph config set osd.0 debug_xio 33
+while ! ceph config show osd.0 | grep debug_xio | grep 33 | grep mon
+do
+    sleep 1
+done
+ceph config set osd.0 debug_xio 22
+while ! ceph config show osd.0 | grep debug_xio | grep 22 | grep mon
+do
+    sleep 1
+done
+
+ceph tell osd.0 config set debug_xio 99
+while ! ceph config show osd.0 | grep debug_xio | grep 99
+do
+    sleep 1
+done
+ceph config show osd.0 | grep debug_xio | grep 'override mon'
+ceph tell osd.0 config unset debug_xio
+ceph tell osd.0 config unset debug_xio
+
+ceph config rm osd.0 debug_xio
+while ceph config show osd.0 | grep debug_xio | grep mon
+do
+    sleep 1
+done
+ceph config show osd.0 | grep -c debug_xio | grep 0
+
+ceph config set osd.0 osd_scrub_cost 123
+while ! ceph config show osd.0 | grep osd_scrub_cost | grep mon
+do
+    sleep 1
+done
+ceph config rm osd.0 osd_scrub_cost
+
+# show-with-defaults
+ceph config show-with-defaults osd.0 | grep debug_xio
+
+# assimilate
+t1=`mktemp`
+t2=`mktemp`
+cat <<EOF > $t1
+[osd.0]
+keyring = foo
+debug_xio = 66
+EOF
+ceph config assimilate-conf -i $t1 | tee $t2
+
+grep keyring $t2
+expect_false grep debug_xio $t2
+rm -f $t1 $t2
+
+echo OK
diff --git a/qa/workunits/mon/crush_ops.sh b/qa/workunits/mon/crush_ops.sh
new file mode 100755
index 00000000..4ad8f354
--- /dev/null
+++ b/qa/workunits/mon/crush_ops.sh
@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+
+set -ex
+
+function expect_false()
+{
+	set -x
+	if "$@"; then return 1; else return 0; fi
+}
+
+ceph osd crush dump
+
+# rules
+ceph osd crush rule dump
+ceph osd crush rule ls
+ceph osd crush rule list
+
+ceph osd crush rule create-simple foo default host
+ceph osd crush rule create-simple foo default host
+ceph osd crush rule create-simple bar default host
+
+# make sure we're at luminous+ before using crush device classes
+ceph osd require-osd-release nautilus
+ceph osd crush rm-device-class all
+ceph osd crush set-device-class ssd osd.0
+ceph osd crush set-device-class hdd osd.1
+ceph osd crush rule create-replicated foo-ssd default host ssd
+ceph osd crush rule create-replicated foo-hdd default host hdd
+ceph osd crush rule ls-by-class ssd | grep 'foo-ssd'
+ceph osd crush rule ls-by-class ssd | expect_false grep 'foo-hdd'
+ceph osd crush rule ls-by-class hdd | grep 'foo-hdd'
+ceph osd crush rule ls-by-class hdd | expect_false grep 'foo-ssd'
+
+ceph osd erasure-code-profile set ec-foo-ssd crush-device-class=ssd m=2 k=2
+ceph osd pool create ec-foo 2 erasure ec-foo-ssd
+ceph osd pool rm ec-foo ec-foo --yes-i-really-really-mean-it
+
+ceph osd crush rule ls | grep foo
+
+ceph osd crush rule rename foo foo-asdf
+ceph osd crush rule rename foo foo-asdf # idempotent
+ceph osd crush rule rename bar bar-asdf
+ceph osd crush rule ls | grep 'foo-asdf'
+ceph osd crush rule ls | grep 'bar-asdf'
+ceph osd crush rule rm foo 2>&1 | grep 'does not exist'
+ceph osd crush rule rm bar 2>&1 | grep 'does not exist'
+ceph osd crush rule rename foo-asdf foo
+ceph osd crush rule rename foo-asdf foo # idempotent
+ceph osd crush rule rename bar-asdf bar
+ceph osd crush rule ls | expect_false grep 'foo-asdf'
+ceph osd crush rule ls | expect_false grep 'bar-asdf'
+ceph osd crush rule rm foo
+ceph osd crush rule rm foo  # idempotent
+ceph osd crush rule rm bar
+
+# can't delete in-use rules, tho:
+ceph osd pool create pinning_pool 1
+expect_false ceph osd crush rule rm replicated_rule
+ceph osd pool rm pinning_pool pinning_pool --yes-i-really-really-mean-it
+
+# build a simple map
+expect_false ceph osd crush add-bucket foo osd
+ceph osd crush add-bucket foo root
+o1=`ceph osd create`
+o2=`ceph osd create`
+ceph osd crush add $o1 1 host=host1 root=foo
+ceph osd crush add $o1 1 host=host1 root=foo  # idemptoent
+ceph osd crush add $o2 1 host=host2 root=foo
+ceph osd crush add $o2 1 host=host2 root=foo  # idempotent
+ceph osd crush add-bucket bar root
+ceph osd crush add-bucket bar root  # idempotent
+ceph osd crush link host1 root=bar
+ceph osd crush link host1 root=bar  # idempotent
+ceph osd crush link host2 root=bar
+ceph osd crush link host2 root=bar  # idempotent
+
+ceph osd tree | grep -c osd.$o1 | grep -q 2
+ceph osd tree | grep -c host1 | grep -q 2
+ceph osd tree | grep -c osd.$o2 | grep -q 2
+ceph osd tree | grep -c host2 | grep -q 2
+expect_false ceph osd crush rm host1 foo   # not empty
+ceph osd crush unlink host1 foo
+ceph osd crush unlink host1 foo
+ceph osd tree | grep -c host1 | grep -q 1
+
+expect_false ceph osd crush rm foo  # not empty
+expect_false ceph osd crush rm bar  # not empty
+ceph osd crush unlink host1 bar
+ceph osd tree | grep -c host1 | grep -q 1   # now an orphan
+ceph osd crush rm osd.$o1 host1
+ceph osd crush rm host1
+ceph osd tree | grep -c host1 | grep -q 0
+expect_false ceph osd tree-from host1
+ceph osd tree-from host2
+expect_false ceph osd tree-from osd.$o2
+
+expect_false ceph osd crush rm bar   # not empty
+ceph osd crush unlink host2
+
+ceph osd crush add-bucket host-for-test host root=root-for-test rack=rack-for-test
+ceph osd tree | grep host-for-test
+ceph osd tree | grep rack-for-test
+ceph osd tree | grep root-for-test
+ceph osd crush rm host-for-test
+ceph osd crush rm rack-for-test
+ceph osd crush rm root-for-test
+
+# reference foo and bar with a rule
+ceph osd crush rule create-simple foo-rule foo host firstn
+expect_false ceph osd crush rm foo
+ceph osd crush rule rm foo-rule
+
+ceph osd crush rm bar
+ceph osd crush rm foo
+ceph osd crush rm osd.$o2 host2
+ceph osd crush rm host2
+
+ceph osd crush add-bucket foo host
+ceph osd crush move foo root=default rack=localrack
+
+ceph osd crush create-or-move osd.$o1 1.0 root=default
+ceph osd crush move osd.$o1 host=foo
+ceph osd find osd.$o1 | grep host | grep foo
+
+ceph osd crush rm osd.$o1
+ceph osd crush rm osd.$o2
+
+ceph osd crush rm foo
+
+# test reweight
+o3=`ceph osd create`
+ceph osd crush add $o3 123 root=default
+ceph osd tree | grep osd.$o3 | grep 123
+ceph osd crush reweight osd.$o3 113
+expect_false ceph osd crush reweight osd.$o3 123456
+ceph osd tree | grep osd.$o3 | grep 113
+ceph osd crush rm osd.$o3
+ceph osd rm osd.$o3
+
+# test reweight-subtree
+o4=`ceph osd create`
+o5=`ceph osd create`
+ceph osd crush add $o4 123 root=default host=foobaz
+ceph osd crush add $o5 123 root=default host=foobaz
+ceph osd tree | grep osd.$o4 | grep 123
+ceph osd tree | grep osd.$o5 | grep 123
+ceph osd crush reweight-subtree foobaz 155
+expect_false ceph osd crush reweight-subtree foobaz 123456
+ceph osd tree | grep osd.$o4 | grep 155
+ceph osd tree | grep osd.$o5 | grep 155
+ceph osd crush rm osd.$o4
+ceph osd crush rm osd.$o5
+ceph osd rm osd.$o4
+ceph osd rm osd.$o5
+
+# weight sets
+# make sure we require luminous before testing weight-sets
+ceph osd set-require-min-compat-client luminous
+ceph osd crush weight-set dump
+ceph osd crush weight-set ls
+expect_false ceph osd crush weight-set reweight fooset osd.0 .9
+ceph osd pool create fooset 8
+ceph osd pool create barset 8
+ceph osd pool set barset size 3
+expect_false ceph osd crush weight-set reweight fooset osd.0 .9
+ceph osd crush weight-set create fooset flat
+ceph osd crush weight-set create barset positional
+ceph osd crush weight-set ls | grep fooset
+ceph osd crush weight-set ls | grep barset
+ceph osd crush weight-set dump
+ceph osd crush weight-set reweight fooset osd.0 .9
+expect_false ceph osd crush weight-set reweight fooset osd.0 .9 .9
+expect_false ceph osd crush weight-set reweight barset osd.0 .9
+ceph osd crush weight-set reweight barset osd.0 .9 .9 .9
+ceph osd crush weight-set ls | grep -c fooset | grep -q 1
+ceph osd crush weight-set rm fooset
+ceph osd crush weight-set ls | grep -c fooset | grep -q 0
+ceph osd crush weight-set ls | grep barset
+ceph osd crush weight-set rm barset
+ceph osd crush weight-set ls | grep -c barset | grep -q 0
+ceph osd crush weight-set create-compat
+ceph osd crush weight-set ls | grep '(compat)'
+ceph osd crush weight-set rm-compat
+
+# weight set vs device classes
+ceph osd pool create cool 2
+ceph osd pool create cold 2
+ceph osd pool set cold size 2
+ceph osd crush weight-set create-compat
+ceph osd crush weight-set create cool flat
+ceph osd crush weight-set create cold positional
+ceph osd crush rm-device-class osd.0
+ceph osd crush weight-set reweight-compat osd.0 10.5
+ceph osd crush weight-set reweight cool osd.0 11.5
+ceph osd crush weight-set reweight cold osd.0 12.5 12.4
+ceph osd crush set-device-class fish osd.0
+ceph osd crush tree --show-shadow | grep osd\\.0 | grep fish | grep 10\\.
+ceph osd crush tree --show-shadow | grep osd\\.0 | grep fish | grep 11\\.
+ceph osd crush tree --show-shadow | grep osd\\.0 | grep fish | grep 12\\.
+ceph osd crush rm-device-class osd.0
+ceph osd crush set-device-class globster osd.0
+ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 10\\.
+ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 11\\.
+ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 12\\.
+ceph osd crush weight-set reweight-compat osd.0 7.5
+ceph osd crush weight-set reweight cool osd.0 8.5
+ceph osd crush weight-set reweight cold osd.0 6.5 6.6
+ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 7\\.
+ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 8\\.
+ceph osd crush tree --show-shadow | grep osd\\.0 | grep globster | grep 6\\.
+ceph osd crush rm-device-class osd.0
+ceph osd pool rm cool cool --yes-i-really-really-mean-it
+ceph osd pool rm cold cold --yes-i-really-really-mean-it
+ceph osd crush weight-set rm-compat
+
+# weight set vs device classes vs move
+ceph osd crush weight-set create-compat
+ceph osd crush add-bucket fooo host
+ceph osd crush move fooo root=default
+ceph osd crush add-bucket barr rack
+ceph osd crush move barr root=default
+ceph osd crush move fooo rack=barr
+ceph osd crush rm fooo
+ceph osd crush rm barr
+ceph osd crush weight-set rm-compat
+
+# this sequence would crash at one point
+ceph osd crush weight-set create-compat
+ceph osd crush add-bucket r1 rack root=default
+for f in `seq 1 32`; do
+    ceph osd crush add-bucket h$f host rack=r1
+done
+for f in `seq 1 32`; do
+    ceph osd crush rm h$f
+done
+ceph osd crush rm r1
+ceph osd crush weight-set rm-compat
+
+echo OK
diff --git a/qa/workunits/mon/osd.sh b/qa/workunits/mon/osd.sh
new file mode 100755
index 00000000..535d6c13
--- /dev/null
+++ b/qa/workunits/mon/osd.sh
@@ -0,0 +1,24 @@
+#!/bin/sh -x
+
+set -e
+
+ua=`uuidgen`
+ub=`uuidgen`
+
+# should get same id with same uuid
+na=`ceph osd create $ua`
+test $na -eq `ceph osd create $ua`
+
+nb=`ceph osd create $ub`
+test $nb -eq `ceph osd create $ub`
+test $nb -ne $na
+
+ceph osd rm $na
+ceph osd rm $na
+ceph osd rm $nb
+ceph osd rm 1000
+
+na2=`ceph osd create $ua`
+
+echo OK
+
diff --git a/qa/workunits/mon/pg_autoscaler.sh b/qa/workunits/mon/pg_autoscaler.sh
new file mode 100755
index 00000000..706f87d0
--- /dev/null
+++ b/qa/workunits/mon/pg_autoscaler.sh
@@ -0,0 +1,79 @@
+#!/bin/bash -ex
+
+NUM_OSDS=$(ceph osd ls | wc -l)
+if [ $NUM_OSDS -lt 6 ]; then
+    echo "test requires at least 6 OSDs"
+    exit 1
+fi
+
+NUM_POOLS=$(ceph osd pool ls | wc -l)
+if [ $NUM_POOLS -gt 0 ]; then
+    echo "test requires no preexisting pools"
+    exit 1
+fi
+
+function wait_for() {
+    local sec=$1
+    local cmd=$2
+
+    while true ; do
+	if bash -c "$cmd" ; then
+	    break
+	fi
+	sec=$(( $sec - 1 ))
+	if [ $sec -eq 0 ]; then
+	    echo failed
+	    return 1
+	fi
+	sleep 1
+    done
+    return 0
+}
+
+# enable
+ceph config set mgr mgr/pg_autoscaler/sleep_interval 5
+ceph mgr module enable pg_autoscaler
+
+# pg_num_min
+ceph osd pool create a 16 --pg-num-min 4
+ceph osd pool create b 16 --pg-num-min 2
+ceph osd pool set a pg_autoscale_mode on
+ceph osd pool set b pg_autoscale_mode on
+
+wait_for 120 "ceph osd pool get a pg_num | grep 4"
+wait_for 120 "ceph osd pool get b pg_num | grep 2"
+
+# target ratio
+ceph osd pool set a target_size_ratio 5
+ceph osd pool set b target_size_ratio 1
+sleep 10
+APGS=$(ceph osd dump -f json-pretty | jq '.pools[0].pg_num_target')
+BPGS=$(ceph osd dump -f json-pretty | jq '.pools[1].pg_num_target')
+test $APGS -gt 100
+test $BPGS -gt 10
+
+# small ratio change does not change pg_num
+ceph osd pool set a target_size_ratio 7
+ceph osd pool set b target_size_ratio 2
+sleep 10
+APGS2=$(ceph osd dump -f json-pretty | jq '.pools[0].pg_num_target')
+BPGS2=$(ceph osd dump -f json-pretty | jq '.pools[1].pg_num_target')
+test $APGS -eq $APGS2
+test $BPGS -eq $BPGS2
+
+# target_size
+ceph osd pool set a target_size_bytes 1000000000000000
+ceph osd pool set b target_size_bytes 1000000000000000
+ceph osd pool set a target_size_ratio 0
+ceph osd pool set b target_size_ratio 0
+wait_for 60 "ceph health detail | grep POOL_TARGET_SIZE_BYTES_OVERCOMMITTED"
+
+ceph osd pool set a target_size_bytes 1000
+ceph osd pool set b target_size_bytes 1000
+ceph osd pool set a target_size_ratio 1
+wait_for 60 "ceph health detail | grep POOL_HAS_TARGET_SIZE_BYTES_AND_RATIO"
+
+ceph osd pool rm a a --yes-i-really-really-mean-it
+ceph osd pool rm b b --yes-i-really-really-mean-it
+
+echo OK
diff --git a/qa/workunits/mon/ping.py b/qa/workunits/mon/ping.py
new file mode 100755
index 00000000..f39da885
--- /dev/null
+++ b/qa/workunits/mon/ping.py
@@ -0,0 +1,108 @@
+#!/usr/bin/python
+
+import json
+import shlex
+import subprocess
+
+import six
+
+
+class UnexpectedReturn(Exception):
+    def __init__(self, cmd, ret, expected, msg):
+        if isinstance(cmd, list):
+            self.cmd = ' '.join(cmd)
+        else:
+            assert isinstance(cmd, six.string_types) or isinstance(cmd, six.text_type), \
+                'cmd needs to be either a list or a str'
+            self.cmd = cmd
+        self.cmd = str(self.cmd)
+        self.ret = int(ret)
+        self.expected = int(expected)
+        self.msg = str(msg)
+
+    def __str__(self):
+        return repr('{c}: expected return {e}, got {r} ({o})'.format(
+            c=self.cmd, e=self.expected, r=self.ret, o=self.msg))
+
+
+def call(cmd):
+    if isinstance(cmd, list):
+        args = cmd
+    elif isinstance(cmd, six.string_types) or isinstance(cmd, six.text_type):
+        args = shlex.split(cmd)
+    else:
+        assert False, 'cmd is not a string/unicode nor a list!'
+
+    print('call: {0}'.format(args))
+    proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    procout, procerr = proc.communicate(None)
+
+    return proc.returncode, procout, procerr
+
+
+def expect(cmd, expected_ret):
+    try:
+        (r, out, err) = call(cmd)
+    except ValueError as e:
+        assert False, \
+            'unable to run {c}: {err}'.format(c=repr(cmd), err=str(e))
+
+    if r != expected_ret:
+        raise UnexpectedReturn(repr(cmd), r, expected_ret, err)
+
+    return out.decode() if isinstance(out, bytes) else out
+
+
+def get_quorum_status(timeout=300):
+    cmd = 'ceph quorum_status'
+    if timeout > 0:
+        cmd += ' --connect-timeout {0}'.format(timeout)
+
+    out = expect(cmd, 0)
+    j = json.loads(out)
+    return j
+
+
+def main():
+    quorum_status = get_quorum_status()
+    mon_names = [mon['name'] for mon in quorum_status['monmap']['mons']]
+
+    print('ping all monitors')
+    for m in mon_names:
+        print('ping mon.{0}'.format(m))
+        out = expect('ceph ping mon.{0}'.format(m), 0)
+        reply = json.loads(out)
+
+        assert reply['mon_status']['name'] == m, \
+            'reply obtained from mon.{0}, expected mon.{1}'.format(
+                reply['mon_status']['name'], m)
+
+    print('test out-of-quorum reply')
+    for m in mon_names:
+        print('testing mon.{0}'.format(m))
+        expect('ceph daemon mon.{0} quorum exit'.format(m), 0)
+
+        quorum_status = get_quorum_status()
+        assert m not in quorum_status['quorum_names'], \
+            'mon.{0} was not supposed to be in quorum ({1})'.format(
+                m, quorum_status['quorum_names'])
+
+        out = expect('ceph ping mon.{0}'.format(m), 0)
+        reply = json.loads(out)
+        mon_status = reply['mon_status']
+
+        assert mon_status['name'] == m, \
+            'reply obtained from mon.{0}, expected mon.{1}'.format(
+                mon_status['name'], m)
+
+        assert mon_status['state'] == 'electing', \
+            'mon.{0} is in state {1}, expected electing'.format(
+                m, mon_status['state'])
+
+        expect('ceph daemon mon.{0} quorum enter'.format(m), 0)
+
+    print('OK')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/qa/workunits/mon/pool_ops.sh b/qa/workunits/mon/pool_ops.sh
new file mode 100755
index 00000000..b0207769
--- /dev/null
+++ b/qa/workunits/mon/pool_ops.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+set -ex
+
+function expect_false()
+{
+	set -x
+	if "$@"; then return 1; else return 0; fi
+}
+
+function get_config_value_or_die()
+{
+  local pool_name config_opt raw val
+
+  pool_name=$1
+  config_opt=$2
+
+  raw="`$SUDO ceph osd pool get $pool_name $config_opt 2>/dev/null`"
+  if [[ $? -ne 0 ]]; then
+    echo "error obtaining config opt '$config_opt' from '$pool_name': $raw"
+    exit 1
+  fi
+
+  raw=`echo $raw | sed -e 's/[{} "]//g'`
+  val=`echo $raw | cut -f2 -d:`
+
+  echo "$val"
+  return 0
+}
+
+function expect_config_value()
+{
+  local pool_name config_opt expected_val val
+  pool_name=$1
+  config_opt=$2
+  expected_val=$3
+
+  val=$(get_config_value_or_die $pool_name $config_opt)
+
+  if [[ "$val" != "$expected_val" ]]; then
+    echo "expected '$expected_val', got '$val'"
+    exit 1
+  fi
+}
+
+# note: we need to pass the other args or ceph_argparse.py will take
+# 'invalid' that is not replicated|erasure and assume it is the next
+# argument, which is a string.
+expect_false ceph osd pool create foo 123 123 invalid foo-profile foo-ruleset
+
+ceph osd pool create foo 123 123 replicated
+ceph osd pool create fooo 123 123 erasure default
+ceph osd pool create foooo 123
+
+ceph osd pool create foo 123 # idempotent
+
+ceph osd pool set foo size 1
+expect_config_value "foo" "min_size" 1
+ceph osd pool set foo size 4
+expect_config_value "foo" "min_size" 2
+ceph osd pool set foo size 10
+expect_config_value "foo" "min_size" 5
+expect_false ceph osd pool set foo size 0
+expect_false ceph osd pool set foo size 20
+
+ceph osd pool set foo size 3
+ceph osd getcrushmap -o crush
+crushtool -d crush -o crush.txt
+sed -i 's/max_size 10/max_size 3/' crush.txt
+crushtool -c crush.txt -o crush.new
+ceph osd setcrushmap -i crush.new
+expect_false ceph osd pool set foo size 4
+ceph osd setcrushmap -i crush
+rm -f crush crush.txt crush.new
+
+# should fail due to safety interlock
+expect_false ceph osd pool delete foo
+expect_false ceph osd pool delete foo foo
+expect_false ceph osd pool delete foo foo --force
+expect_false ceph osd pool delete foo fooo --yes-i-really-mean-it
+expect_false ceph osd pool delete foo --yes-i-really-mean-it foo
+
+ceph osd pool delete foooo foooo --yes-i-really-really-mean-it
+ceph osd pool delete fooo fooo --yes-i-really-really-mean-it
+ceph osd pool delete foo foo --yes-i-really-really-mean-it
+
+# idempotent
+ceph osd pool delete foo foo --yes-i-really-really-mean-it
+ceph osd pool delete fooo fooo --yes-i-really-really-mean-it
+ceph osd pool delete fooo fooo --yes-i-really-really-mean-it
+
+# non-existent pool
+ceph osd pool delete fuggg fuggg --yes-i-really-really-mean-it
+
+echo OK
+
+
diff --git a/qa/workunits/mon/rbd_snaps_ops.sh b/qa/workunits/mon/rbd_snaps_ops.sh
new file mode 100755
index 00000000..eb88565e
--- /dev/null
+++ b/qa/workunits/mon/rbd_snaps_ops.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# attempt to trigger #6047
+
+
+cmd_no=0
+expect()
+{
+  cmd_no=$(($cmd_no+1))
+  cmd="$1"
+  expected=$2
+  echo "[$cmd_no] $cmd"
+  eval  $cmd
+  ret=$?
+  if [[ $ret -ne $expected ]]; then
+    echo "[$cmd_no] unexpected return '$ret', expected '$expected'"
+    exit 1
+  fi
+}
+
+ceph osd pool delete test test --yes-i-really-really-mean-it || true
+expect 'ceph osd pool create test 8 8' 0
+expect 'ceph osd pool application enable test rbd'
+expect 'ceph osd pool mksnap test snapshot' 0
+expect 'ceph osd pool rmsnap test snapshot' 0
+
+expect 'rbd --pool=test --rbd_validate_pool=false create --size=102400 image' 0
+expect 'rbd --pool=test snap create image@snapshot' 22
+
+expect 'ceph osd pool delete test test --yes-i-really-really-mean-it' 0
+expect 'ceph osd pool create test 8 8' 0
+expect 'rbd --pool=test pool init' 0
+expect 'rbd --pool=test create --size=102400 image' 0
+expect 'rbd --pool=test snap create image@snapshot' 0
+expect 'rbd --pool=test snap ls image' 0
+expect 'rbd --pool=test snap rm image@snapshot' 0
+
+expect 'ceph osd pool mksnap test snapshot' 22
+
+expect 'ceph osd pool delete test test --yes-i-really-really-mean-it' 0
+
+# reproduce 7210 and expect it to be fixed
+# basically create such a scenario where we end up deleting what used to
+# be an unmanaged snapshot from a not-unmanaged pool
+
+ceph osd pool delete test-foo test-foo --yes-i-really-really-mean-it || true
+expect 'ceph osd pool create test-foo 8' 0
+expect 'ceph osd pool application enable test-foo rbd'
+expect 'rbd --pool test-foo create --size 1024 image' 0
+expect 'rbd --pool test-foo snap create image@snapshot' 0
+
+ceph osd pool delete test-bar test-bar --yes-i-really-really-mean-it || true
+expect 'ceph osd pool create test-bar 8' 0
+expect 'ceph osd pool application enable test-bar rbd'
+expect 'rados cppool test-foo test-bar --yes-i-really-mean-it' 0
+expect 'rbd --pool test-bar snap rm image@snapshot' 95
+expect 'ceph osd pool delete test-foo test-foo --yes-i-really-really-mean-it' 0
+expect 'ceph osd pool delete test-bar test-bar --yes-i-really-really-mean-it' 0
+
+
+echo OK
diff --git a/qa/workunits/mon/test_config_key_caps.sh b/qa/workunits/mon/test_config_key_caps.sh
new file mode 100755
index 00000000..77b4b53b
--- /dev/null
+++ b/qa/workunits/mon/test_config_key_caps.sh
@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+
+set -x
+set -e
+
+tmp=$(mktemp -d -p /tmp test_mon_config_key_caps.XXXXX)
+entities=()
+
+function cleanup()
+{
+	set +e
+	set +x
+	if [[ -e $tmp/keyring ]] && [[ -e $tmp/keyring.orig ]]; then
+		grep '\[.*\..*\]' $tmp/keyring.orig > $tmp/entities.orig
+		for e in $(grep '\[.*\..*\]' $tmp/keyring | \
+			diff $tmp/entities.orig - | \
+			sed -n 's/^.*\[\(.*\..*\)\]/\1/p');
+		do
+			ceph auth rm $e 2>&1 >& /dev/null
+		done
+	fi
+	#rm -fr $tmp
+}
+
+trap cleanup 0 # cleanup on exit
+
+function expect_false()
+{
+	set -x
+	if "$@"; then return 1; else return 0; fi
+}
+
+# for cleanup purposes
+ceph auth export -o $tmp/keyring.orig
+
+k=$tmp/keyring
+
+# setup a few keys
+ceph config-key ls
+ceph config-key set daemon-private/osd.123/test-foo
+ceph config-key set mgr/test-foo
+ceph config-key set device/test-foo
+ceph config-key set test/foo
+
+allow_aa=client.allow_aa
+allow_bb=client.allow_bb
+allow_cc=client.allow_cc
+
+mgr_a=mgr.a
+mgr_b=mgr.b
+osd_a=osd.100
+osd_b=osd.200
+
+prefix_aa=client.prefix_aa
+prefix_bb=client.prefix_bb
+prefix_cc=client.prefix_cc
+match_aa=client.match_aa
+match_bb=client.match_bb
+
+fail_aa=client.fail_aa
+fail_bb=client.fail_bb
+fail_cc=client.fail_cc
+fail_dd=client.fail_dd
+fail_ee=client.fail_ee
+fail_ff=client.fail_ff
+fail_gg=client.fail_gg
+fail_writes=client.fail_writes
+
+ceph auth get-or-create $allow_aa mon 'allow *'
+ceph auth get-or-create $allow_bb mon 'allow service config-key rwx'
+ceph auth get-or-create $allow_cc mon 'allow command "config-key get"'
+
+ceph auth get-or-create $mgr_a mon 'allow profile mgr'
+ceph auth get-or-create $mgr_b mon 'allow profile mgr'
+ceph auth get-or-create $osd_a mon 'allow profile osd'
+ceph auth get-or-create $osd_b mon 'allow profile osd'
+
+ceph auth get-or-create $prefix_aa mon \
+	"allow command \"config-key get\" with key prefix client/$prefix_aa"
+
+cap="allow command \"config-key set\" with key prefix client/"
+cap="$cap,allow command \"config-key get\" with key prefix client/$prefix_bb"
+ceph auth get-or-create $prefix_bb mon "$cap"
+
+cap="allow command \"config-key get\" with key prefix client/"
+cap="$cap, allow command \"config-key set\" with key prefix client/"
+cap="$cap, allow command \"config-key ls\""
+ceph auth get-or-create $prefix_cc mon "$cap"
+
+cap="allow command \"config-key get\" with key=client/$match_aa/foo"
+ceph auth get-or-create $match_aa mon "$cap"
+cap="allow command \"config-key get\" with key=client/$match_bb/foo"
+cap="$cap,allow command \"config-key set\" with key=client/$match_bb/foo"
+ceph auth get-or-create $match_bb mon "$cap"
+
+ceph auth get-or-create $fail_aa mon 'allow rx'
+ceph auth get-or-create $fail_bb mon 'allow r,allow w'
+ceph auth get-or-create $fail_cc mon 'allow rw'
+ceph auth get-or-create $fail_dd mon 'allow rwx'
+ceph auth get-or-create $fail_ee mon 'allow profile bootstrap-rgw'
+ceph auth get-or-create $fail_ff mon 'allow profile bootstrap-rbd'
+# write commands will require rw; wx is not enough
+ceph auth get-or-create $fail_gg mon 'allow service config-key wx'
+# read commands will only require 'r'; 'rx' should be enough.
+ceph auth get-or-create $fail_writes mon 'allow service config-key rx'
+
+# grab keyring
+ceph auth export -o $k
+
+# keys will all the caps can do whatever
+for c in $allow_aa $allow_bb $allow_cc $mgr_a $mgr_b; do
+	ceph -k $k --name $c config-key get daemon-private/osd.123/test-foo
+	ceph -k $k --name $c config-key get mgr/test-foo
+	ceph -k $k --name $c config-key get device/test-foo
+	ceph -k $k --name $c config-key get test/foo
+done
+
+for c in $osd_a $osd_b; do
+	ceph -k $k --name $c config-key put daemon-private/$c/test-foo
+	ceph -k $k --name $c config-key get daemon-private/$c/test-foo
+	expect_false ceph -k $k --name $c config-key ls
+	expect_false ceph -k $k --name $c config-key get mgr/test-foo
+	expect_false ceph -k $k --name $c config-key get device/test-foo
+	expect_false ceph -k $k --name $c config-key get test/foo
+done
+
+expect_false ceph -k $k --name $osd_a get daemon-private/$osd_b/test-foo
+expect_false ceph -k $k --name $osd_b get daemon-private/$osd_a/test-foo
+
+expect_false ceph -k $k --name $prefix_aa \
+	config-key ls
+expect_false ceph -k $k --name $prefix_aa \
+	config-key get daemon-private/osd.123/test-foo
+expect_false ceph -k $k --name $prefix_aa \
+	config-key set test/bar
+expect_false ceph -k $k --name $prefix_aa \
+	config-key set client/$prefix_aa/foo
+
+# write something so we can read, use a custom entity
+ceph -k $k --name $allow_bb config-key set client/$prefix_aa/foo
+ceph -k $k --name $prefix_aa config-key get client/$prefix_aa/foo
+# check one writes to the other's prefix, the other is able to read
+ceph -k $k --name $prefix_bb config-key set client/$prefix_aa/bar
+ceph -k $k --name $prefix_aa config-key get client/$prefix_aa/bar
+
+ceph -k $k --name $prefix_bb config-key set client/$prefix_bb/foo
+ceph -k $k --name $prefix_bb config-key get client/$prefix_bb/foo
+
+expect_false ceph -k $k --name $prefix_bb config-key get client/$prefix_aa/bar
+expect_false ceph -k $k --name $prefix_bb config-key ls
+expect_false ceph -k $k --name $prefix_bb \
+	config-key get daemon-private/osd.123/test-foo
+expect_false ceph -k $k --name $prefix_bb config-key get mgr/test-foo
+expect_false ceph -k $k --name $prefix_bb config-key get device/test-foo
+expect_false ceph -k $k --name $prefix_bb config-key get test/bar
+expect_false ceph -k $k --name $prefix_bb config-key set test/bar
+
+ceph -k $k --name $prefix_cc config-key set client/$match_aa/foo
+ceph -k $k --name $prefix_cc config-key set client/$match_bb/foo
+ceph -k $k --name $prefix_cc config-key get client/$match_aa/foo
+ceph -k $k --name $prefix_cc config-key get client/$match_bb/foo
+expect_false ceph -k $k --name $prefix_cc config-key set other/prefix
+expect_false ceph -k $k --name $prefix_cc config-key get mgr/test-foo
+ceph -k $k --name $prefix_cc config-key ls >& /dev/null
+
+ceph -k $k --name $match_aa config-key get client/$match_aa/foo
+expect_false ceph -k $k --name $match_aa config-key get client/$match_bb/foo
+expect_false ceph -k $k --name $match_aa config-key set client/$match_aa/foo
+ceph -k $k --name $match_bb config-key get client/$match_bb/foo
+ceph -k $k --name $match_bb config-key set client/$match_bb/foo
+expect_false ceph -k $k --name $match_bb config-key get client/$match_aa/foo
+expect_false ceph -k $k --name $match_bb config-key set client/$match_aa/foo
+
+keys=(daemon-private/osd.123/test-foo
+	  mgr/test-foo
+	  device/test-foo
+	  test/foo
+	  client/$prefix_aa/foo
+	  client/$prefix_bb/foo
+	  client/$match_aa/foo
+	  client/$match_bb/foo
+)
+# expect these all to fail accessing config-key
+for c in $fail_aa $fail_bb $fail_cc \
+		 $fail_dd $fail_ee $fail_ff \
+		 $fail_gg; do
+	for m in get set; do
+		for key in ${keys[*]} client/$prefix_aa/foo client/$prefix_bb/foo; do
+			expect_false ceph -k $k --name $c config-key $m $key
+		done
+	done
+done
+
+# fail writes but succeed on reads
+expect_false ceph -k $k --name $fail_writes config-key set client/$match_aa/foo
+expect_false ceph -k $k --name $fail_writes config-key set test/foo
+ceph -k $k --name $fail_writes config-key ls
+ceph -k $k --name $fail_writes config-key get client/$match_aa/foo 
+ceph -k $k --name $fail_writes config-key get daemon-private/osd.123/test-foo
+
+echo "OK"
diff --git a/qa/workunits/mon/test_mon_config_key.py b/qa/workunits/mon/test_mon_config_key.py
new file mode 100755
index 00000000..c0cb8299
--- /dev/null
+++ b/qa/workunits/mon/test_mon_config_key.py
@@ -0,0 +1,481 @@
+#!/usr/bin/python
+#
+# test_mon_config_key - Test 'ceph config-key' interface
+#
+# Copyright (C) 2013 Inktank
+#
+# This is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1, as published by the Free Software
+# Foundation.  See file COPYING.
+#
+import argparse
+import base64
+import errno
+import json
+import logging
+import os
+import random
+import string
+import subprocess
+import sys
+import time
+
+#
+# Accepted Environment variables:
+#   CEPH_TEST_VERBOSE     - be more verbose; '1' enables; '0' disables
+#   CEPH_TEST_DURATION    - test duration in seconds
+#   CEPH_TEST_SEED        - seed to be used during the test
+#
+# Accepted arguments and options (see --help):
+#   -v, --verbose         - be more verbose
+#   -d, --duration SECS   - test duration in seconds
+#   -s, --seed SEED       - seed to be used during the test
+#
+
+
+LOG = logging.getLogger(os.path.basename(sys.argv[0].replace('.py', '')))
+
+SIZES = [
+    (0, 0),
+    (10, 0),
+    (25, 0),
+    (50, 0),
+    (100, 0),
+    (1000, 0),
+    (64 * 1024, 0),
+    (64 * 1024 + 1, -errno.EFBIG),
+    (128 * 1024, -errno.EFBIG)
+]
+
+# tests will be randomly selected from the keys here, and the test
+# suboperation will be randomly selected from the list in the values
+# here.  i.e. 'exists/existing' would test that a key the test put into
+# the store earlier actually does still exist in the config store,
+# and that's a separate test case from 'exists/enoent', which tests
+# nonexistence of a key known to not be present.
+
+OPS = {
+    'put': ['existing', 'new'],
+    'del': ['existing', 'enoent'],
+    'exists': ['existing', 'enoent'],
+    'get': ['existing', 'enoent'],
+    'list': ['existing', 'enoent'],
+    'dump': ['existing', 'enoent'],
+}
+
+CONFIG_PUT = []  # list: keys
+CONFIG_DEL = []  # list: keys
+CONFIG_EXISTING = {}  # map: key -> size
+
+
+def run_cmd(cmd, expects=0):
+    full_cmd = ['ceph', 'config-key'] + cmd
+
+    if expects < 0:
+        expects = -expects
+
+    cmdlog = LOG.getChild('run_cmd')
+    cmdlog.debug('{fc}'.format(fc=' '.join(full_cmd)))
+
+    proc = subprocess.Popen(full_cmd,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+
+    stdout = []
+    stderr = []
+    while True:
+        try:
+            out, err = proc.communicate()
+            if out is not None:
+                stdout += out.decode().split('\n')
+                cmdlog.debug('stdout: {s}'.format(s=out))
+            if err is not None:
+                stdout += err.decode().split('\n')
+                cmdlog.debug('stderr: {s}'.format(s=err))
+        except ValueError:
+            ret = proc.wait()
+            break
+
+    if ret != expects:
+        cmdlog.error('cmd > {cmd}'.format(cmd=full_cmd))
+        cmdlog.error("expected return '{expected}' got '{got}'".format(
+            expected=expects, got=ret))
+        cmdlog.error('stdout')
+        for i in stdout:
+            cmdlog.error('{x}'.format(x=i))
+        cmdlog.error('stderr')
+        for i in stderr:
+            cmdlog.error('{x}'.format(x=i))
+
+
+# end run_cmd
+
+def gen_data(size, rnd):
+    chars = string.ascii_letters + string.digits
+    return ''.join(rnd.choice(chars) for _ in range(size))
+
+
+def gen_key(rnd):
+    return gen_data(20, rnd)
+
+
+def gen_tmp_file_path(rnd):
+    file_name = gen_data(20, rnd)
+    file_path = os.path.join('/tmp', 'ceph-test.' + file_name)
+    return file_path
+
+
+def destroy_tmp_file(fpath):
+    if os.path.exists(fpath) and os.path.isfile(fpath):
+        os.unlink(fpath)
+
+
+def write_data_file(data, rnd):
+    file_path = gen_tmp_file_path(rnd)
+    data_file = open(file_path, 'a+')
+    data_file.truncate()
+    data_file.write(data)
+    data_file.close()
+    return file_path
+
+
+# end write_data_file
+
+def choose_random_op(rnd):
+    op = rnd.choice(
+        list(OPS.keys())
+    )
+    sop = rnd.choice(OPS[op])
+    return op, sop
+
+
+def parse_args(args):
+    parser = argparse.ArgumentParser(
+        description="Test the monitor's 'config-key' API",
+    )
+    parser.add_argument(
+        '-v', '--verbose',
+        action='store_true',
+        help='be more verbose',
+    )
+    parser.add_argument(
+        '-s', '--seed',
+        metavar='SEED',
+        help='use SEED instead of generating it in run-time',
+    )
+    parser.add_argument(
+        '-d', '--duration',
+        metavar='SECS',
+        help='run test for SECS seconds (default: 300)',
+    )
+    parser.set_defaults(
+        seed=None,
+        duration=300,
+        verbose=False,
+    )
+    return parser.parse_args(args)
+
+
+def main():
+    args = parse_args(sys.argv[1:])
+
+    verbose = args.verbose
+    if os.environ.get('CEPH_TEST_VERBOSE') is not None:
+        verbose = (os.environ.get('CEPH_TEST_VERBOSE') == '1')
+
+    duration = int(os.environ.get('CEPH_TEST_DURATION', args.duration))
+    seed = os.environ.get('CEPH_TEST_SEED', args.seed)
+    seed = int(time.time()) if seed is None else int(seed)
+
+    rnd = random.Random()
+    rnd.seed(seed)
+
+    loglevel = logging.INFO
+    if verbose:
+        loglevel = logging.DEBUG
+
+    logging.basicConfig(level=loglevel)
+
+    LOG.info('seed: {s}'.format(s=seed))
+
+    start = time.time()
+
+    while (time.time() - start) < duration:
+        (op, sop) = choose_random_op(rnd)
+
+        LOG.info('{o}({s})'.format(o=op, s=sop))
+        op_log = LOG.getChild('{o}({s})'.format(o=op, s=sop))
+
+        if op == 'put':
+            via_file = (rnd.uniform(0, 100) < 50.0)
+
+            expected = 0
+            cmd = ['put']
+            key = None
+
+            if sop == 'existing':
+                if len(CONFIG_EXISTING) == 0:
+                    op_log.debug('no existing keys; continue')
+                    continue
+                key = rnd.choice(CONFIG_PUT)
+                assert key in CONFIG_EXISTING, \
+                    "key '{k_}' not in CONFIG_EXISTING".format(k_=key)
+
+                expected = 0  # the store just overrides the value if the key exists
+            # end if sop == 'existing'
+            elif sop == 'new':
+                for x in range(0, 10):
+                    key = gen_key(rnd)
+                    if key not in CONFIG_EXISTING:
+                        break
+                    key = None
+                if key is None:
+                    op_log.error('unable to generate an unique key -- try again later.')
+                    continue
+
+                assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \
+                    'key {k} was not supposed to exist!'.format(k=key)
+
+            assert key is not None, \
+                'key must be != None'
+
+            cmd += [key]
+
+            (size, error) = rnd.choice(SIZES)
+            if size > 25:
+                via_file = True
+
+            data = gen_data(size, rnd)
+
+            if error == 0:  # only add if we expect the put to be successful
+                if sop == 'new':
+                    CONFIG_PUT.append(key)
+                CONFIG_EXISTING[key] = size
+            expected = error
+
+            if via_file:
+                data_file = write_data_file(data, rnd)
+                cmd += ['-i', data_file]
+            else:
+                cmd += [data]
+
+            op_log.debug('size: {sz}, via: {v}'.format(
+                sz=size,
+                v='file: {f}'.format(f=data_file) if via_file == True else 'cli')
+            )
+            run_cmd(cmd, expects=expected)
+            if via_file:
+                destroy_tmp_file(data_file)
+            continue
+
+        elif op == 'del':
+            expected = 0
+            cmd = ['del']
+            key = None
+
+            if sop == 'existing':
+                if len(CONFIG_EXISTING) == 0:
+                    op_log.debug('no existing keys; continue')
+                    continue
+                key = rnd.choice(CONFIG_PUT)
+                assert key in CONFIG_EXISTING, \
+                    "key '{k_}' not in CONFIG_EXISTING".format(k_=key)
+
+            if sop == 'enoent':
+                for x in range(0, 10):
+                    key = base64.b64encode(os.urandom(20)).decode()
+                    if key not in CONFIG_EXISTING:
+                        break
+                    key = None
+                if key is None:
+                    op_log.error('unable to generate an unique key -- try again later.')
+                    continue
+                assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \
+                    'key {k} was not supposed to exist!'.format(k=key)
+                expected = 0  # deleting a non-existent key succeeds
+
+            assert key is not None, \
+                'key must be != None'
+
+            cmd += [key]
+            op_log.debug('key: {k}'.format(k=key))
+            run_cmd(cmd, expects=expected)
+            if sop == 'existing':
+                CONFIG_DEL.append(key)
+                CONFIG_PUT.remove(key)
+                del CONFIG_EXISTING[key]
+            continue
+
+        elif op == 'exists':
+            expected = 0
+            cmd = ['exists']
+            key = None
+
+            if sop == 'existing':
+                if len(CONFIG_EXISTING) == 0:
+                    op_log.debug('no existing keys; continue')
+                    continue
+                key = rnd.choice(CONFIG_PUT)
+                assert key in CONFIG_EXISTING, \
+                    "key '{k_}' not in CONFIG_EXISTING".format(k_=key)
+
+            if sop == 'enoent':
+                for x in range(0, 10):
+                    key = base64.b64encode(os.urandom(20)).decode()
+                    if key not in CONFIG_EXISTING:
+                        break
+                    key = None
+                if key is None:
+                    op_log.error('unable to generate an unique key -- try again later.')
+                    continue
+                assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \
+                    'key {k} was not supposed to exist!'.format(k=key)
+                expected = -errno.ENOENT
+
+            assert key is not None, \
+                'key must be != None'
+
+            cmd += [key]
+            op_log.debug('key: {k}'.format(k=key))
+            run_cmd(cmd, expects=expected)
+            continue
+
+        elif op == 'get':
+            expected = 0
+            cmd = ['get']
+            key = None
+
+            if sop == 'existing':
+                if len(CONFIG_EXISTING) == 0:
+                    op_log.debug('no existing keys; continue')
+                    continue
+                key = rnd.choice(CONFIG_PUT)
+                assert key in CONFIG_EXISTING, \
+                    "key '{k_}' not in CONFIG_EXISTING".format(k_=key)
+
+            if sop == 'enoent':
+                for x in range(0, 10):
+                    key = base64.b64encode(os.urandom(20)).decode()
+                    if key not in CONFIG_EXISTING:
+                        break
+                    key = None
+                if key is None:
+                    op_log.error('unable to generate an unique key -- try again later.')
+                    continue
+                assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \
+                    'key {k} was not supposed to exist!'.format(k=key)
+                expected = -errno.ENOENT
+
+            assert key is not None, \
+                'key must be != None'
+
+            file_path = gen_tmp_file_path(rnd)
+            cmd += [key, '-o', file_path]
+            op_log.debug('key: {k}'.format(k=key))
+            run_cmd(cmd, expects=expected)
+            if sop == 'existing':
+                try:
+                    temp_file = open(file_path, 'r+')
+                except IOError as err:
+                    if err.errno == errno.ENOENT:
+                        assert CONFIG_EXISTING[key] == 0, \
+                            "error opening '{fp}': {e}".format(fp=file_path, e=err)
+                        continue
+                    else:
+                        assert False, \
+                            'some error occurred: {e}'.format(e=err)
+                cnt = 0
+                while True:
+                    read_data = temp_file.read()
+                    if read_data == '':
+                        break
+                    cnt += len(read_data)
+                assert cnt == CONFIG_EXISTING[key], \
+                    "wrong size from store for key '{k}': {sz}, expected {es}".format(
+                        k=key, sz=cnt, es=CONFIG_EXISTING[key])
+                destroy_tmp_file(file_path)
+            continue
+
+        elif op == 'list' or op == 'dump':
+            expected = 0
+            cmd = [op]
+            key = None
+
+            if sop == 'existing':
+                if len(CONFIG_EXISTING) == 0:
+                    op_log.debug('no existing keys; continue')
+                    continue
+                key = rnd.choice(CONFIG_PUT)
+                assert key in CONFIG_EXISTING, \
+                    "key '{k_}' not in CONFIG_EXISTING".format(k_=key)
+
+            if sop == 'enoent':
+                for x in range(0, 10):
+                    key = base64.b64encode(os.urandom(20)).decode()
+                    if key not in CONFIG_EXISTING:
+                        break
+                    key = None
+                if key is None:
+                    op_log.error('unable to generate an unique key -- try again later.')
+                    continue
+                assert key not in CONFIG_PUT and key not in CONFIG_EXISTING, \
+                    'key {k} was not supposed to exist!'.format(k=key)
+
+            assert key is not None, \
+                'key must be != None'
+
+            file_path = gen_tmp_file_path(rnd)
+            cmd += ['-o', file_path]
+            op_log.debug('key: {k}'.format(k=key))
+            run_cmd(cmd, expects=expected)
+            try:
+                temp_file = open(file_path, 'r+')
+            except IOError as err:
+                if err.errno == errno.ENOENT:
+                    assert CONFIG_EXISTING[key] == 0, \
+                        "error opening '{fp}': {e}".format(fp=file_path, e=err)
+                    continue
+                else:
+                    assert False, \
+                        'some error occurred: {e}'.format(e=err)
+            cnt = 0
+            try:
+                read_data = json.load(temp_file)
+            except ValueError:
+                temp_file.seek(0)
+                assert False, "{op} output was not valid JSON:\n{filedata}".format(op, temp_file.readlines())
+
+            if sop == 'existing':
+                assert key in read_data, "key '{k}' not found in list/dump output".format(k=key)
+                if op == 'dump':
+                    cnt = len(read_data[key])
+                    assert cnt == CONFIG_EXISTING[key], \
+                        "wrong size from list for key '{k}': {sz}, expected {es}".format(
+                        k=key, sz=cnt, es=CONFIG_EXISTING[key])
+            elif sop == 'enoent':
+                assert key not in read_data, "key '{k}' found in list/dump output".format(k=key)
+            destroy_tmp_file(file_path)
+            continue
+        else:
+            assert False, 'unknown op {o}'.format(o=op)
+
+    # check if all keys in 'CONFIG_PUT' exist and
+    # if all keys on 'CONFIG_DEL' don't.
+    # but first however, remove all keys in CONFIG_PUT that might
+    # be in CONFIG_DEL as well.
+    config_put_set = set(CONFIG_PUT)
+    config_del_set = set(CONFIG_DEL).difference(config_put_set)
+
+    LOG.info('perform sanity checks on store')
+
+    for k in config_put_set:
+        LOG.getChild('check(puts)').debug('key: {k_}'.format(k_=k))
+        run_cmd(['exists', k], expects=0)
+    for k in config_del_set:
+        LOG.getChild('check(dels)').debug('key: {k_}'.format(k_=k))
+        run_cmd(['exists', k], expects=-errno.ENOENT)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/qa/workunits/mon/test_mon_osdmap_prune.sh b/qa/workunits/mon/test_mon_osdmap_prune.sh
new file mode 100755
index 00000000..9cdd7217
--- /dev/null
+++ b/qa/workunits/mon/test_mon_osdmap_prune.sh
@@ -0,0 +1,205 @@
+#!/bin/bash
+
+. $(dirname $0)/../../standalone/ceph-helpers.sh
+
+set -x
+
+function wait_for_osdmap_manifest() {
+
+  local what=${1:-"true"}
+
+  local -a delays=($(get_timeout_delays $TIMEOUT .1))
+  local -i loop=0
+
+  for ((i=0; i < ${#delays[*]}; ++i)); do
+    has_manifest=$(ceph report | jq 'has("osdmap_manifest")')
+    if [[ "$has_manifest" == "$what" ]]; then
+      return 0
+    fi
+
+    sleep ${delays[$i]}
+  done
+
+  echo "osdmap_manifest never outputted on report"
+  ceph report
+  return 1
+}
+
+function wait_for_trim() {
+
+  local -i epoch=$1
+  local -a delays=($(get_timeout_delays $TIMEOUT .1))
+  local -i loop=0
+
+  for ((i=0; i < ${#delays[*]}; ++i)); do
+    fc=$(ceph report | jq '.osdmap_first_committed')
+    if [[ $fc -eq $epoch ]]; then
+      return 0
+    fi
+    sleep ${delays[$i]}
+  done
+
+  echo "never trimmed up to epoch $epoch"
+  ceph report
+  return 1
+}
+
+function test_osdmap() {
+
+  local epoch=$1
+  local ret=0
+
+  tmp_map=$(mktemp)
+  ceph osd getmap $epoch -o $tmp_map || return 1
+  if ! osdmaptool --print $tmp_map | grep "epoch $epoch" ; then
+    echo "ERROR: failed processing osdmap epoch $epoch"
+    ret=1
+  fi
+  rm $tmp_map
+  return $ret
+}
+
+function generate_osdmaps() {
+
+  local -i num=$1
+
+  cmds=( set unset )
+  for ((i=0; i < num; ++i)); do
+    ceph osd ${cmds[$((i%2))]} noup || return 1
+  done
+  return 0
+}
+
+function test_mon_osdmap_prune() {
+
+  create_pool foo 32
+  wait_for_clean || return 1
+
+  ceph config set mon mon_debug_block_osdmap_trim true || return 1
+
+  generate_osdmaps 500 || return 1
+
+  report="$(ceph report)"
+  fc=$(jq '.osdmap_first_committed' <<< $report)
+  lc=$(jq '.osdmap_last_committed' <<< $report)
+
+  [[ $((lc-fc)) -ge 500 ]] || return 1
+
+  wait_for_osdmap_manifest || return 1
+
+  manifest="$(ceph report | jq '.osdmap_manifest')"
+
+  first_pinned=$(jq '.first_pinned' <<< $manifest)
+  last_pinned=$(jq '.last_pinned' <<< $manifest)
+  pinned_maps=( $(jq '.pinned_maps[]' <<< $manifest) )
+
+  # validate pinned maps list
+  [[ $first_pinned -eq ${pinned_maps[0]} ]] || return 1
+  [[ $last_pinned -eq ${pinned_maps[-1]} ]] || return 1
+
+  # validate pinned maps range
+  [[ $first_pinned -lt $last_pinned ]] || return 1
+  [[ $last_pinned -lt $lc ]] || return 1
+  [[ $first_pinned -eq $fc ]] || return 1
+
+  # ensure all the maps are available, and work as expected
+  # this can take a while...
+
+  for ((i=$first_pinned; i <= $last_pinned; ++i)); do
+    test_osdmap $i || return 1
+  done
+
+  # update pinned maps state:
+  #  the monitor may have pruned & pinned additional maps since we last
+  #  assessed state, given it's an iterative process.
+  #
+  manifest="$(ceph report | jq '.osdmap_manifest')"
+  first_pinned=$(jq '.first_pinned' <<< $manifest)
+  last_pinned=$(jq '.last_pinned' <<< $manifest)
+  pinned_maps=( $(jq '.pinned_maps[]' <<< $manifest) )
+
+  # test trimming maps
+  #
+  # we're going to perform the following tests:
+  #
+  #  1. force trim to a pinned map
+  #  2. force trim to a pinned map's previous epoch
+  #  3. trim all maps except the last 200 or so.
+  #
+
+  # 1. force trim to a pinned map
+  #
+  [[ ${#pinned_maps[@]} -gt 10 ]] || return 1
+
+  trim_to=${pinned_maps[1]}
+  ceph config set mon mon_osd_force_trim_to $trim_to
+  ceph config set mon mon_min_osdmap_epochs 100
+  ceph config set mon paxos_service_trim_min 1
+  ceph config set mon mon_debug_block_osdmap_trim false
+
+  # generate an epoch so we get to trim maps
+  ceph osd set noup
+  ceph osd unset noup
+
+  wait_for_trim $trim_to || return 1
+
+  report="$(ceph report)"
+  fc=$(jq '.osdmap_first_committed' <<< $report)
+  [[ $fc -eq $trim_to ]] || return 1
+
+  old_first_pinned=$first_pinned
+  old_last_pinned=$last_pinned
+  first_pinned=$(jq '.osdmap_manifest.first_pinned' <<< $report)
+  last_pinned=$(jq '.osdmap_manifest.last_pinned' <<< $report)
+  [[ $first_pinned -eq $trim_to ]] || return 1
+  [[ $first_pinned -gt $old_first_pinned ]] || return 1
+  [[ $last_pinned -gt $old_first_pinned ]] || return 1
+
+  test_osdmap $trim_to || return 1
+  test_osdmap $(( trim_to+1 )) || return 1
+
+  pinned_maps=( $(jq '.osdmap_manifest.pinned_maps[]' <<< $report) )
+
+  # 2. force trim to a pinned map's previous epoch
+  #
+  [[ ${#pinned_maps[@]} -gt 2 ]] || return 1
+  trim_to=$(( ${pinned_maps[1]} - 1))
+  ceph config set mon mon_osd_force_trim_to $trim_to
+
+  # generate an epoch so we get to trim maps
+  ceph osd set noup
+  ceph osd unset noup
+
+  wait_for_trim $trim_to || return 1
+
+  report="$(ceph report)"
+  fc=$(jq '.osdmap_first_committed' <<< $report)
+  [[ $fc -eq $trim_to ]] || return 1
+
+  old_first_pinned=$first_pinned
+  old_last_pinned=$last_pinned
+  first_pinned=$(jq '.osdmap_manifest.first_pinned' <<< $report)
+  last_pinned=$(jq '.osdmap_manifest.last_pinned' <<< $report)
+  pinned_maps=( $(jq '.osdmap_manifest.pinned_maps[]' <<< $report) )
+  [[ $first_pinned -eq $trim_to ]] || return 1
+  [[ ${pinned_maps[1]} -eq $(( trim_to+1)) ]] || return 1
+
+  test_osdmap $first_pinned || return 1
+  test_osdmap $(( first_pinned + 1 )) || return 1
+
+  # 3. trim everything
+  #
+  ceph config set mon mon_osd_force_trim_to 0
+
+  # generate an epoch so we get to trim maps
+  ceph osd set noup
+  ceph osd unset noup
+
+  wait_for_osdmap_manifest "false" || return 1
+
+  return 0
+}
+
+test_mon_osdmap_prune || exit 1
+
+echo "OK"
diff --git a/qa/workunits/objectstore/test_fuse.sh b/qa/workunits/objectstore/test_fuse.sh
new file mode 100755
index 00000000..f1dcbd04
--- /dev/null
+++ b/qa/workunits/objectstore/test_fuse.sh
@@ -0,0 +1,129 @@
+#!/bin/sh -ex
+
+if ! id -u | grep -q '^0$'; then
+    echo "not root, re-running self via sudo"
+    sudo PATH=$PATH TYPE=$TYPE $0
+    exit 0
+fi
+
+expect_false()
+{
+        set -x
+        if "$@"; then return 1; else return 0; fi
+}
+
+COT=ceph-objectstore-tool
+DATA=store_test_fuse_dir
+[ -z "$TYPE" ] && TYPE=bluestore
+MNT=store_test_fuse_mnt
+
+rm -rf $DATA
+mkdir -p $DATA
+
+test -d $MNT && fusermount -u $MNT || true
+rmdir $MNT || true
+mkdir $MNT
+
+export CEPH_ARGS=--enable_experimental_unrecoverable_data_corrupting_features=bluestore
+
+$COT --no-mon-config --op mkfs --data-path $DATA --type $TYPE
+$COT --no-mon-config --op fuse --data-path $DATA --mountpoint $MNT &
+
+while ! test -e $MNT/type ; do
+    echo waiting for $MNT/type to appear
+    sleep 1
+done
+
+umask 0
+
+grep $TYPE $MNT/type
+
+# create collection
+mkdir $MNT/meta
+test -e $MNT/meta/bitwise_hash_start
+test -d $MNT/meta/all
+test -d $MNT/meta/by_bitwise_hash
+
+# create object
+mkdir $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#
+test -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data
+test -d $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr
+test -d $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap
+test -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/bitwise_hash
+test -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap_header
+
+# omap header
+echo omap header > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap_header
+grep -q omap $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap_header
+
+# omap
+echo value a > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keya
+echo value b > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keyb
+ls $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap | grep -c key | grep -q 2
+grep 'value a' $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keya
+grep 'value b' $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keyb
+rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keya
+test ! -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keya
+rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keyb
+test ! -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/omap/keyb
+
+# attr
+echo value a > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keya
+echo value b > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keyb
+ls $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr | grep -c key | grep -q 2
+grep 'value a' $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keya
+grep 'value b' $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keyb
+rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keya
+test ! -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keya
+rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keyb
+test ! -e $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/attr/keyb
+
+# data
+test ! -s $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data
+echo asdfasdfasdf > $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data
+test -s $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data
+grep -q asdfasdfasdf $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data
+truncate --size 4 $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data
+stat --format=%s $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data | grep -q ^4$
+expect_false grep -q asdfasdfasdf $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data
+rm $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data
+test ! -s $MNT/meta/all/#-1:7b3f43c4:::osd_superblock:0#/data
+
+
+# create pg collection
+mkdir --mode 0003 $MNT/0.0_head
+grep -q 00000000 $MNT/0.0_head/bitwise_hash_start
+if [ "$TYPE" = "bluestore" ]; then
+    cat $MNT/0.0_head/bitwise_hash_bits
+    grep -q 3 $MNT/0.0_head/bitwise_hash_bits
+    grep -q 1fffffff $MNT/0.0_head/bitwise_hash_end
+fi
+test -d $MNT/0.0_head/all
+
+mkdir --mode 0003 $MNT/0.1_head
+grep -q 80000000 $MNT/0.1_head/bitwise_hash_start
+if [ "$TYPE" = "bluestore" ]; then
+    grep -q 3 $MNT/0.1_head/bitwise_hash_bits
+    grep -q 9fffffff $MNT/0.1_head/bitwise_hash_end
+fi
+
+# create pg object
+mkdir $MNT/0.0_head/all/#0:00000000::::head#/
+mkdir $MNT/0.0_head/all/#0:10000000:::foo:head#/
+
+# verify pg bounds check
+if [ "$TYPE" = "bluestore" ]; then
+    expect_false mkdir $MNT/0.0_head/all/#0:20000000:::bar:head#/
+fi
+
+# remove a collection
+expect_false rmdir $MNT/0.0_head
+rmdir $MNT/0.0_head/all/#0:10000000:::foo:head#/
+rmdir $MNT/0.0_head/all/#0:00000000::::head#/
+rmdir $MNT/0.0_head
+rmdir $MNT/0.1_head
+
+fusermount -u $MNT
+wait
+
+echo OK
diff --git a/qa/workunits/osdc/stress_objectcacher.sh b/qa/workunits/osdc/stress_objectcacher.sh
new file mode 100755
index 00000000..67baadc3
--- /dev/null
+++ b/qa/workunits/osdc/stress_objectcacher.sh
@@ -0,0 +1,28 @@
+#!/bin/sh -ex
+
+for i in $(seq 1 10)
+do
+    for DELAY in 0 1000
+    do
+        for OPS in 1000 10000
+        do
+            for OBJECTS in 10 50 100
+            do
+                for READS in 0.90 0.50 0.10
+                do
+                    for OP_SIZE in 4096 131072 1048576
+                    do
+                        for MAX_DIRTY in 0 25165824
+                        do
+                            ceph_test_objectcacher_stress --ops $OPS --percent-read $READS --delay-ns $DELAY --objects $OBJECTS --max-op-size $OP_SIZE --client-oc-max-dirty $MAX_DIRTY --stress-test > /dev/null 2>&1
+                        done
+                    done
+                done
+            done
+        done
+    done
+done
+
+ceph_test_objectcacher_stress --correctness-test > /dev/null 2>&1
+
+echo OK
diff --git a/qa/workunits/post-file.sh b/qa/workunits/post-file.sh
new file mode 100755
index 00000000..120fb263
--- /dev/null
+++ b/qa/workunits/post-file.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -ex
+
+what="$1"
+[ -z "$what" ] && what=/etc/udev/rules.d
+sudo ceph-post-file -d ceph-test-workunit $what
+
+echo OK
diff --git a/qa/workunits/rados/clone.sh b/qa/workunits/rados/clone.sh
new file mode 100755
index 00000000..281e89f7
--- /dev/null
+++ b/qa/workunits/rados/clone.sh
@@ -0,0 +1,13 @@
+#!/bin/sh -x
+
+set -e
+
+rados -p data rm foo || true
+rados -p data put foo.tmp /etc/passwd --object-locator foo
+rados -p data clonedata foo.tmp foo --object-locator foo
+rados -p data get foo /tmp/foo
+cmp /tmp/foo /etc/passwd
+rados -p data rm foo.tmp --object-locator foo
+rados -p data rm foo
+
+echo OK
\ No newline at end of file
diff --git a/qa/workunits/rados/load-gen-big.sh b/qa/workunits/rados/load-gen-big.sh
new file mode 100755
index 00000000..6715658e
--- /dev/null
+++ b/qa/workunits/rados/load-gen-big.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+rados -p rbd load-gen \
+    --num-objects 10240 \
+    --min-object-size 1048576 \
+    --max-object-size 25600000 \
+    --max-ops 1024 \
+    --max-backlog 1024 \
+    --read-percent 50 \
+    --run-length 1200
diff --git a/qa/workunits/rados/load-gen-mix-small-long.sh b/qa/workunits/rados/load-gen-mix-small-long.sh
new file mode 100755
index 00000000..593bad51
--- /dev/null
+++ b/qa/workunits/rados/load-gen-mix-small-long.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+rados -p rbd load-gen \
+    --num-objects 1024 \
+    --min-object-size 1 \
+    --max-object-size 1048576 \
+    --max-ops 128 \
+    --max-backlog 128 \
+    --read-percent 50 \
+    --run-length 1800
diff --git a/qa/workunits/rados/load-gen-mix-small.sh b/qa/workunits/rados/load-gen-mix-small.sh
new file mode 100755
index 00000000..02db77bd
--- /dev/null
+++ b/qa/workunits/rados/load-gen-mix-small.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+rados -p rbd load-gen \
+    --num-objects 1024 \
+    --min-object-size 1 \
+    --max-object-size 1048576 \
+    --max-ops 128 \
+    --max-backlog 128 \
+    --read-percent 50 \
+    --run-length 600
diff --git a/qa/workunits/rados/load-gen-mix.sh b/qa/workunits/rados/load-gen-mix.sh
new file mode 100755
index 00000000..ad3b4be8
--- /dev/null
+++ b/qa/workunits/rados/load-gen-mix.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+rados -p rbd load-gen \
+    --num-objects 10240 \
+    --min-object-size 1 \
+    --max-object-size 1048576 \
+    --max-ops 128 \
+    --max-backlog 128 \
+    --read-percent 50 \
+    --run-length 600
diff --git a/qa/workunits/rados/load-gen-mostlyread.sh b/qa/workunits/rados/load-gen-mostlyread.sh
new file mode 100755
index 00000000..236f82dd
--- /dev/null
+++ b/qa/workunits/rados/load-gen-mostlyread.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+rados -p rbd load-gen \
+    --num-objects 51200 \
+    --min-object-size 1 \
+    --max-object-size 1048576 \
+    --max-ops 128 \
+    --max-backlog 128 \
+    --read-percent 90 \
+    --run-length 600
diff --git a/qa/workunits/rados/stress_watch.sh b/qa/workunits/rados/stress_watch.sh
new file mode 100755
index 00000000..49f144bb
--- /dev/null
+++ b/qa/workunits/rados/stress_watch.sh
@@ -0,0 +1,7 @@
+#!/bin/sh -e
+
+ceph_test_stress_watch
+ceph_multi_stress_watch rep reppool repobj
+ceph_multi_stress_watch ec ecpool ecobj
+
+exit 0
diff --git a/qa/workunits/rados/test.sh b/qa/workunits/rados/test.sh
new file mode 100755
index 00000000..a0b2aed5
--- /dev/null
+++ b/qa/workunits/rados/test.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -ex
+
+parallel=1
+[ "$1" = "--serial" ] && parallel=0
+
+color=""
+[ -t 1 ] && color="--gtest_color=yes"
+
+function cleanup() {
+    pkill -P $$ || true
+}
+trap cleanup EXIT ERR HUP INT QUIT
+
+declare -A pids
+
+for f in \
+    api_aio api_aio_pp \
+    api_io api_io_pp \
+    api_asio api_list \
+    api_lock api_lock_pp \
+    api_misc api_misc_pp \
+    api_tier_pp \
+    api_pool \
+    api_snapshots api_snapshots_pp \
+    api_stat api_stat_pp \
+    api_watch_notify api_watch_notify_pp \
+    api_cmd api_cmd_pp \
+    api_service api_service_pp \
+    api_c_write_operations \
+    api_c_read_operations \
+    list_parallel \
+    open_pools_parallel \
+    delete_pools_parallel
+do
+    if [ $parallel -eq 1 ]; then
+	r=`printf '%25s' $f`
+	ff=`echo $f | awk '{print $1}'`
+	bash -o pipefail -exc "ceph_test_rados_$f $color 2>&1 | tee ceph_test_rados_$ff.log | sed \"s/^/$r: /\"" &
+	pid=$!
+	echo "test $f on pid $pid"
+	pids[$f]=$pid
+    else
+	ceph_test_rados_$f
+    fi
+done
+
+ret=0
+if [ $parallel -eq 1 ]; then
+for t in "${!pids[@]}"
+do
+  pid=${pids[$t]}
+  if ! wait $pid
+  then
+    echo "error in $t ($pid)"
+    ret=1
+  fi
+done
+fi
+
+exit $ret
diff --git a/qa/workunits/rados/test_alloc_hint.sh b/qa/workunits/rados/test_alloc_hint.sh
new file mode 100755
index 00000000..2323915f
--- /dev/null
+++ b/qa/workunits/rados/test_alloc_hint.sh
@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+
+set -ex
+shopt -s nullglob # fns glob expansion in expect_alloc_hint_eq()
+
+#
+# Helpers
+#
+
+function get_xml_val() {
+    local xml="$1"
+    local tag="$2"
+
+    local regex=".*<${tag}>(.*)</${tag}>.*"
+    if [[ ! "${xml}" =~ ${regex} ]]; then
+        echo "'${xml}' xml doesn't match '${tag}' tag regex" >&2
+        return 2
+    fi
+
+    echo "${BASH_REMATCH[1]}"
+}
+
+function get_conf_val() {
+    set -e
+
+    local entity="$1"
+    local option="$2"
+
+    local val
+    val="$(sudo ceph daemon "${entity}" config get --format=xml "${option}")"
+    val="$(get_xml_val "${val}" "${option}")"
+
+    echo "${val}"
+}
+
+function setup_osd_data() {
+    for (( i = 0 ; i < "${NUM_OSDS}" ; i++ )); do
+        OSD_DATA[i]="$(get_conf_val "osd.$i" "osd_data")"
+    done
+}
+
+function setup_pgid() {
+    local poolname="$1"
+    local objname="$2"
+
+    local pgid
+    pgid="$(ceph osd map "${poolname}" "${objname}" --format=xml)"
+    pgid="$(get_xml_val "${pgid}" "pgid")"
+
+    PGID="${pgid}"
+}
+
+function expect_alloc_hint_eq() {
+    export CEPH_ARGS="--osd-objectstore=filestore"
+    local expected_extsize="$1"
+
+    for (( i = 0 ; i < "${NUM_OSDS}" ; i++ )); do
+        # Make sure that stuff is flushed from the journal to the store
+        # by the time we get to it, as we prod the actual files and not
+        # the journal.
+        sudo ceph daemon "osd.${i}" "flush_journal"
+
+        # e.g., .../25.6_head/foo__head_7FC1F406__19
+        #       .../26.bs1_head/bar__head_EFE6384B__1a_ffffffffffffffff_1
+        local fns=$(sudo sh -c "ls ${OSD_DATA[i]}/current/${PGID}*_head/${OBJ}_*")
+        local count="${#fns[@]}"
+        if [ "${count}" -ne 1 ]; then
+            echo "bad fns count: ${count}" >&2
+            return 2
+        fi
+
+        local extsize
+        extsize="$(sudo xfs_io -c extsize "${fns[0]}")"
+        local extsize_regex="^\[(.*)\] ${fns[0]}$"
+        if [[ ! "${extsize}" =~ ${extsize_regex} ]]; then
+            echo "extsize doesn't match extsize_regex: ${extsize}" >&2
+            return 2
+        fi
+        extsize="${BASH_REMATCH[1]}"
+
+        if [ "${extsize}" -ne "${expected_extsize}" ]; then
+            echo "FAIL: alloc_hint: actual ${extsize}, expected ${expected_extsize}" >&2
+            return 1
+        fi
+    done
+}
+
+#
+# Global setup
+#
+
+EC_K="2"
+EC_M="1"
+NUM_OSDS="$((EC_K + EC_M))"
+
+NUM_PG="12"
+NUM_PGP="${NUM_PG}"
+
+LOW_CAP="$(get_conf_val "osd.0" "filestore_max_alloc_hint_size")"
+HIGH_CAP="$((LOW_CAP * 10))" # 10M, assuming 1M default cap
+SMALL_HINT="$((LOW_CAP / 4))" # 256K, assuming 1M default cap
+BIG_HINT="$((LOW_CAP * 6))" # 6M, assuming 1M default cap
+
+setup_osd_data
+
+#
+# ReplicatedBackend tests
+#
+
+POOL="alloc_hint-rep"
+ceph osd pool create "${POOL}" "${NUM_PG}"
+ceph osd pool set "${POOL}" size "${NUM_OSDS}"
+ceph osd pool application enable "${POOL}" rados
+
+OBJ="foo"
+setup_pgid "${POOL}" "${OBJ}"
+rados -p "${POOL}" create "${OBJ}"
+
+# Empty object, SMALL_HINT - expect SMALL_HINT
+rados -p "${POOL}" set-alloc-hint "${OBJ}" "${SMALL_HINT}" "${SMALL_HINT}"
+expect_alloc_hint_eq "${SMALL_HINT}"
+
+# Try changing to BIG_HINT (1) - expect LOW_CAP (BIG_HINT > LOW_CAP)
+rados -p "${POOL}" set-alloc-hint "${OBJ}" "${BIG_HINT}" "${BIG_HINT}"
+expect_alloc_hint_eq "${LOW_CAP}"
+
+# Bump the cap to HIGH_CAP
+ceph tell 'osd.*' injectargs "--filestore_max_alloc_hint_size ${HIGH_CAP}"
+
+# Try changing to BIG_HINT (2) - expect BIG_HINT (BIG_HINT < HIGH_CAP)
+rados -p "${POOL}" set-alloc-hint "${OBJ}" "${BIG_HINT}" "${BIG_HINT}"
+expect_alloc_hint_eq "${BIG_HINT}"
+
+ceph tell 'osd.*' injectargs "--filestore_max_alloc_hint_size ${LOW_CAP}"
+
+# Populate object with some data
+rados -p "${POOL}" put "${OBJ}" /etc/passwd
+
+# Try changing back to SMALL_HINT - expect BIG_HINT (non-empty object)
+rados -p "${POOL}" set-alloc-hint "${OBJ}" "${SMALL_HINT}" "${SMALL_HINT}"
+expect_alloc_hint_eq "${BIG_HINT}"
+
+OBJ="bar"
+setup_pgid "${POOL}" "${OBJ}"
+
+# Non-existent object, SMALL_HINT - expect SMALL_HINT (object creation)
+rados -p "${POOL}" set-alloc-hint "${OBJ}" "${SMALL_HINT}" "${SMALL_HINT}"
+expect_alloc_hint_eq "${SMALL_HINT}"
+
+ceph osd pool delete "${POOL}" "${POOL}" --yes-i-really-really-mean-it
+
+#
+# ECBackend tests
+#
+
+PROFILE="alloc_hint-ecprofile"
+POOL="alloc_hint-ec"
+ceph osd erasure-code-profile set "${PROFILE}" k=2 m=1 crush-failure-domain=osd
+ceph osd erasure-code-profile get "${PROFILE}" # just so it's logged
+ceph osd pool create "${POOL}" "${NUM_PG}" "${NUM_PGP}" erasure "${PROFILE}"
+ceph osd pool application enable "${POOL}" rados
+
+OBJ="baz"
+setup_pgid "${POOL}" "${OBJ}"
+rados -p "${POOL}" create "${OBJ}"
+
+# Empty object, SMALL_HINT - expect scaled-down SMALL_HINT
+rados -p "${POOL}" set-alloc-hint "${OBJ}" "${SMALL_HINT}" "${SMALL_HINT}"
+expect_alloc_hint_eq "$((SMALL_HINT / EC_K))"
+
+ceph osd pool delete "${POOL}" "${POOL}" --yes-i-really-really-mean-it
+
+#
+# Global teardown
+#
+
+echo "OK"
diff --git a/qa/workunits/rados/test_cache_pool.sh b/qa/workunits/rados/test_cache_pool.sh
new file mode 100755
index 00000000..5e28b355
--- /dev/null
+++ b/qa/workunits/rados/test_cache_pool.sh
@@ -0,0 +1,170 @@
+#!/usr/bin/env bash
+
+set -ex
+
+expect_false()
+{
+	set -x
+	if "$@"; then return 1; else return 0; fi
+}
+
+# create pools, set up tier relationship
+ceph osd pool create base_pool 2
+ceph osd pool application enable base_pool rados
+ceph osd pool create partial_wrong 2
+ceph osd pool create wrong_cache 2
+ceph osd tier add base_pool partial_wrong
+ceph osd tier add base_pool wrong_cache
+
+# populate base_pool with some data
+echo "foo" > foo.txt
+echo "bar" > bar.txt
+echo "baz" > baz.txt
+rados -p base_pool put fooobj foo.txt
+rados -p base_pool put barobj bar.txt
+# fill in wrong_cache backwards so we can tell we read from it
+rados -p wrong_cache put fooobj bar.txt
+rados -p wrong_cache put barobj foo.txt
+# partial_wrong gets barobj backwards so we can check promote and non-promote
+rados -p partial_wrong put barobj foo.txt
+
+# get the objects back before setting a caching pool
+rados -p base_pool get fooobj tmp.txt
+diff -q tmp.txt foo.txt
+rados -p base_pool get barobj tmp.txt
+diff -q tmp.txt bar.txt
+
+# set up redirect and make sure we get backwards results
+ceph osd tier set-overlay base_pool wrong_cache
+ceph osd tier cache-mode wrong_cache writeback
+rados -p base_pool get fooobj tmp.txt
+diff -q tmp.txt bar.txt
+rados -p base_pool get barobj tmp.txt
+diff -q tmp.txt foo.txt
+
+# switch cache pools and make sure we're doing promote
+ceph osd tier remove-overlay base_pool
+ceph osd tier set-overlay base_pool partial_wrong
+ceph osd tier cache-mode partial_wrong writeback
+rados -p base_pool get fooobj tmp.txt
+diff -q tmp.txt foo.txt # hurray, it promoted!
+rados -p base_pool get barobj tmp.txt
+diff -q tmp.txt foo.txt # yep, we read partial_wrong's local object!
+
+# try a nonexistent object and make sure we get an error
+expect_false rados -p base_pool get bazobj tmp.txt
+
+# drop the cache entirely and make sure contents are still the same
+ceph osd tier remove-overlay base_pool
+rados -p base_pool get fooobj tmp.txt
+diff -q tmp.txt foo.txt
+rados -p base_pool get barobj tmp.txt
+diff -q tmp.txt bar.txt
+
+# create an empty cache pool and make sure it has objects after reading
+ceph osd pool create empty_cache 2
+
+touch empty.txt
+rados -p empty_cache ls > tmp.txt
+diff -q tmp.txt empty.txt
+
+ceph osd tier add base_pool empty_cache
+ceph osd tier set-overlay base_pool empty_cache
+ceph osd tier cache-mode empty_cache writeback
+rados -p base_pool get fooobj tmp.txt
+rados -p base_pool get barobj tmp.txt
+expect_false rados -p base_pool get bazobj tmp.txt
+
+rados -p empty_cache ls > tmp.txt
+expect_false diff -q tmp.txt empty.txt
+
+# cleanup
+ceph osd tier remove-overlay base_pool
+ceph osd tier remove base_pool wrong_cache
+ceph osd tier remove base_pool partial_wrong
+ceph osd tier remove base_pool empty_cache
+ceph osd pool delete base_pool base_pool --yes-i-really-really-mean-it
+ceph osd pool delete empty_cache empty_cache --yes-i-really-really-mean-it
+ceph osd pool delete wrong_cache wrong_cache --yes-i-really-really-mean-it
+ceph osd pool delete partial_wrong partial_wrong --yes-i-really-really-mean-it
+
+## set of base, cache
+ceph osd pool create base 8
+ceph osd pool application enable base rados
+ceph osd pool create cache 8
+
+ceph osd tier add base cache
+ceph osd tier cache-mode cache writeback
+ceph osd tier set-overlay base cache
+
+# cache-flush, cache-evict
+rados -p base put foo /etc/passwd
+expect_false rados -p base cache-evict foo
+expect_false rados -p base cache-flush foo
+expect_false rados -p cache cache-evict foo
+rados -p cache cache-flush foo
+rados -p cache cache-evict foo
+rados -p cache ls - | wc -l | grep 0
+
+# cache-try-flush, cache-evict
+rados -p base put foo /etc/passwd
+expect_false rados -p base cache-evict foo
+expect_false rados -p base cache-flush foo
+expect_false rados -p cache cache-evict foo
+rados -p cache cache-try-flush foo
+rados -p cache cache-evict foo
+rados -p cache ls - | wc -l | grep 0
+
+# cache-flush-evict-all
+rados -p base put bar /etc/passwd
+rados -p cache ls - | wc -l | grep 1
+expect_false rados -p base cache-flush-evict-all
+rados -p cache cache-flush-evict-all
+rados -p cache ls - | wc -l | grep 0
+
+# cache-try-flush-evict-all
+rados -p base put bar /etc/passwd
+rados -p cache ls - | wc -l | grep 1
+expect_false rados -p base cache-flush-evict-all
+rados -p cache cache-try-flush-evict-all
+rados -p cache ls - | wc -l | grep 0
+
+# cache flush/evit when clone objects exist
+rados -p base put testclone /etc/passwd
+rados -p cache ls - | wc -l | grep 1
+ceph osd pool mksnap base snap
+rados -p base put testclone /etc/hosts
+rados -p cache cache-flush-evict-all
+rados -p cache ls - | wc -l | grep 0
+
+ceph osd tier cache-mode cache forward --yes-i-really-mean-it
+rados -p base -s snap get testclone testclone.txt
+diff -q testclone.txt /etc/passwd
+rados -p base get testclone testclone.txt
+diff -q testclone.txt /etc/hosts
+
+# test --with-clones option
+ceph osd tier cache-mode cache writeback
+rados -p base put testclone2 /etc/passwd
+rados -p cache ls - | wc -l | grep 1
+ceph osd pool mksnap base snap1
+rados -p base put testclone2 /etc/hosts
+expect_false rados -p cache cache-flush testclone2
+rados -p cache cache-flush testclone2 --with-clones
+expect_false rados -p cache cache-evict testclone2
+rados -p cache cache-evict testclone2 --with-clones
+rados -p cache ls - | wc -l | grep 0
+
+rados -p base -s snap1 get testclone2 testclone2.txt
+diff -q testclone2.txt /etc/passwd
+rados -p base get testclone2 testclone2.txt
+diff -q testclone2.txt /etc/hosts
+
+# cleanup
+ceph osd tier remove-overlay base
+ceph osd tier remove base cache
+
+ceph osd pool delete cache cache --yes-i-really-really-mean-it
+ceph osd pool delete base base --yes-i-really-really-mean-it
+
+echo OK
diff --git a/qa/workunits/rados/test_crash.sh b/qa/workunits/rados/test_crash.sh
new file mode 100755
index 00000000..6608d787
--- /dev/null
+++ b/qa/workunits/rados/test_crash.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+set -x
+
+# run on a single-node three-OSD cluster
+
+sudo killall -ABRT ceph-osd
+sleep 5
+
+# kill caused coredumps; find them and delete them, carefully, so as
+# not to disturb other coredumps, or else teuthology will see them
+# and assume test failure.  sudos are because the core files are
+# root/600
+for f in $(find $TESTDIR/archive/coredump -type f); do
+	gdb_output=$(echo "quit" | sudo gdb /usr/bin/ceph-osd $f)
+	if expr match "$gdb_output" ".*generated.*ceph-osd.*" && \
+	   ( \
+
+	   	expr match "$gdb_output" ".*terminated.*signal 6.*" || \
+	   	expr match "$gdb_output" ".*terminated.*signal SIGABRT.*" \
+	   )
+	then
+		sudo rm $f
+	fi
+done
+
+# let daemon find crashdumps on startup
+sudo systemctl restart ceph-crash
+sleep 30
+
+# must be 3 crashdumps registered and moved to crash/posted
+[ $(ceph crash ls | wc -l) = 4 ]  || exit 1   # 4 here bc of the table header
+[ $(sudo find /var/lib/ceph/crash/posted/ -name meta | wc -l) = 3 ] || exit 1
+
+# there should be a health warning
+ceph health detail | grep RECENT_CRASH || exit 1
+ceph crash archive-all
+sleep 30
+ceph health detail | grep -c RECENT_CRASH | grep 0     # should be gone!
diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh
new file mode 100755
index 00000000..25994401
--- /dev/null
+++ b/qa/workunits/rados/test_dedup_tool.sh
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+
+set -x
+
+die() {
+    echo "$@"
+    exit 1
+}
+
+do_run() {
+    if [ "$1" == "--tee" ]; then
+      shift
+      tee_out="$1"
+      shift
+      "$@" | tee $tee_out
+    else
+      "$@"
+    fi
+}
+
+run_expect_succ() {
+    echo "RUN_EXPECT_SUCC: " "$@"
+    do_run "$@"
+    [ $? -ne 0 ] && die "expected success, but got failure! cmd: $@"
+}
+
+run() {
+    echo "RUN: " $@
+    do_run "$@"
+}
+
+if [ -n "$CEPH_BIN" ] ; then
+   # CMake env
+   RADOS_TOOL="$CEPH_BIN/rados"
+   CEPH_TOOL="$CEPH_BIN/ceph"
+   DEDUP_TOOL="$CEPH_BIN/cephdeduptool"
+else
+   # executables should be installed by the QA env 
+   RADOS_TOOL=$(which rados)
+   CEPH_TOOL=$(which ceph)
+   DEDUP_TOOL=$(which cephdeduptool)
+fi
+
+POOL=dedup_pool
+OBJ=test_rados_obj
+
+[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test"
+[ -x "$CEPH_TOOL" ] || die "couldn't find $CEPH_TOOL binary to test"
+
+run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8
+
+function test_dedup_ratio_fixed()
+{
+  # case 1
+  dd if=/dev/urandom of=dedup_object_1k bs=1K count=1
+  dd if=dedup_object_1k of=dedup_object_100k bs=1K count=100
+
+  $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_100k
+  RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 1024  --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}')
+  if [ 1024 -ne $RESULT ];
+  then
+    die "Estimate failed expecting 1024 result $RESULT"
+  fi
+
+  # case 2
+  dd if=/dev/zero of=dedup_object_10m bs=10M count=1
+
+  $RADOS_TOOL -p $POOL put $OBJ ./dedup_object_10m
+  RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096  --chunk-algorithm fixed --fingerprint-algorithm sha1 --debug | grep result | awk '{print$4}')
+  if [ 4096 -ne $RESULT ];
+  then
+    die "Estimate failed expecting 4096 result $RESULT"
+  fi
+
+  # case 3 max_thread
+  for num in `seq 0 20`
+  do
+    dd if=/dev/zero of=dedup_object_$num bs=4M count=1
+    $RADOS_TOOL -p $POOL put dedup_object_$num ./dedup_object_$num
+  done
+
+  RESULT=$($DEDUP_TOOL --op estimate --pool $POOL --chunk-size 4096  --chunk-algorithm fixed --fingerprint-algorithm sha1 --max-thread 4 --debug | grep result | awk '{print$2}')
+
+  if [ 98566144 -ne $RESULT ];
+  then
+    die "Estimate failed expecting 98566144 result $RESULT"
+  fi
+
+  rm -rf ./dedup_object_1k ./dedup_object_100k ./dedup_object_10m
+  for num in `seq 0 20`
+  do
+    rm -rf ./dedup_object_$num
+  done
+}
+
+function test_dedup_chunk_scrub()
+{
+
+  CHUNK_POOL=dedup_chunk_pool
+  run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8
+
+  echo "hi there" > foo
+
+  echo "hi there" > bar
+ 
+  echo "there" > foo-chunk
+
+  echo "CHUNK" > bar-chunk
+  
+  $CEPH_TOOL osd pool set $POOL fingerprint_algorithm sha1 --yes-i-really-mean-it
+
+  $RADOS_TOOL -p $POOL put foo ./foo
+  $RADOS_TOOL -p $POOL put bar ./bar
+
+  $RADOS_TOOL -p $CHUNK_POOL put bar-chunk ./bar-chunk
+  $RADOS_TOOL -p $CHUNK_POOL put foo-chunk ./foo-chunk
+
+  $RADOS_TOOL -p $POOL set-chunk bar 0 8 --target-pool $CHUNK_POOL bar-chunk 0 --with-reference
+  $RADOS_TOOL -p $POOL set-chunk foo 0 8 --target-pool $CHUNK_POOL foo-chunk 0 --with-reference
+
+  echo "There hi" > test_obj
+  # dirty
+  $RADOS_TOOL -p $POOL put foo ./test_obj
+  # flush
+  $RADOS_TOOL -p $POOL put foo ./test_obj
+  sleep 2
+
+  rados ls -p $CHUNK_POOL
+  CHUNK_OID=$(echo -n "There hi" | sha1sum)
+
+  $DEDUP_TOOL --op add_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID --target_ref bar
+  RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID)
+
+  $DEDUP_TOOL --op chunk_scrub --pool $POOL --chunk_pool $CHUNK_POOL
+
+  RESULT=$($DEDUP_TOOL --op get_chunk_ref --pool $POOL --chunk_pool $CHUNK_POOL --object $CHUNK_OID | grep bar)
+  if [ -n "$RESULT" ] ; then
+    $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+    $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+    die "Scrub failed expecting bar is removed"
+  fi
+
+  $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it
+  
+  rm -rf ./foo ./bar ./foo-chunk ./bar-chunk ./test_obj
+}
+
+test_dedup_ratio_fixed
+test_dedup_chunk_scrub
+
+$CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+
+echo "SUCCESS!"
+exit 0
+
+
diff --git a/qa/workunits/rados/test_envlibrados_for_rocksdb.sh b/qa/workunits/rados/test_envlibrados_for_rocksdb.sh
new file mode 100755
index 00000000..7099dafb
--- /dev/null
+++ b/qa/workunits/rados/test_envlibrados_for_rocksdb.sh
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+set -ex
+
+############################################
+#			Helper functions
+############################################
+source $(dirname $0)/../ceph-helpers-root.sh
+
+############################################
+#			Install required tools
+############################################
+echo "Install required tools"
+
+CURRENT_PATH=`pwd`
+
+############################################
+#			Compile&Start RocksDB
+############################################
+# install prerequisites
+# for rocksdb
+case $(distro_id) in
+	ubuntu|debian|devuan)
+		install git g++ libsnappy-dev zlib1g-dev libbz2-dev libradospp-dev cmake
+		;;
+	centos|fedora|rhel)
+		install git gcc-c++.x86_64 snappy-devel zlib zlib-devel bzip2 bzip2-devel libradospp-devel.x86_64
+        if [ $(distro_id) = "fedora" ]; then
+            install cmake
+        else
+            install_cmake3_on_centos7
+        fi
+		;;
+	opensuse*|suse|sles)
+		install git gcc-c++ snappy-devel zlib-devel libbz2-devel libradospp-devel
+		;;
+	*)
+        echo "$(distro_id) is unknown, $@ will have to be installed manually."
+        ;;
+esac
+
+# # gflags
+# sudo yum install gflags-devel
+# 
+# wget https://github.com/schuhschuh/gflags/archive/master.zip
+# unzip master.zip
+# cd gflags-master
+# mkdir build && cd build
+# export CXXFLAGS="-fPIC" && cmake .. && make VERBOSE=1
+# make && make install
+
+# # snappy-devel
+
+
+echo "Compile rocksdb"
+if [ -e rocksdb ]; then
+	rm -fr rocksdb
+fi
+
+pushd $(dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/bash.sh)/../../../
+git submodule update --init src/rocksdb
+popd
+git clone $(dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/bash.sh)/../../../src/rocksdb rocksdb
+
+# compile code
+cd rocksdb
+if type cmake3 > /dev/null 2>&1 ; then
+    CMAKE=cmake3
+else
+    CMAKE=cmake
+fi
+mkdir build && cd build && ${CMAKE} -DWITH_LIBRADOS=ON -DWITH_SNAPPY=ON -DWITH_GFLAGS=OFF -DFAIL_ON_WARNINGS=OFF ..
+make rocksdb_env_librados_test -j8
+
+echo "Copy ceph.conf"
+# prepare ceph.conf
+mkdir -p ../ceph/src/
+if [ -f "/etc/ceph/ceph.conf" ]; then
+    cp /etc/ceph/ceph.conf ../ceph/src/
+elif [ -f "/etc/ceph/ceph/ceph.conf" ]; then
+	cp /etc/ceph/ceph/ceph.conf ../ceph/src/
+else 
+	echo "/etc/ceph/ceph/ceph.conf doesn't exist"
+fi
+
+echo "Run EnvLibrados test"
+# run test
+if [ -f "../ceph/src/ceph.conf" ]
+	then
+	cp env_librados_test ~/cephtest/archive
+	./env_librados_test
+else 
+	echo "../ceph/src/ceph.conf doesn't exist"
+fi
+cd ${CURRENT_PATH}
diff --git a/qa/workunits/rados/test_hang.sh b/qa/workunits/rados/test_hang.sh
new file mode 100755
index 00000000..724e0bb8
--- /dev/null
+++ b/qa/workunits/rados/test_hang.sh
@@ -0,0 +1,8 @@
+#!/bin/sh -ex
+
+# Hang forever for manual testing using the thrasher
+while(true)
+do
+   sleep 300
+done
+exit 0
diff --git a/qa/workunits/rados/test_health_warnings.sh b/qa/workunits/rados/test_health_warnings.sh
new file mode 100755
index 00000000..d393e5c6
--- /dev/null
+++ b/qa/workunits/rados/test_health_warnings.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -uex
+
+# number of osds = 10
+crushtool -o crushmap --build --num_osds 10 host straw 2 rack straw 2 row straw 2 root straw 0
+ceph osd setcrushmap -i crushmap
+ceph osd tree
+ceph tell osd.* injectargs --osd_max_markdown_count 1024 --osd_max_markdown_period 1
+ceph osd set noout
+
+wait_for_healthy() {
+  while ceph health | grep down
+  do
+    sleep 1
+  done
+}
+
+test_mark_two_osds_same_host_down() {
+  ceph osd set noup
+  ceph osd down osd.0 osd.1
+  ceph health detail
+  ceph health | grep "1 host"
+  ceph health | grep "2 osds"
+  ceph health detail | grep "osd.0"
+  ceph health detail | grep "osd.1"
+  ceph osd unset noup
+  wait_for_healthy
+}
+
+test_mark_two_osds_same_rack_down() {
+  ceph osd set noup
+  ceph osd down osd.8 osd.9
+  ceph health detail
+  ceph health | grep "1 host"
+  ceph health | grep "1 rack"
+  ceph health | grep "1 row"
+  ceph health | grep "2 osds"
+  ceph health detail | grep "osd.8"
+  ceph health detail | grep "osd.9"
+  ceph osd unset noup
+  wait_for_healthy
+}
+
+test_mark_all_but_last_osds_down() {
+  ceph osd set noup
+  ceph osd down $(ceph osd ls | sed \$d)
+  ceph health detail
+  ceph health | grep "1 row"
+  ceph health | grep "2 racks"
+  ceph health | grep "4 hosts"
+  ceph health | grep "9 osds"
+  ceph osd unset noup
+  wait_for_healthy
+}
+
+test_mark_two_osds_same_host_down_with_classes() {
+    ceph osd set noup
+    ceph osd crush set-device-class ssd osd.0 osd.2 osd.4 osd.6 osd.8
+    ceph osd crush set-device-class hdd osd.1 osd.3 osd.5 osd.7 osd.9
+    ceph osd down osd.0 osd.1
+    ceph health detail
+    ceph health | grep "1 host"
+    ceph health | grep "2 osds"
+    ceph health detail | grep "osd.0"
+    ceph health detail | grep "osd.1"
+    ceph osd unset noup
+    wait_for_healthy
+}
+
+test_mark_two_osds_same_host_down
+test_mark_two_osds_same_rack_down
+test_mark_all_but_last_osds_down
+test_mark_two_osds_same_host_down_with_classes
+
+exit 0
diff --git a/qa/workunits/rados/test_large_omap_detection.py b/qa/workunits/rados/test_large_omap_detection.py
new file mode 100755
index 00000000..c6cf195d
--- /dev/null
+++ b/qa/workunits/rados/test_large_omap_detection.py
@@ -0,0 +1,134 @@
+#!/usr/bin/python
+# -*- mode:python -*-
+# vim: ts=4 sw=4 smarttab expandtab
+#
+# Copyright (C) 2017 Red Hat <contact@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Library Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library Public License for more details.
+#
+
+import json
+import rados
+import shlex
+import subprocess
+import time
+
+def cleanup(cluster):
+    cluster.delete_pool('large-omap-test-pool')
+    cluster.shutdown()
+
+def init():
+    # For local testing
+    #cluster = rados.Rados(conffile='./ceph.conf')
+    cluster = rados.Rados(conffile='/etc/ceph/ceph.conf')
+    cluster.connect()
+    print("\nCluster ID: " + cluster.get_fsid())
+    cluster.create_pool('large-omap-test-pool')
+    ioctx = cluster.open_ioctx('large-omap-test-pool')
+    ioctx.write_full('large-omap-test-object1', "Lorem ipsum")
+    op = ioctx.create_write_op()
+
+    keys = []
+    values = []
+    for x in range(20001):
+        keys.append(str(x))
+        values.append("X")
+
+    ioctx.set_omap(op, tuple(keys), tuple(values))
+    ioctx.operate_write_op(op, 'large-omap-test-object1', 0)
+    ioctx.release_write_op(op)
+
+    ioctx.write_full('large-omap-test-object2', "Lorem ipsum dolor")
+    op = ioctx.create_write_op()
+
+    buffer = ("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do "
+              "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut "
+              "enim ad minim veniam, quis nostrud exercitation ullamco laboris "
+              "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in "
+              "reprehenderit in voluptate velit esse cillum dolore eu fugiat "
+              "nulla pariatur. Excepteur sint occaecat cupidatat non proident, "
+              "sunt in culpa qui officia deserunt mollit anim id est laborum.")
+
+    keys = []
+    values = []
+    for x in range(20000):
+        keys.append(str(x))
+        values.append(buffer)
+
+    ioctx.set_omap(op, tuple(keys), tuple(values))
+    ioctx.operate_write_op(op, 'large-omap-test-object2', 0)
+    ioctx.release_write_op(op)
+    ioctx.close()
+    return cluster
+
+def get_deep_scrub_timestamp(pgid):
+    cmd = ['ceph', 'pg', 'dump', '--format=json-pretty']
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    out = proc.communicate()[0]
+    try:
+        pgstats = json.loads(out)['pg_map']['pg_stats']
+    except KeyError:
+        pgstats = json.loads(out)['pg_stats']
+    for stat in pgstats:
+        if stat['pgid'] == pgid:
+            return stat['last_deep_scrub_stamp']
+
+def wait_for_scrub():
+    osds = set();
+    pgs = dict();
+    cmd = ['ceph', 'osd', 'map', 'large-omap-test-pool',
+           'large-omap-test-object1', '--format=json-pretty']
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    out = proc.communicate()[0]
+    osds.add(json.loads(out)['acting_primary'])
+    pgs[json.loads(out)['pgid']] = get_deep_scrub_timestamp(json.loads(out)['pgid'])
+    cmd = ['ceph', 'osd', 'map', 'large-omap-test-pool',
+           'large-omap-test-object2', '--format=json-pretty']
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+    out = proc.communicate()[0]
+    osds.add(json.loads(out)['acting_primary'])
+    pgs[json.loads(out)['pgid']] = get_deep_scrub_timestamp(json.loads(out)['pgid'])
+
+    for pg in pgs:
+        command = "ceph pg deep-scrub " + str(pg)
+        subprocess.check_call(shlex.split(command))
+
+    for pg in pgs:
+        RETRIES = 0
+        while RETRIES < 60 and pgs[pg] == get_deep_scrub_timestamp(pg):
+            time.sleep(10)
+            RETRIES += 1
+
+def check_health_output():
+    RETRIES = 0
+    result = 0
+    while RETRIES < 6 and result != 2:
+        result = 0
+        RETRIES += 1
+        output = subprocess.check_output(["ceph", "health", "detail"])
+        for line in output.splitlines():
+            result += int(line.find('2 large omap objects') != -1)
+        time.sleep(10)
+
+    if result != 2:
+        print("Error, got invalid output:")
+        print(output)
+        raise Exception
+
+def main():
+    cluster = init()
+    wait_for_scrub()
+    check_health_output()
+
+    cleanup(cluster)
+
+if __name__ == '__main__':
+    main()
diff --git a/qa/workunits/rados/test_librados_build.sh b/qa/workunits/rados/test_librados_build.sh
new file mode 100755
index 00000000..0bca5050
--- /dev/null
+++ b/qa/workunits/rados/test_librados_build.sh
@@ -0,0 +1,74 @@
+#!/bin/bash -ex
+#
+# Compile and run a librados application outside of the ceph build system, so
+# that we can be sure librados.h[pp] is still usable and hasn't accidentally
+# started depending on internal headers.
+#
+# The script assumes all dependencies - e.g. curl, make, gcc, librados headers,
+# libradosstriper headers, boost headers, etc. - are already installed.
+#
+
+source $(dirname $0)/../ceph-helpers-root.sh
+
+trap cleanup EXIT
+
+SOURCES="hello_radosstriper.cc
+hello_world_c.c
+hello_world.cc
+Makefile
+"
+BINARIES_TO_RUN="hello_world_c
+hello_world_cpp
+"
+BINARIES="${BINARIES_TO_RUN}hello_radosstriper_cpp
+"
+DL_PREFIX="http://git.ceph.com/?p=ceph.git;a=blob_plain;hb=nautilus;f=examples/librados/"
+#DL_PREFIX="https://raw.githubusercontent.com/ceph/ceph/nautilus/examples/librados/"
+DESTDIR=$(pwd)
+
+function cleanup () {
+    for f in $BINARIES$SOURCES ; do
+        rm -f "${DESTDIR}/$f"
+    done
+}
+
+function get_sources () {
+    for s in $SOURCES ; do
+        curl --progress-bar --output $s ${DL_PREFIX}$s
+    done
+}
+
+function check_sources () {
+    for s in $SOURCES ; do
+        test -f $s
+    done
+}
+
+function check_binaries () {
+    for b in $BINARIES ; do
+        file $b
+        test -f $b
+    done
+}
+
+function run_binaries () {
+    for b in $BINARIES_TO_RUN ; do
+        ./$b -c /etc/ceph/ceph.conf
+    done
+}
+
+pushd $DESTDIR
+case $(distro_id) in
+    centos|fedora|rhel|opensuse*|suse|sles)
+        install gcc-c++ make libradospp-devel librados-devel;;
+    ubuntu|debian|devuan)
+        install g++ make libradospp-dev librados-dev;;
+    *)
+        echo "$(distro_id) is unknown, $@ will have to be installed manually."
+esac
+get_sources
+check_sources
+make all-system
+check_binaries
+run_binaries
+popd
diff --git a/qa/workunits/rados/test_pool_access.sh b/qa/workunits/rados/test_pool_access.sh
new file mode 100755
index 00000000..2a7077a4
--- /dev/null
+++ b/qa/workunits/rados/test_pool_access.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+set -ex
+
+KEYRING=$(mktemp)
+trap cleanup EXIT ERR HUP INT QUIT
+
+cleanup() {
+    (ceph auth del client.mon_read || true) >/dev/null 2>&1
+    (ceph auth del client.mon_write || true) >/dev/null 2>&1
+
+    rm -f $KEYRING
+}
+
+expect_false()
+{
+	set -x
+	if "$@"; then return 1; else return 0; fi
+}
+
+create_pool_op() {
+  ID=$1
+  POOL=$2
+
+  cat << EOF | CEPH_ARGS="-k $KEYRING" python
+import rados
+
+cluster = rados.Rados(conffile="", rados_id="${ID}")
+cluster.connect()
+cluster.create_pool("${POOL}")
+EOF
+}
+
+delete_pool_op() {
+  ID=$1
+  POOL=$2
+
+  cat << EOF | CEPH_ARGS="-k $KEYRING" python
+import rados
+
+cluster = rados.Rados(conffile="", rados_id="${ID}")
+cluster.connect()
+cluster.delete_pool("${POOL}")
+EOF
+}
+
+create_pool_snap_op() {
+  ID=$1
+  POOL=$2
+  SNAP=$3
+
+  cat << EOF | CEPH_ARGS="-k $KEYRING" python
+import rados
+
+cluster = rados.Rados(conffile="", rados_id="${ID}")
+cluster.connect()
+ioctx = cluster.open_ioctx("${POOL}")
+
+ioctx.create_snap("${SNAP}")
+EOF
+}
+
+remove_pool_snap_op() {
+  ID=$1
+  POOL=$2
+  SNAP=$3
+
+  cat << EOF | CEPH_ARGS="-k $KEYRING" python
+import rados
+
+cluster = rados.Rados(conffile="", rados_id="${ID}")
+cluster.connect()
+ioctx = cluster.open_ioctx("${POOL}")
+
+ioctx.remove_snap("${SNAP}")
+EOF
+}
+
+test_pool_op()
+{
+    ceph auth get-or-create client.mon_read mon 'allow r' >> $KEYRING
+    ceph auth get-or-create client.mon_write mon 'allow *' >> $KEYRING
+
+    expect_false create_pool_op mon_read pool1
+    create_pool_op mon_write pool1
+
+    expect_false create_pool_snap_op mon_read pool1 snap1
+    create_pool_snap_op mon_write pool1 snap1
+
+    expect_false remove_pool_snap_op mon_read pool1 snap1
+    remove_pool_snap_op mon_write pool1 snap1
+
+    expect_false delete_pool_op mon_read pool1
+    delete_pool_op mon_write pool1
+}
+
+key=`ceph auth get-or-create-key client.poolaccess1 mon 'allow r' osd 'allow *'`
+rados --id poolaccess1 --key $key -p rbd ls
+
+key=`ceph auth get-or-create-key client.poolaccess2 mon 'allow r' osd 'allow * pool=nopool'`
+expect_false rados --id poolaccess2 --key $key -p rbd ls
+
+key=`ceph auth get-or-create-key client.poolaccess3 mon 'allow r' osd 'allow rw pool=nopool'`
+expect_false rados --id poolaccess3 --key $key -p rbd ls
+
+test_pool_op
+
+echo OK
diff --git a/qa/workunits/rados/test_pool_quota.sh b/qa/workunits/rados/test_pool_quota.sh
new file mode 100755
index 00000000..0eacefc6
--- /dev/null
+++ b/qa/workunits/rados/test_pool_quota.sh
@@ -0,0 +1,68 @@
+#!/bin/sh -ex
+
+p=`uuidgen`
+
+# objects
+ceph osd pool create $p 12
+ceph osd pool set-quota $p max_objects 10
+ceph osd pool application enable $p rados
+
+for f in `seq 1 10` ; do
+ rados -p $p put obj$f /etc/passwd
+done
+
+sleep 30
+
+rados -p $p put onemore /etc/passwd  &
+pid=$!
+
+ceph osd pool set-quota $p max_objects 100
+wait $pid 
+[ $? -ne 0 ] && exit 1 || true
+
+rados -p $p put twomore /etc/passwd
+
+# bytes
+ceph osd pool set-quota $p max_bytes 100
+sleep 30
+
+rados -p $p put two /etc/passwd &
+pid=$!
+
+ceph osd pool set-quota $p max_bytes 0
+ceph osd pool set-quota $p max_objects 0
+wait $pid 
+[ $? -ne 0 ] && exit 1 || true
+
+rados -p $p put three /etc/passwd
+
+
+#one pool being full does not block a different pool
+
+pp=`uuidgen`
+
+ceph osd pool create $pp 12
+ceph osd pool application enable $pp rados
+
+# set objects quota 
+ceph osd pool set-quota $pp max_objects 10
+sleep 30
+
+for f in `seq 1 10` ; do
+ rados -p $pp put obj$f /etc/passwd
+done
+
+sleep 30
+
+rados -p $p put threemore /etc/passwd 
+
+ceph osd pool set-quota $p max_bytes 0
+ceph osd pool set-quota $p max_objects 0
+
+sleep 30
+# done
+ceph osd pool delete $p $p --yes-i-really-really-mean-it
+ceph osd pool delete $pp $pp --yes-i-really-really-mean-it
+
+echo OK
+
diff --git a/qa/workunits/rados/test_python.sh b/qa/workunits/rados/test_python.sh
new file mode 100755
index 00000000..80369c8d
--- /dev/null
+++ b/qa/workunits/rados/test_python.sh
@@ -0,0 +1,4 @@
+#!/bin/sh -ex
+
+${PYTHON:-python} -m nose -v $(dirname $0)/../../../src/test/pybind/test_rados.py
+exit 0
diff --git a/qa/workunits/rados/test_rados_timeouts.sh b/qa/workunits/rados/test_rados_timeouts.sh
new file mode 100755
index 00000000..327c7ab3
--- /dev/null
+++ b/qa/workunits/rados/test_rados_timeouts.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -x
+
+delay_mon() {
+    MSGTYPE=$1
+    shift
+    $@ --rados-mon-op-timeout 1 --ms-inject-delay-type mon --ms-inject-delay-max 10000000 --ms-inject-delay-probability 1 --ms-inject-delay-msg-type $MSGTYPE
+    if [ $? -eq 0 ]; then
+        exit 1
+    fi
+}
+
+delay_osd() {
+    MSGTYPE=$1
+    shift
+    $@ --rados-osd-op-timeout 1 --ms-inject-delay-type osd --ms-inject-delay-max 10000000 --ms-inject-delay-probability 1 --ms-inject-delay-msg-type $MSGTYPE
+    if [ $? -eq 0 ]; then
+        exit 2
+    fi
+}
+
+# pool ops
+delay_mon omap rados lspools
+delay_mon poolopreply ceph osd pool create test 8
+delay_mon poolopreply rados mksnap -p test snap
+delay_mon poolopreply ceph osd pool rm test test --yes-i-really-really-mean-it
+
+# other mon ops
+delay_mon getpoolstats rados df
+delay_mon mon_command ceph df
+delay_mon omap ceph osd dump
+delay_mon omap ceph -s
+
+# osd ops
+delay_osd osd_op_reply rados -p data put ls /bin/ls
+delay_osd osd_op_reply rados -p data get ls - >/dev/null
+delay_osd osd_op_reply rados -p data ls
+delay_osd command_reply ceph tell osd.0 bench 1 1
+
+# rbd commands, using more kinds of osd ops
+rbd create -s 1 test
+delay_osd osd_op_reply rbd watch test
+delay_osd osd_op_reply rbd info test
+delay_osd osd_op_reply rbd snap create test@snap
+delay_osd osd_op_reply rbd import /bin/ls ls
+rbd rm test
+
+echo OK
diff --git a/qa/workunits/rados/test_rados_tool.sh b/qa/workunits/rados/test_rados_tool.sh
new file mode 100755
index 00000000..5325743f
--- /dev/null
+++ b/qa/workunits/rados/test_rados_tool.sh
@@ -0,0 +1,924 @@
+#!/usr/bin/env bash
+
+set -x
+
+die() {
+    echo "$@"
+    exit 1
+}
+
+usage() {
+    cat <<EOF
+test_rados_tool.sh: tests rados_tool
+-c:        RADOS configuration file to use [optional]
+-k:        keep temp files
+-h:        this help message
+-p:        set temporary pool to use [optional]
+EOF
+}
+
+do_run() {
+    if [ "$1" == "--tee" ]; then
+      shift
+      tee_out="$1"
+      shift
+      "$@" | tee $tee_out
+    else
+      "$@"
+    fi
+}
+
+run_expect_fail() {
+    echo "RUN_EXPECT_FAIL: " "$@"
+    do_run "$@"
+    [ $? -eq 0 ] && die "expected failure, but got success! cmd: $@"
+}
+
+run_expect_succ() {
+    echo "RUN_EXPECT_SUCC: " "$@"
+    do_run "$@"
+    [ $? -ne 0 ] && die "expected success, but got failure! cmd: $@"
+}
+
+run_expect_nosignal() {
+    echo "RUN_EXPECT_NOSIGNAL: " "$@"
+    do_run "$@"
+    [ $? -ge 128 ] && die "expected success or fail, but got signal! cmd: $@"
+}
+
+run() {
+    echo "RUN: " $@
+    do_run "$@"
+}
+
+if [ -n "$CEPH_BIN" ] ; then
+   # CMake env
+   RADOS_TOOL="$CEPH_BIN/rados"
+   CEPH_TOOL="$CEPH_BIN/ceph"
+else
+   # executables should be installed by the QA env 
+   RADOS_TOOL=$(which rados)
+   CEPH_TOOL=$(which ceph)
+fi
+
+KEEP_TEMP_FILES=0
+POOL=trs_pool
+POOL_CP_TARGET=trs_pool.2
+POOL_EC=trs_pool_ec
+
+[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test"
+[ -x "$CEPH_TOOL" ] || die "couldn't find $CEPH_TOOL binary to test"
+
+while getopts  "c:hkp:" flag; do
+    case $flag in
+        c)  RADOS_TOOL="$RADOS_TOOL -c $OPTARG";;
+        k)  KEEP_TEMP_FILES=1;;
+        h)  usage; exit 0;;
+        p)  POOL=$OPTARG;;
+        *)  echo; usage; exit 1;;
+    esac
+done
+
+TDIR=`mktemp -d -t test_rados_tool.XXXXXXXXXX` || die "mktemp failed"
+[ $KEEP_TEMP_FILES -eq 0 ] && trap "rm -rf ${TDIR}; exit" INT TERM EXIT
+
+# ensure rados doesn't segfault without --pool
+run_expect_nosignal "$RADOS_TOOL" --snap "asdf" ls
+run_expect_nosignal "$RADOS_TOOL" --snapid "0" ls
+run_expect_nosignal "$RADOS_TOOL" --object_locator "asdf" ls
+run_expect_nosignal "$RADOS_TOOL" --namespace "asdf" ls
+
+run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8
+run_expect_succ "$CEPH_TOOL" osd erasure-code-profile set myprofile k=2 m=1 stripe_unit=2K crush-failure-domain=osd --force
+run_expect_succ "$CEPH_TOOL" osd pool create "$POOL_EC" 100 100 erasure myprofile
+
+
+# expb happens to be the empty export for legacy reasons
+run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/expb"
+
+# expa has objects foo, foo2 and bar
+run_expect_succ "$RADOS_TOOL" -p "$POOL" put foo /etc/fstab
+run_expect_succ "$RADOS_TOOL" -p "$POOL" put foo2 /etc/fstab
+run_expect_succ "$RADOS_TOOL" -p "$POOL" put bar /etc/fstab
+run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/expa"
+
+# expc has foo and foo2 with some attributes and omaps set
+run_expect_succ "$RADOS_TOOL" -p "$POOL" rm bar
+run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr foo "rados.toothbrush" "toothbrush"
+run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr foo "rados.toothpaste" "crest"
+run_expect_succ "$RADOS_TOOL" -p "$POOL" setomapval foo "rados.floss" "myfloss"
+run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr foo2 "rados.toothbrush" "green"
+run_expect_succ "$RADOS_TOOL" -p "$POOL" setomapheader foo2 "foo2.header"
+run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/expc"
+
+# make sure that --create works
+run "$CEPH_TOOL" osd pool rm "$POOL" "$POOL" --yes-i-really-really-mean-it
+run_expect_succ "$RADOS_TOOL" -p "$POOL" --create import "$TDIR/expa"
+
+# make sure that lack of --create fails
+run_expect_succ "$CEPH_TOOL" osd pool rm "$POOL" "$POOL" --yes-i-really-really-mean-it
+run_expect_fail "$RADOS_TOOL" -p "$POOL" import "$TDIR/expa"
+
+run_expect_succ "$RADOS_TOOL" -p "$POOL" --create import "$TDIR/expa"
+
+# inaccessible import src should fail
+run_expect_fail "$RADOS_TOOL" -p "$POOL" import "$TDIR/dir_nonexistent"
+
+# export an empty pool to test purge
+run_expect_succ "$RADOS_TOOL" purge "$POOL" --yes-i-really-really-mean-it
+run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/empty"
+cmp -s "$TDIR/expb" "$TDIR/empty" \
+    || die "failed to export the same stuff we imported!"
+rm -f "$TDIR/empty"
+
+# import some stuff with extended attributes on it
+run_expect_succ "$RADOS_TOOL" -p "$POOL" import "$TDIR/expc"
+VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"`
+[ ${VAL} = "toothbrush" ] || die "Invalid attribute after import"
+
+# the second time, the xattrs should match, so there should be nothing to do.
+run_expect_succ "$RADOS_TOOL" -p "$POOL" import "$TDIR/expc"
+VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"`
+[ "${VAL}" = "toothbrush" ] || die "Invalid attribute after second import"
+
+# Now try with --no-overwrite option after changing an attribute
+run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr foo "rados.toothbrush" "dentist"
+run_expect_succ "$RADOS_TOOL" -p "$POOL" import --no-overwrite "$TDIR/expc"
+VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"`
+[ "${VAL}" = "dentist" ] || die "Invalid attribute after second import"
+
+# now force it to copy everything
+run_expect_succ "$RADOS_TOOL" -p "$POOL" import "$TDIR/expc"
+VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"`
+[ "${VAL}" = "toothbrush" ] || die "Invalid attribute after second import"
+
+# test copy pool
+run "$CEPH_TOOL" osd pool rm "$POOL" "$POOL" --yes-i-really-really-mean-it
+run "$CEPH_TOOL" osd pool rm "$POOL_CP_TARGET" "$POOL_CP_TARGET" --yes-i-really-really-mean-it
+run_expect_succ "$CEPH_TOOL" osd pool create "$POOL" 8
+run_expect_succ "$CEPH_TOOL" osd pool create "$POOL_CP_TARGET" 8
+
+# create src files
+mkdir -p "$TDIR/dir_cp_src"
+for i in `seq 1 5`; do
+  fname="$TDIR/dir_cp_src/f.$i"
+  objname="f.$i"
+  dd if=/dev/urandom of="$fname" bs=$((1024*1024)) count=$i
+  run_expect_succ "$RADOS_TOOL" -p "$POOL" put $objname "$fname"
+
+# a few random attrs
+  for j in `seq 1 4`; do
+    rand_str=`dd if=/dev/urandom bs=4 count=1 | hexdump -x`
+    run_expect_succ "$RADOS_TOOL" -p "$POOL" setxattr $objname attr.$j "$rand_str"
+    run_expect_succ --tee "$fname.attr.$j" "$RADOS_TOOL" -p "$POOL" getxattr $objname attr.$j
+  done
+
+  rand_str=`dd if=/dev/urandom bs=4 count=1 | hexdump -x`
+  run_expect_succ "$RADOS_TOOL" -p "$POOL" setomapheader $objname "$rand_str"
+  run_expect_succ --tee "$fname.omap.header" "$RADOS_TOOL" -p "$POOL" getomapheader $objname
+
+# a few random omap keys
+  for j in `seq 1 4`; do
+    rand_str=`dd if=/dev/urandom bs=4 count=1 | hexdump -x`
+    run_expect_succ "$RADOS_TOOL" -p "$POOL" setomapval $objname key.$j "$rand_str"
+  done
+  run_expect_succ --tee "$fname.omap.vals" "$RADOS_TOOL" -p "$POOL" listomapvals $objname
+done
+
+run_expect_succ "$RADOS_TOOL" cppool "$POOL" "$POOL_CP_TARGET"
+
+mkdir -p "$TDIR/dir_cp_dst"
+for i in `seq 1 5`; do
+  fname="$TDIR/dir_cp_dst/f.$i"
+  objname="f.$i"
+  run_expect_succ "$RADOS_TOOL" -p "$POOL_CP_TARGET" get $objname "$fname"
+
+# a few random attrs
+  for j in `seq 1 4`; do
+    run_expect_succ --tee "$fname.attr.$j" "$RADOS_TOOL" -p "$POOL_CP_TARGET" getxattr $objname attr.$j
+  done
+
+  run_expect_succ --tee "$fname.omap.header" "$RADOS_TOOL" -p "$POOL_CP_TARGET" getomapheader $objname
+  run_expect_succ --tee "$fname.omap.vals" "$RADOS_TOOL" -p "$POOL_CP_TARGET" listomapvals $objname
+done
+
+diff -q -r "$TDIR/dir_cp_src" "$TDIR/dir_cp_dst" \
+    || die "copy pool validation failed!"
+
+for opt in \
+    block-size \
+    concurrent-ios \
+    min-object-size \
+    max-object-size \
+    min-op-len \
+    max-op-len \
+    max-ops \
+    max-backlog \
+    target-throughput \
+    read-percent \
+    num-objects \
+    run-length \
+    ; do
+    run_expect_succ "$RADOS_TOOL" --$opt 4 df
+    run_expect_fail "$RADOS_TOOL" --$opt 4k df
+done
+
+run_expect_succ "$RADOS_TOOL" lock list f.1 --lock-duration 4 --pool "$POOL"
+echo # previous command doesn't output an end of line: issue #9735
+run_expect_fail "$RADOS_TOOL" lock list f.1 --lock-duration 4k --pool "$POOL"
+
+run_expect_succ "$RADOS_TOOL" mksnap snap1 --pool "$POOL"
+snapid=$("$RADOS_TOOL" lssnap --pool "$POOL" | grep snap1 | cut -f1)
+[ $? -ne 0 ] && die "expected success, but got failure! cmd: \"$RADOS_TOOL\" lssnap --pool \"$POOL\" | grep snap1 | cut -f1"
+run_expect_succ "$RADOS_TOOL" ls --pool "$POOL" --snapid="$snapid"
+run_expect_fail "$RADOS_TOOL" ls --pool "$POOL" --snapid="$snapid"k
+
+run_expect_succ "$RADOS_TOOL" truncate f.1 0 --pool "$POOL"
+run_expect_fail "$RADOS_TOOL" truncate f.1 0k --pool "$POOL"
+
+run "$CEPH_TOOL" osd pool rm delete_me_mkpool_test delete_me_mkpool_test --yes-i-really-really-mean-it
+run_expect_succ "$CEPH_TOOL" osd pool create delete_me_mkpool_test 1
+
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 write
+run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 1k write
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 write --format json --output "$TDIR/bench.json"
+run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 1 write --output "$TDIR/bench.json"
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --format json --no-cleanup
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 rand --format json
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 rand -f json
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 seq --format json
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 1 seq -f json
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-omap
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-object
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-xattr
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-xattr --write-object
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-xattr --write-omap
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-omap --write-object
+run_expect_succ "$RADOS_TOOL" --pool "$POOL" bench 5 write --write-xattr --write-omap --write-object
+run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-omap
+run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-object
+run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-xattr
+run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-xattr --write-object
+run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-xattr --write-omap
+run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-omap --write-object
+run_expect_fail "$RADOS_TOOL" --pool "$POOL" bench 5 read --write-xattr --write-omap --write-object
+
+for i in $("$RADOS_TOOL" --pool "$POOL" ls | grep "benchmark_data"); do
+    "$RADOS_TOOL" --pool "$POOL" truncate $i 0
+done
+
+run_expect_nosignal "$RADOS_TOOL" --pool "$POOL" bench 1 rand
+run_expect_nosignal "$RADOS_TOOL" --pool "$POOL" bench 1 seq
+
+set -e
+
+OBJ=test_rados_obj
+
+expect_false()
+{
+	if "$@"; then return 1; else return 0; fi
+}
+
+cleanup() {
+    $RADOS_TOOL -p $POOL rm $OBJ > /dev/null 2>&1 || true
+    $RADOS_TOOL -p $POOL_EC rm $OBJ > /dev/null 2>&1 || true
+}
+
+test_omap() {
+    cleanup
+    for i in $(seq 1 1 10)
+    do
+	if [ $(($i % 2)) -eq 0 ]; then
+            $RADOS_TOOL -p $POOL setomapval $OBJ $i $i
+	else
+            echo -n "$i" | $RADOS_TOOL -p $POOL setomapval $OBJ $i
+	fi
+        $RADOS_TOOL -p $POOL getomapval $OBJ $i | grep -q "|$i|\$"
+    done
+    $RADOS_TOOL -p $POOL listomapvals $OBJ | grep -c value | grep 10
+    for i in $(seq 1 1 5)
+    do
+        $RADOS_TOOL -p $POOL rmomapkey $OBJ $i
+    done
+    $RADOS_TOOL -p $POOL listomapvals $OBJ | grep -c value | grep 5
+    $RADOS_TOOL -p $POOL clearomap $OBJ
+    $RADOS_TOOL -p $POOL listomapvals $OBJ | wc -l | grep 0
+    cleanup
+
+    for i in $(seq 1 1 10)
+    do
+        dd if=/dev/urandom bs=128 count=1 > $TDIR/omap_key
+        if [ $(($i % 2)) -eq 0 ]; then
+            $RADOS_TOOL -p $POOL --omap-key-file $TDIR/omap_key setomapval $OBJ $i
+        else
+            echo -n "$i" | $RADOS_TOOL -p $POOL --omap-key-file $TDIR/omap_key setomapval $OBJ
+        fi
+        $RADOS_TOOL -p $POOL --omap-key-file $TDIR/omap_key getomapval $OBJ | grep -q "|$i|\$"
+        $RADOS_TOOL -p $POOL --omap-key-file $TDIR/omap_key rmomapkey $OBJ
+        $RADOS_TOOL -p $POOL listomapvals $OBJ | grep -c value | grep 0
+    done
+    cleanup
+}
+
+test_xattr() {
+    cleanup
+    $RADOS_TOOL -p $POOL put $OBJ /etc/passwd
+    V1=`mktemp fooattrXXXXXXX`
+    V2=`mktemp fooattrXXXXXXX`
+    echo -n fooval > $V1
+    expect_false $RADOS_TOOL -p $POOL setxattr $OBJ 2>/dev/null
+    expect_false $RADOS_TOOL -p $POOL setxattr $OBJ foo fooval extraarg 2>/dev/null
+    $RADOS_TOOL -p $POOL setxattr $OBJ foo fooval
+    $RADOS_TOOL -p $POOL getxattr $OBJ foo > $V2
+    cmp $V1 $V2
+    cat $V1 | $RADOS_TOOL -p $POOL setxattr $OBJ bar
+    $RADOS_TOOL -p $POOL getxattr $OBJ bar > $V2
+    cmp $V1 $V2
+    $RADOS_TOOL -p $POOL listxattr $OBJ > $V1
+    grep -q foo $V1
+    grep -q bar $V1
+    [ `cat $V1 | wc -l` -eq 2 ]
+    rm $V1 $V2
+    cleanup
+}
+test_rmobj() {
+    p=`uuidgen`
+    $CEPH_TOOL osd pool create $p 1
+    $CEPH_TOOL osd pool set-quota $p max_objects 1
+    V1=`mktemp fooattrXXXXXXX`
+    $RADOS_TOOL put $OBJ $V1 -p $p
+    while ! $CEPH_TOOL osd dump | grep 'full_quota max_objects'
+    do
+	sleep 2
+    done
+    $RADOS_TOOL -p $p rm $OBJ --force-full
+    $CEPH_TOOL osd pool rm $p $p --yes-i-really-really-mean-it
+    rm $V1
+}
+
+test_ls() {
+    echo "Testing rados ls command"
+    p=`uuidgen`
+    $CEPH_TOOL osd pool create $p 1
+    NS=10
+    OBJS=20
+    # Include default namespace (0) in the total
+    TOTAL=$(expr $OBJS \* $(expr $NS + 1))
+
+    for nsnum in `seq 0 $NS`
+    do
+        for onum in `seq 1 $OBJS`
+        do
+	    if [ "$nsnum" = "0" ];
+	    then
+                "$RADOS_TOOL" -p $p put obj${onum} /etc/fstab 2> /dev/null
+            else
+                "$RADOS_TOOL" -p $p -N "NS${nsnum}" put obj${onum} /etc/fstab 2> /dev/null
+	    fi
+	done
+    done
+    CHECK=$("$RADOS_TOOL" -p $p ls 2> /dev/null | wc -l)
+    if [ "$OBJS" -ne "$CHECK" ];
+    then
+        die "Created $OBJS objects in default namespace but saw $CHECK"
+    fi
+    TESTNS=NS${NS}
+    CHECK=$("$RADOS_TOOL" -p $p -N $TESTNS ls 2> /dev/null | wc -l)
+    if [ "$OBJS" -ne "$CHECK" ];
+    then
+        die "Created $OBJS objects in $TESTNS namespace but saw $CHECK"
+    fi
+    CHECK=$("$RADOS_TOOL" -p $p --all ls 2> /dev/null | wc -l)
+    if [ "$TOTAL" -ne "$CHECK" ];
+    then
+        die "Created $TOTAL objects but saw $CHECK"
+    fi
+
+    $CEPH_TOOL osd pool rm $p $p --yes-i-really-really-mean-it
+}
+
+test_cleanup() {
+    echo "Testing rados cleanup command"
+    p=`uuidgen`
+    $CEPH_TOOL osd pool create $p 1
+    NS=5
+    OBJS=4
+    # Include default namespace (0) in the total
+    TOTAL=$(expr $OBJS \* $(expr $NS + 1))
+
+    for nsnum in `seq 0 $NS`
+    do
+        for onum in `seq 1 $OBJS`
+        do
+	    if [ "$nsnum" = "0" ];
+	    then
+                "$RADOS_TOOL" -p $p put obj${onum} /etc/fstab 2> /dev/null
+            else
+                "$RADOS_TOOL" -p $p -N "NS${nsnum}" put obj${onum} /etc/fstab 2> /dev/null
+	    fi
+	done
+    done
+
+    $RADOS_TOOL -p $p --all ls > $TDIR/before.ls.out 2> /dev/null
+
+    $RADOS_TOOL -p $p bench 3 write --no-cleanup 2> /dev/null
+    $RADOS_TOOL -p $p -N NS1 bench 3 write --no-cleanup 2> /dev/null
+    $RADOS_TOOL -p $p -N NS2 bench 3 write --no-cleanup 2> /dev/null
+    $RADOS_TOOL -p $p -N NS3 bench 3 write --no-cleanup 2> /dev/null
+    # Leave dangling objects without a benchmark_last_metadata in NS4
+    expect_false timeout 3 $RADOS_TOOL -p $p -N NS4 bench 30 write --no-cleanup 2> /dev/null
+    $RADOS_TOOL -p $p -N NS5 bench 3 write --no-cleanup 2> /dev/null
+
+    $RADOS_TOOL -p $p -N NS3 cleanup 2> /dev/null
+    #echo "Check NS3 after specific cleanup"
+    CHECK=$($RADOS_TOOL -p $p -N NS3 ls | wc -l)
+    if [ "$OBJS" -ne "$CHECK" ] ;
+    then
+        die "Expected $OBJS objects in NS3 but saw $CHECK"
+    fi
+
+    #echo "Try to cleanup all"
+    $RADOS_TOOL -p $p --all cleanup
+    #echo "Check all namespaces"
+    $RADOS_TOOL -p $p --all ls > $TDIR/after.ls.out 2> /dev/null
+    CHECK=$(cat $TDIR/after.ls.out | wc -l)
+    if [ "$TOTAL" -ne "$CHECK" ];
+    then
+        die "Expected $TOTAL objects but saw $CHECK"
+    fi
+    if ! diff $TDIR/before.ls.out $TDIR/after.ls.out
+    then
+        die "Different objects found after cleanup"
+    fi
+
+    set +e
+    run_expect_fail $RADOS_TOOL -p $p cleanup --prefix illegal_prefix
+    run_expect_succ $RADOS_TOOL -p $p cleanup --prefix benchmark_data_otherhost
+    set -e
+
+    $CEPH_TOOL osd pool rm $p $p --yes-i-really-really-mean-it
+}
+
+function test_append()
+{
+  cleanup
+
+  # create object
+  touch ./rados_append_null
+  $RADOS_TOOL -p $POOL append $OBJ ./rados_append_null
+  $RADOS_TOOL -p $POOL get $OBJ ./rados_append_0_out
+  cmp ./rados_append_null ./rados_append_0_out
+
+  # append 4k, total size 4k
+  dd if=/dev/zero of=./rados_append_4k bs=4k count=1
+  $RADOS_TOOL -p $POOL append $OBJ ./rados_append_4k
+  $RADOS_TOOL -p $POOL get $OBJ ./rados_append_4k_out
+  cmp ./rados_append_4k ./rados_append_4k_out
+
+  # append 4k, total size 8k
+  $RADOS_TOOL -p $POOL append $OBJ ./rados_append_4k
+  $RADOS_TOOL -p $POOL get $OBJ ./rados_append_4k_out
+  read_size=`ls -l ./rados_append_4k_out | awk -F ' '  '{print $5}'`
+  if [ 8192 -ne $read_size ];
+  then
+    die "Append failed expecting 8192 read $read_size"
+  fi
+
+  # append 10M, total size 10493952
+  dd if=/dev/zero of=./rados_append_10m bs=10M count=1
+  $RADOS_TOOL -p $POOL append $OBJ ./rados_append_10m
+  $RADOS_TOOL -p $POOL get $OBJ ./rados_append_10m_out
+  read_size=`ls -l ./rados_append_10m_out | awk -F ' '  '{print $5}'`
+  if [ 10493952 -ne $read_size ];
+  then
+    die "Append failed expecting 10493952 read $read_size"
+  fi
+
+  # cleanup
+  cleanup
+
+  # create object
+  $RADOS_TOOL -p $POOL_EC append $OBJ ./rados_append_null
+  $RADOS_TOOL -p $POOL_EC get $OBJ ./rados_append_0_out
+  cmp rados_append_null rados_append_0_out
+
+  # append 4k, total size 4k
+  $RADOS_TOOL -p $POOL_EC append $OBJ ./rados_append_4k
+  $RADOS_TOOL -p $POOL_EC get $OBJ ./rados_append_4k_out
+  cmp rados_append_4k rados_append_4k_out
+
+  # append 4k, total size 8k
+  $RADOS_TOOL -p $POOL_EC append $OBJ ./rados_append_4k
+  $RADOS_TOOL -p $POOL_EC get $OBJ ./rados_append_4k_out
+  read_size=`ls -l ./rados_append_4k_out | awk -F ' '  '{print $5}'`
+  if [ 8192 -ne $read_size ];
+  then
+    die "Append failed expecting 8192 read $read_size"
+  fi
+
+  # append 10M, total size 10493952
+  $RADOS_TOOL -p $POOL_EC append $OBJ ./rados_append_10m
+  $RADOS_TOOL -p $POOL_EC get $OBJ ./rados_append_10m_out
+  read_size=`ls -l ./rados_append_10m_out | awk -F ' '  '{print $5}'`
+  if [ 10493952 -ne $read_size ];
+  then
+    die "Append failed expecting 10493952 read $read_size"
+  fi
+
+  cleanup
+  rm -rf ./rados_append_null ./rados_append_0_out
+  rm -rf ./rados_append_4k ./rados_append_4k_out ./rados_append_10m ./rados_append_10m_out
+}
+
+function test_put()
+{
+  # rados put test:
+  cleanup
+
+  # create file in local fs
+  dd if=/dev/urandom of=rados_object_10k bs=1K count=10
+
+  # test put command
+  $RADOS_TOOL -p $POOL put $OBJ ./rados_object_10k
+  $RADOS_TOOL -p $POOL get $OBJ ./rados_object_10k_out
+  cmp ./rados_object_10k ./rados_object_10k_out
+  cleanup
+
+  # test put command with offset 0
+  $RADOS_TOOL -p $POOL put $OBJ ./rados_object_10k --offset 0
+  $RADOS_TOOL -p $POOL get $OBJ ./rados_object_offset_0_out
+  cmp ./rados_object_10k ./rados_object_offset_0_out
+  cleanup
+
+  # test put command with offset 1000
+  $RADOS_TOOL -p $POOL put $OBJ ./rados_object_10k --offset 1000
+  $RADOS_TOOL -p $POOL get $OBJ ./rados_object_offset_1000_out
+  cmp ./rados_object_10k ./rados_object_offset_1000_out 0 1000
+  cleanup
+
+  rm -rf ./rados_object_10k ./rados_object_10k_out ./rados_object_offset_0_out ./rados_object_offset_1000_out
+}
+
+function test_stat()
+{
+  bluestore=$("$CEPH_TOOL" osd metadata | grep '"osd_objectstore": "bluestore"' | cut -f1)
+  # create file in local fs
+  dd if=/dev/urandom of=rados_object_128k bs=64K count=2
+
+  # rados df test (replicated_pool):
+  $RADOS_TOOL purge $POOL --yes-i-really-really-mean-it
+  $CEPH_TOOL osd pool rm $POOL $POOL --yes-i-really-really-mean-it
+  $CEPH_TOOL osd pool create $POOL 8
+  $CEPH_TOOL osd pool set $POOL size 3
+
+  # put object with 1 MB gap in front
+  $RADOS_TOOL -p $POOL put $OBJ ./rados_object_128k --offset=1048576
+  MATCH_CNT=0
+  if [ "" == "$bluestore" ];
+  then
+    STORED=1.1
+    STORED_UNIT="MiB"
+  else
+    STORED=384
+    STORED_UNIT="KiB"
+  fi
+  for i in {1..60}
+  do
+    IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "")
+    [[ -z $IN ]] && sleep 1 && continue
+    IFS=' ' read -ra VALS <<< "$IN"
+
+    # verification is a bit tricky due to stats report's eventual model
+    # VALS[1] - STORED
+    # VALS[2] - STORED units
+    # VALS[3] - OBJECTS
+    # VALS[5] - COPIES
+    # VALS[12] - WR_OPS
+    # VALS[13] - WR
+    # VALS[14] - WR uints
+    # implies replication factor 3
+    if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "1" ] && [ ${VALS[13]} == 128 ] && [ ${VALS[14]} == "KiB" ]
+    then
+      # enforce multiple match to make sure stats aren't changing any more
+      MATCH_CNT=$((MATCH_CNT+1))
+      [[ $MATCH_CNT == 3 ]] && break
+      sleep 1
+      continue
+    fi
+    MATCH_CNT=0
+    sleep 1
+    continue
+  done
+  [[ -z $IN ]] && die "Failed to retrieve any pool stats within 60 seconds"
+  if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "1" ] || [ ${VALS[13]} != 128 ] || [ ${VALS[14]} != "KiB" ]
+  then
+    die "Failed to retrieve proper pool stats within 60 seconds"
+  fi
+
+  # overwrite data at 1MB offset
+  $RADOS_TOOL -p $POOL put $OBJ ./rados_object_128k --offset=1048576
+  MATCH_CNT=0
+  if [ "" == "$bluestore" ];
+  then
+    STORED=1.1
+    STORED_UNIT="MiB"
+  else
+    STORED=384
+    STORED_UNIT="KiB"
+  fi
+  for i in {1..60}
+  do
+    IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "")
+    IFS=' ' read -ra VALS <<< "$IN"
+
+    # verification is a bit tricky due to stats report's eventual model
+    # VALS[1] - STORED
+    # VALS[2] - STORED units
+    # VALS[3] - OBJECTS
+    # VALS[5] - COPIES
+    # VALS[12] - WR_OPS
+    # VALS[13] - WR
+    # VALS[14] - WR uints
+    # implies replication factor 3
+    if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "2" ] && [ ${VALS[13]} == 256 ] && [ ${VALS[14]} == "KiB" ]
+    then
+      # enforce multiple match to make sure stats aren't changing any more
+      MATCH_CNT=$((MATCH_CNT+1))
+      [[ $MATCH_CNT == 3 ]] && break
+      sleep 1
+      continue
+    fi
+    MATCH_CNT=0
+    sleep 1
+    continue
+  done
+  if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "2" ] || [ ${VALS[13]} != 256 ] || [ ${VALS[14]} != "KiB" ]
+  then
+    die "Failed to retrieve proper pool stats within 60 seconds"
+  fi
+
+  # write data at 64K offset
+  $RADOS_TOOL -p $POOL put $OBJ ./rados_object_128k --offset=65536
+  MATCH_CNT=0
+  if [ "" == "$bluestore" ];
+  then
+    STORED=1.1
+    STORED_UNIT="MiB"
+  else
+    STORED=768
+    STORED_UNIT="KiB"
+  fi
+  for i in {1..60}
+  do
+    IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "")
+    IFS=' ' read -ra VALS <<< "$IN"
+
+    # verification is a bit tricky due to stats report's eventual model
+    # VALS[1] - STORED
+    # VALS[2] - STORED units
+    # VALS[3] - OBJECTS
+    # VALS[5] - COPIES
+    # VALS[12] - WR_OPS
+    # VALS[13] - WR
+    # VALS[14] - WR uints
+    # implies replication factor 3
+    if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "3" ] && [ ${VALS[13]} == 384 ] && [ ${VALS[14]} == "KiB" ]
+    then
+      # enforce multiple match to make sure stats aren't changing any more
+      MATCH_CNT=$((MATCH_CNT+1))
+      [[ $MATCH_CNT == 3 ]] && break
+      sleep 1
+      continue
+    fi
+    MATCH_CNT=0
+    sleep 1
+    continue
+  done
+  if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "3" ] || [ ${VALS[13]} != 384 ] || [ ${VALS[14]} != "KiB" ]
+  then
+    die "Failed to retrieve proper pool stats within 60 seconds"
+  fi
+
+  # overwrite object totally
+  $RADOS_TOOL -p $POOL put $OBJ ./rados_object_128k
+  MATCH_CNT=0
+  if [ "" == "$bluestore" ];
+  then
+    STORED=128
+    STORED_UNIT="KiB"
+  else
+    STORED=384
+    STORED_UNIT="KiB"
+  fi
+  for i in {1..60}
+  do
+    IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "")
+    IFS=' ' read -ra VALS <<< "$IN"
+
+    # verification is a bit tricky due to stats report's eventual model
+    # VALS[1] - STORED
+    # VALS[2] - STORED units
+    # VALS[3] - OBJECTS
+    # VALS[5] - COPIES
+    # VALS[12] - WR_OPS
+    # VALS[13] - WR
+    # VALS[14] - WR uints
+    # implies replication factor 3
+    if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "4" ] && [ ${VALS[13]} == 512 ] && [ ${VALS[14]} == "KiB" ]
+    then
+      # enforce multiple match to make sure stats aren't changing any more
+      MATCH_CNT=$((MATCH_CNT+1))
+      [[ $MATCH_CNT == 3 ]] && break
+      sleep 1
+      continue
+    fi
+    MATCH_CNT=0
+    sleep 1
+    continue
+  done
+  if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "4" ] || [ ${VALS[13]} != 512 ] || [ ${VALS[14]} != "KiB" ]
+  then
+    die "Failed to retrieve proper pool stats within 60 seconds"
+  fi
+
+  cleanup
+
+  # after cleanup?
+  MATCH_CNT=0
+  for i in {1..60}
+  do
+    IN=$($RADOS_TOOL -p $POOL df | grep $POOL ; [[ ! -z $? ]] && echo "")
+    IFS=' ' read -ra VALS <<< "$IN"
+
+    # verification is a bit tricky due to stats report's eventual model
+    # VALS[1] - STORED
+    # VALS[2] - STORED units
+    # VALS[3] - OBJECTS
+    # VALS[5] - COPIES
+    # VALS[12] - WR_OPS
+    # VALS[13] - WR
+    # VALS[14] - WR uints
+    # implies replication factor 3
+    if [ ${VALS[1]} == 0 ] && [ ${VALS[2]} == "B" ] && [ ${VALS[3]} == "0" ] && [ ${VALS[5]} == "0" ] && [ ${VALS[12]} == "5" ] && [ ${VALS[13]} == 512 ] && [ ${VALS[14]} == "KiB" ]
+    then
+      # enforce multiple match to make sure stats aren't changing any more
+      MATCH_CNT=$((MATCH_CNT+1))
+      [[ $MATCH_CNT == 3 ]] && break
+      sleep 1
+      continue
+    fi
+    MATCH_CNT=0
+    sleep 1
+    continue
+  done
+  if [ ${VALS[1]} != 0 ] || [ ${VALS[2]} != "B" ] || [ ${VALS[3]} != "0" ] || [ ${VALS[5]} != "0" ] || [ ${VALS[12]} != "5" ] || [ ${VALS[13]} != 512 ] || [ ${VALS[14]} != "KiB" ]
+  then
+    die "Failed to retrieve proper pool stats within 60 seconds"
+  fi
+
+  ############ rados df test (EC pool): ##############
+  $RADOS_TOOL purge $POOL_EC --yes-i-really-really-mean-it
+  $CEPH_TOOL osd pool rm $POOL_EC $POOL_EC --yes-i-really-really-mean-it
+  $CEPH_TOOL osd erasure-code-profile set myprofile k=2 m=1 stripe_unit=2K crush-failure-domain=osd --force
+  $CEPH_TOOL osd pool create $POOL_EC 8 8 erasure
+
+  # put object
+  $RADOS_TOOL -p $POOL_EC put $OBJ ./rados_object_128k
+  MATCH_CNT=0
+  if [ "" == "$bluestore" ];
+  then
+    STORED=128
+    STORED_UNIT="KiB"
+  else
+    STORED=192
+    STORED_UNIT="KiB"
+  fi
+  for i in {1..60}
+  do
+    IN=$($RADOS_TOOL -p $POOL_EC df | grep $POOL_EC ; [[ ! -z $? ]] && echo "")
+    [[ -z $IN ]] && sleep 1 && continue
+    IFS=' ' read -ra VALS <<< "$IN"
+
+    # verification is a bit tricky due to stats report's eventual model
+    # VALS[1] - STORED
+    # VALS[2] - STORED units
+    # VALS[3] - OBJECTS
+    # VALS[5] - COPIES
+    # VALS[12] - WR_OPS
+    # VALS[13] - WR
+    # VALS[14] - WR uints
+    # implies replication factor 2+1
+    if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "1" ] && [ ${VALS[13]} == 128 ] && [ ${VALS[14]} == "KiB" ]
+    then
+      # enforce multiple match to make sure stats aren't changing any more
+      MATCH_CNT=$((MATCH_CNT+1))
+      [[ $MATCH_CNT == 3 ]] && break
+      sleep 1
+      continue
+    fi
+    MATCH_CNT=0
+    sleep 1
+    continue
+  done
+  [[ -z $IN ]] && die "Failed to retrieve any pool stats within 60 seconds"
+  if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "1" ] || [ ${VALS[13]} != 128 ] || [ ${VALS[14]} != "KiB" ]
+  then
+    die "Failed to retrieve proper pool stats within 60 seconds"
+  fi
+
+  # overwrite object
+  $RADOS_TOOL -p $POOL_EC put $OBJ ./rados_object_128k
+  MATCH_CNT=0
+  if [ "" == "$bluestore" ];
+  then
+    STORED=128
+    STORED_UNIT="KiB"
+  else
+    STORED=192
+    STORED_UNIT="KiB"
+  fi
+  for i in {1..60}
+  do
+    IN=$($RADOS_TOOL -p $POOL_EC df | grep $POOL_EC ; [[ ! -z $? ]] && echo "")
+    IFS=' ' read -ra VALS <<< "$IN"
+
+    # verification is a bit tricky due to stats report's eventual model
+    # VALS[1] - STORED
+    # VALS[2] - STORED units
+    # VALS[3] - OBJECTS
+    # VALS[5] - COPIES
+    # VALS[12] - WR_OPS
+    # VALS[13] - WR
+    # VALS[14] - WR uints
+    # implies replication factor 2+1
+    if [ ${VALS[1]} == $STORED ] && [ ${VALS[2]} == $STORED_UNIT ] && [ ${VALS[3]} == "1" ] && [ ${VALS[5]} == "3" ] && [ ${VALS[12]} == "2" ] && [ ${VALS[13]} == 256 ] && [ ${VALS[14]} == "KiB" ]
+    then
+      # enforce multiple match to make sure stats aren't changing any more
+      MATCH_CNT=$((MATCH_CNT+1))
+      [[ $MATCH_CNT == 3 ]] && break
+      sleep 1
+      continue
+    fi
+    MATCH_CNT=0
+    sleep 1
+    continue
+  done
+  if [ ${VALS[1]} != $STORED ] || [ ${VALS[2]} != $STORED_UNIT ] || [ ${VALS[3]} != "1" ] || [ ${VALS[5]} != "3" ] || [ ${VALS[12]} != "2" ] || [ ${VALS[13]} != 256 ] || [ ${VALS[14]} != "KiB" ]
+  then
+    die "Failed to retrieve proper pool stats within 60 seconds"
+  fi
+
+  cleanup
+
+  # after cleanup?
+  MATCH_CNT=0
+  for i in {1..60}
+  do
+    IN=$($RADOS_TOOL -p $POOL_EC df | grep $POOL_EC ; [[ ! -z $? ]] && echo "")
+    IFS=' ' read -ra VALS <<< "$IN"
+
+    # verification is a bit tricky due to stats report's eventual model
+    # VALS[1] - STORED
+    # VALS[2] - STORED units
+    # VALS[3] - OBJECTS
+    # VALS[5] - COPIES
+    # VALS[12] - WR_OPS
+    # VALS[13] - WR
+    # VALS[14] - WR uints
+    # implies replication factor 2+1
+    if [ ${VALS[1]} == 0 ] && [ ${VALS[2]} == "B" ] && [ ${VALS[3]} == "0" ] && [ ${VALS[5]} == "0" ] && [ ${VALS[12]} == "3" ] && [ ${VALS[13]} == 256 ] && [ ${VALS[14]} == "KiB" ]
+    then
+      # enforce multiple match to make sure stats aren't changing any more
+      MATCH_CNT=$((MATCH_CNT+1))
+      [[ $MATCH_CNT == 3 ]] && break
+      sleep 1
+      continue
+    fi
+    MATCH_CNT=0
+    sleep 1
+    continue
+  done
+  if [ ${VALS[1]} != 0 ] || [ ${VALS[2]} != "B" ] || [ ${VALS[3]} != "0" ] || [ ${VALS[5]} != "0" ] || [ ${VALS[12]} != "3" ] || [ ${VALS[13]} != 256 ] || [ ${VALS[14]} != "KiB" ]
+  then
+    die "Failed to retrieve proper pool stats within 60 seconds"
+  fi
+
+  rm -rf ./rados_object_128k
+}
+
+test_xattr
+test_omap
+test_rmobj
+test_ls
+test_cleanup
+test_append
+test_put
+test_stat
+
+# clean up environment, delete pool
+$CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it
+$CEPH_TOOL osd pool delete $POOL_EC $POOL_EC --yes-i-really-really-mean-it
+$CEPH_TOOL osd pool delete $POOL_CP_TARGET $POOL_CP_TARGET --yes-i-really-really-mean-it
+
+echo "SUCCESS!"
+exit 0
diff --git a/qa/workunits/rbd/cli_generic.sh b/qa/workunits/rbd/cli_generic.sh
new file mode 100755
index 00000000..7f44d932
--- /dev/null
+++ b/qa/workunits/rbd/cli_generic.sh
@@ -0,0 +1,933 @@
+#!/usr/bin/env bash
+set -ex
+
+. $(dirname $0)/../../standalone/ceph-helpers.sh
+
+export RBD_FORCE_ALLOW_V1=1
+
+# make sure rbd pool is EMPTY.. this is a test script!!
+rbd ls | wc -l | grep -v '^0$' && echo "nonempty rbd pool, aborting!  run this script on an empty test cluster only." && exit 1
+
+IMGS="testimg1 testimg2 testimg3 testimg4 testimg5 testimg6 testimg-diff1 testimg-diff2 testimg-diff3 foo foo2 bar bar2 test1 test2 test3 test4 clone2"
+
+expect_fail() {
+    "$@" && return 1 || return 0
+}
+
+tiered=0
+if ceph osd dump | grep ^pool | grep "'rbd'" | grep tier; then
+    tiered=1
+fi
+
+remove_images() {
+    for img in $IMGS
+    do
+        (rbd snap purge $img || true) >/dev/null 2>&1
+        (rbd rm $img || true) >/dev/null 2>&1
+    done
+}
+
+test_others() {
+    echo "testing import, export, resize, and snapshots..."
+    TMP_FILES="/tmp/img1 /tmp/img1.new /tmp/img2 /tmp/img2.new /tmp/img3 /tmp/img3.new /tmp/img-diff1.new /tmp/img-diff2.new /tmp/img-diff3.new /tmp/img1.snap1 /tmp/img1.snap1 /tmp/img-diff1.snap1"
+
+    remove_images
+    rm -f $TMP_FILES
+
+    # create an image
+    dd if=/bin/sh of=/tmp/img1 bs=1k count=1 seek=10
+    dd if=/bin/dd of=/tmp/img1 bs=1k count=10 seek=100
+    dd if=/bin/rm of=/tmp/img1 bs=1k count=100 seek=1000
+    dd if=/bin/ls of=/tmp/img1 bs=1k seek=10000
+    dd if=/bin/ln of=/tmp/img1 bs=1k seek=100000
+
+    # import, snapshot
+    rbd import $RBD_CREATE_ARGS /tmp/img1 testimg1
+    rbd resize testimg1 --size=256 --allow-shrink
+    rbd export testimg1 /tmp/img2
+    rbd snap create testimg1 --snap=snap1
+    rbd resize testimg1 --size=128 && exit 1 || true   # shrink should fail
+    rbd resize testimg1 --size=128 --allow-shrink
+    rbd export testimg1 /tmp/img3
+
+    # info
+    rbd info testimg1 | grep 'size 128 MiB'
+    rbd info --snap=snap1 testimg1 | grep 'size 256 MiB'
+
+    # export-diff
+    rm -rf /tmp/diff-testimg1-1 /tmp/diff-testimg1-2
+    rbd export-diff testimg1 --snap=snap1 /tmp/diff-testimg1-1
+    rbd export-diff testimg1 --from-snap=snap1 /tmp/diff-testimg1-2
+
+    # import-diff
+    rbd create $RBD_CREATE_ARGS --size=1 testimg-diff1
+    rbd import-diff --sparse-size 8K /tmp/diff-testimg1-1 testimg-diff1
+    rbd import-diff --sparse-size 8K /tmp/diff-testimg1-2 testimg-diff1
+
+    # info
+    rbd info testimg1 | grep 'size 128 MiB'
+    rbd info --snap=snap1 testimg1 | grep 'size 256 MiB'
+    rbd info testimg-diff1 | grep 'size 128 MiB'
+    rbd info --snap=snap1 testimg-diff1 | grep 'size 256 MiB'
+
+    # make copies
+    rbd copy testimg1 --snap=snap1 testimg2
+    rbd copy testimg1 testimg3
+    rbd copy testimg-diff1 --sparse-size 768K --snap=snap1 testimg-diff2
+    rbd copy testimg-diff1 --sparse-size 768K testimg-diff3
+
+    # verify the result
+    rbd info testimg2 | grep 'size 256 MiB'
+    rbd info testimg3 | grep 'size 128 MiB'
+    rbd info testimg-diff2 | grep 'size 256 MiB'
+    rbd info testimg-diff3 | grep 'size 128 MiB'
+
+    # deep copies
+    rbd deep copy testimg1 testimg4
+    rbd deep copy testimg1 --snap=snap1 testimg5
+    rbd info testimg4 | grep 'size 128 MiB'
+    rbd info testimg5 | grep 'size 256 MiB'
+    rbd snap ls testimg4 | grep -v 'SNAPID' | wc -l | grep 1
+    rbd snap ls testimg4 | grep '.*snap1.*'
+
+    rbd export testimg1 /tmp/img1.new
+    rbd export testimg2 /tmp/img2.new
+    rbd export testimg3 /tmp/img3.new
+    rbd export testimg-diff1 /tmp/img-diff1.new
+    rbd export testimg-diff2 /tmp/img-diff2.new
+    rbd export testimg-diff3 /tmp/img-diff3.new
+
+    cmp /tmp/img2 /tmp/img2.new
+    cmp /tmp/img3 /tmp/img3.new
+    cmp /tmp/img2 /tmp/img-diff2.new
+    cmp /tmp/img3 /tmp/img-diff3.new
+
+    # rollback
+    rbd snap rollback --snap=snap1 testimg1
+    rbd snap rollback --snap=snap1 testimg-diff1
+    rbd info testimg1 | grep 'size 256 MiB'
+    rbd info testimg-diff1 | grep 'size 256 MiB'
+    rbd export testimg1 /tmp/img1.snap1
+    rbd export testimg-diff1 /tmp/img-diff1.snap1
+    cmp /tmp/img2 /tmp/img1.snap1
+    cmp /tmp/img2 /tmp/img-diff1.snap1
+
+    # test create, copy of zero-length images
+    rbd rm testimg2
+    rbd rm testimg3
+    rbd create testimg2 -s 0
+    rbd cp testimg2 testimg3
+    rbd deep cp testimg2 testimg6
+
+    # remove snapshots
+    rbd snap rm --snap=snap1 testimg1
+    rbd snap rm --snap=snap1 testimg-diff1
+    rbd info --snap=snap1 testimg1 2>&1 | grep 'error setting snapshot context: (2) No such file or directory'
+    rbd info --snap=snap1 testimg-diff1 2>&1 | grep 'error setting snapshot context: (2) No such file or directory'
+
+    # sparsify
+    rbd sparsify testimg1
+
+    remove_images
+    rm -f $TMP_FILES
+}
+
+test_rename() {
+    echo "testing rename..."
+    remove_images
+
+    rbd create --image-format 1 -s 1 foo
+    rbd create --image-format 2 -s 1 bar
+    rbd rename foo foo2
+    rbd rename foo2 bar 2>&1 | grep exists
+    rbd rename bar bar2
+    rbd rename bar2 foo2 2>&1 | grep exists
+
+    ceph osd pool create rbd2 8
+    rbd pool init rbd2
+    rbd create -p rbd2 -s 1 foo
+    rbd rename rbd2/foo rbd2/bar
+    rbd -p rbd2 ls | grep bar
+    rbd rename rbd2/bar foo
+    rbd rename --pool rbd2 foo bar
+    ! rbd rename rbd2/bar --dest-pool rbd foo
+    rbd rename --pool rbd2 bar --dest-pool rbd2 foo
+    rbd -p rbd2 ls | grep foo
+    ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it
+
+    remove_images
+}
+
+test_ls() {
+    echo "testing ls..."
+    remove_images
+
+    rbd create --image-format 1 -s 1 test1
+    rbd create --image-format 1 -s 1 test2
+    rbd ls | grep test1
+    rbd ls | grep test2
+    rbd ls | wc -l | grep 2
+    # look for fields in output of ls -l without worrying about space
+    rbd ls -l | grep 'test1.*1 MiB.*1'
+    rbd ls -l | grep 'test2.*1 MiB.*1'
+
+    rbd rm test1
+    rbd rm test2
+
+    rbd create --image-format 2 -s 1 test1
+    rbd create --image-format 2 -s 1 test2
+    rbd ls | grep test1
+    rbd ls | grep test2
+    rbd ls | wc -l | grep 2
+    rbd ls -l | grep 'test1.*1 MiB.*2'
+    rbd ls -l | grep 'test2.*1 MiB.*2'
+
+    rbd rm test1
+    rbd rm test2
+
+    rbd create --image-format 2 -s 1 test1
+    rbd create --image-format 1 -s 1 test2
+    rbd ls | grep test1
+    rbd ls | grep test2
+    rbd ls | wc -l | grep 2
+    rbd ls -l | grep 'test1.*1 MiB.*2'
+    rbd ls -l | grep 'test2.*1 MiB.*1'
+    remove_images
+
+    # test that many images can be shown by ls
+    for i in $(seq -w 00 99); do
+	rbd create image.$i -s 1
+    done
+    rbd ls | wc -l | grep 100
+    rbd ls -l | grep image | wc -l | grep 100
+    for i in $(seq -w 00 99); do
+	rbd rm image.$i
+    done
+
+    for i in $(seq -w 00 99); do
+	rbd create image.$i --image-format 2 -s 1
+    done
+    rbd ls | wc -l | grep 100
+    rbd ls -l | grep image |  wc -l | grep 100
+    for i in $(seq -w 00 99); do
+	rbd rm image.$i
+    done
+}
+
+test_remove() {
+    echo "testing remove..."
+    remove_images
+
+    rbd remove "NOT_EXIST" && exit 1 || true	# remove should fail
+    rbd create --image-format 1 -s 1 test1
+    rbd rm test1
+    rbd ls | wc -l | grep "^0$"
+
+    rbd create --image-format 2 -s 1 test2
+    rbd rm test2
+    rbd ls | wc -l | grep "^0$"
+
+    # check that remove succeeds even if it's
+    # interrupted partway through. simulate this
+    # by removing some objects manually.
+
+    # remove with header missing (old format)
+    rbd create --image-format 1 -s 1 test1
+    rados rm -p rbd test1.rbd
+    rbd rm test1
+    rbd ls | wc -l | grep "^0$"
+
+    if [ $tiered -eq 0 ]; then
+        # remove with header missing
+	rbd create --image-format 2 -s 1 test2
+	HEADER=$(rados -p rbd ls | grep '^rbd_header')
+	rados -p rbd rm $HEADER
+	rbd rm test2
+	rbd ls | wc -l | grep "^0$"
+
+        # remove with id missing
+	rbd create --image-format 2 -s 1 test2
+	rados -p rbd rm rbd_id.test2
+	rbd rm test2
+	rbd ls | wc -l | grep "^0$"
+
+        # remove with header and id missing
+	rbd create --image-format 2 -s 1 test2
+	HEADER=$(rados -p rbd ls | grep '^rbd_header')
+	rados -p rbd rm $HEADER
+	rados -p rbd rm rbd_id.test2
+	rbd rm test2
+	rbd ls | wc -l | grep "^0$"
+    fi
+
+    # remove with rbd_children object missing (and, by extension,
+    # with child not mentioned in rbd_children)
+    rbd create --image-format 2 -s 1 test2
+    rbd snap create test2@snap
+    rbd snap protect test2@snap
+    rbd clone test2@snap clone --rbd-default-clone-format 1
+
+    rados -p rbd rm rbd_children
+    rbd rm clone
+    rbd ls | grep clone | wc -l | grep '^0$'
+
+    rbd snap unprotect test2@snap
+    rbd snap rm test2@snap
+    rbd rm test2
+}
+
+test_locking() {
+    echo "testing locking..."
+    remove_images
+
+    rbd create $RBD_CREATE_ARGS -s 1 test1
+    rbd lock list test1 | wc -l | grep '^0$'
+    rbd lock add test1 id
+    rbd lock list test1 | grep ' 1 '
+    LOCKER=$(rbd lock list test1 | tail -n 1 | awk '{print $1;}')
+    rbd lock remove test1 id $LOCKER
+    rbd lock list test1 | wc -l | grep '^0$'
+
+    rbd lock add test1 id --shared tag
+    rbd lock list test1 | grep ' 1 '
+    rbd lock add test1 id --shared tag
+    rbd lock list test1 | grep ' 2 '
+    rbd lock add test1 id2 --shared tag
+    rbd lock list test1 | grep ' 3 '
+    rbd lock list test1 | tail -n 1 | awk '{print $2, $1;}' | xargs rbd lock remove test1
+    if rbd info test1 | grep -qE "features:.*exclusive"
+    then
+      # new locking functionality requires all locks to be released
+      while [ -n "$(rbd lock list test1)" ]
+      do
+        rbd lock list test1 | tail -n 1 | awk '{print $2, $1;}' | xargs rbd lock remove test1
+      done
+    fi
+    rbd rm test1
+}
+
+test_pool_image_args() {
+    echo "testing pool and image args..."
+    remove_images
+
+    ceph osd pool delete test test --yes-i-really-really-mean-it || true
+    ceph osd pool create test 32
+    rbd pool init test
+    truncate -s 1 /tmp/empty /tmp/empty@snap
+
+    rbd ls | wc -l | grep 0
+    rbd create -s 1 test1
+    rbd ls | grep -q test1
+    rbd import --image test2 /tmp/empty
+    rbd ls | grep -q test2
+    rbd --dest test3 import /tmp/empty
+    rbd ls | grep -q test3
+    rbd import /tmp/empty foo
+    rbd ls | grep -q foo
+
+    # should fail due to "destination snapname specified"
+    rbd import --dest test/empty@snap /tmp/empty && exit 1 || true
+    rbd import /tmp/empty test/empty@snap && exit 1 || true
+    rbd import --image test/empty@snap /tmp/empty && exit 1 || true
+    rbd import /tmp/empty@snap && exit 1 || true
+
+    rbd ls test | wc -l | grep 0
+    rbd import /tmp/empty test/test1
+    rbd ls test | grep -q test1
+    rbd -p test import /tmp/empty test2
+    rbd ls test | grep -q test2
+    rbd --image test3 -p test import /tmp/empty
+    rbd ls test | grep -q test3
+    rbd --image test4 -p test import /tmp/empty
+    rbd ls test | grep -q test4
+    rbd --dest test5 -p test import /tmp/empty
+    rbd ls test | grep -q test5
+    rbd --dest test6 --dest-pool test import /tmp/empty
+    rbd ls test | grep -q test6
+    rbd --image test7 --dest-pool test import /tmp/empty
+    rbd ls test | grep -q test7
+    rbd --image test/test8 import /tmp/empty
+    rbd ls test | grep -q test8
+    rbd --dest test/test9 import /tmp/empty
+    rbd ls test | grep -q test9
+    rbd import --pool test /tmp/empty
+    rbd ls test | grep -q empty
+
+    # copy with no explicit pool goes to pool rbd
+    rbd copy test/test9 test10
+    rbd ls test | grep -qv test10
+    rbd ls | grep -q test10
+    rbd copy test/test9 test/test10
+    rbd ls test | grep -q test10
+    rbd copy --pool test test10 --dest-pool test test11
+    rbd ls test | grep -q test11
+    rbd copy --dest-pool rbd --pool test test11 test12
+    rbd ls | grep test12
+    rbd ls test | grep -qv test12
+
+    rm -f /tmp/empty /tmp/empty@snap
+    ceph osd pool delete test test --yes-i-really-really-mean-it
+
+    for f in foo test1 test10 test12 test2 test3 ; do
+	rbd rm $f
+    done
+}
+
+test_clone() {
+    echo "testing clone..."
+    remove_images
+    rbd create test1 $RBD_CREATE_ARGS -s 1
+    rbd snap create test1@s1
+    rbd snap protect test1@s1
+
+    ceph osd pool create rbd2 8
+    rbd pool init rbd2
+    rbd clone test1@s1 rbd2/clone
+    rbd -p rbd2 ls | grep clone
+    rbd -p rbd2 ls -l | grep clone | grep test1@s1
+    rbd ls | grep -v clone
+    rbd flatten rbd2/clone
+    rbd snap create rbd2/clone@s1
+    rbd snap protect rbd2/clone@s1
+    rbd clone rbd2/clone@s1 clone2
+    rbd ls | grep clone2
+    rbd ls -l | grep clone2 | grep rbd2/clone@s1
+    rbd -p rbd2 ls | grep -v clone2
+
+    rbd rm clone2
+    rbd snap unprotect rbd2/clone@s1
+    rbd snap rm rbd2/clone@s1
+    rbd rm rbd2/clone
+    rbd snap unprotect test1@s1
+    rbd snap rm test1@s1
+    rbd rm test1
+    ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it
+}
+
+test_trash() {
+    echo "testing trash..."
+    remove_images
+
+    rbd create $RBD_CREATE_ARGS -s 1 test1
+    rbd create $RBD_CREATE_ARGS -s 1 test2
+    rbd ls | grep test1
+    rbd ls | grep test2
+    rbd ls | wc -l | grep 2
+    rbd ls -l | grep 'test1.*2.*'
+    rbd ls -l | grep 'test2.*2.*'
+
+    rbd trash mv test1
+    rbd ls | grep test2
+    rbd ls | wc -l | grep 1
+    rbd ls -l | grep 'test2.*2.*'
+
+    rbd trash ls | grep test1
+    rbd trash ls | wc -l | grep 1
+    rbd trash ls -l | grep 'test1.*USER.*'
+    rbd trash ls -l | grep -v 'protected until'
+
+    ID=`rbd trash ls | cut -d ' ' -f 1`
+    rbd trash rm $ID
+
+    rbd trash mv test2
+    ID=`rbd trash ls | cut -d ' ' -f 1`
+    rbd info --image-id $ID | grep "rbd image 'test2'"
+
+    rbd trash restore $ID
+    rbd ls | grep test2
+    rbd ls | wc -l | grep 1
+    rbd ls -l | grep 'test2.*2.*'
+
+    rbd trash mv test2 --expires-at "3600 sec"
+    rbd trash ls | grep test2
+    rbd trash ls | wc -l | grep 1
+    rbd trash ls -l | grep 'test2.*USER.*protected until'
+
+    rbd trash rm $ID 2>&1 | grep 'Deferment time has not expired'
+    rbd trash rm --image-id $ID --force
+
+    rbd create $RBD_CREATE_ARGS -s 1 test1
+    rbd snap create test1@snap1
+    rbd snap protect test1@snap1
+    rbd trash mv test1
+
+    rbd trash ls | grep test1
+    rbd trash ls | wc -l | grep 1
+    rbd trash ls -l | grep 'test1.*USER.*'
+    rbd trash ls -l | grep -v 'protected until'
+
+    ID=`rbd trash ls | cut -d ' ' -f 1`
+    rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 1
+    rbd snap ls --image-id $ID | grep '.*snap1.*'
+
+    rbd snap unprotect --image-id $ID --snap snap1
+    rbd snap rm --image-id $ID --snap snap1
+    rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 0
+
+    rbd trash restore $ID
+    rbd snap create test1@snap1
+    rbd snap create test1@snap2
+    rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 2
+    rbd snap purge --image-id $ID
+    rbd snap ls --image-id $ID | grep -v 'SNAPID' | wc -l | grep 0
+
+    rbd rm --rbd_move_to_trash_on_remove=true --rbd_move_to_trash_on_remove_expire_seconds=3600 test1
+    rbd trash ls | grep test1
+    rbd trash ls | wc -l | grep 1
+    rbd trash ls -l | grep 'test1.*USER.*protected until'
+    rbd trash rm $ID 2>&1 | grep 'Deferment time has not expired'
+    rbd trash rm --image-id $ID --force
+
+    remove_images
+}
+
+test_purge() {
+    echo "testing trash purge..."
+    remove_images
+
+    rbd trash purge
+    rbd trash ls | wc -l | grep 0
+
+    rbd create $RBD_CREATE_ARGS foo -s 1
+    rbd create $RBD_CREATE_ARGS bar -s 1
+
+    rbd trash mv foo --expires-at "10 sec"
+    rbd trash mv bar --expires-at "30 sec"
+
+    rbd trash purge --expired-before "now + 10 sec"
+    rbd trash ls | grep -v foo | wc -l | grep 1
+    rbd trash ls | grep bar
+
+    LAST_IMG=$(rbd trash ls | grep bar | awk '{print $1;}')
+    rbd trash rm $LAST_IMG --force --no-progress | grep -v '.' | wc -l | grep 0
+}
+
+test_deep_copy_clone() {
+    echo "testing deep copy clone..."
+    remove_images
+
+    rbd create testimg1 $RBD_CREATE_ARGS --size 256
+    rbd snap create testimg1 --snap=snap1
+    rbd snap protect testimg1@snap1
+    rbd clone testimg1@snap1 testimg2
+    rbd snap create testimg2@snap2
+    rbd deep copy testimg2 testimg3
+    rbd info testimg3 | grep 'size 256 MiB'
+    rbd info testimg3 | grep 'parent: rbd/testimg1@snap1'
+    rbd snap ls testimg3 | grep -v 'SNAPID' | wc -l | grep 1
+    rbd snap ls testimg3 | grep '.*snap2.*'
+    rbd info testimg2 | grep 'features:.*deep-flatten' || rbd snap rm testimg2@snap2
+    rbd info testimg3 | grep 'features:.*deep-flatten' || rbd snap rm testimg3@snap2
+    rbd flatten testimg2
+    rbd flatten testimg3
+    rbd snap unprotect testimg1@snap1
+    rbd snap purge testimg2
+    rbd snap purge testimg3
+    rbd rm testimg2
+    rbd rm testimg3
+
+    rbd snap protect testimg1@snap1
+    rbd clone testimg1@snap1 testimg2
+    rbd snap create testimg2@snap2
+    rbd deep copy --flatten testimg2 testimg3
+    rbd info testimg3 | grep 'size 256 MiB'
+    rbd info testimg3 | grep -v 'parent:'
+    rbd snap ls testimg3 | grep -v 'SNAPID' | wc -l | grep 1
+    rbd snap ls testimg3 | grep '.*snap2.*'
+    rbd info testimg2 | grep 'features:.*deep-flatten' || rbd snap rm testimg2@snap2
+    rbd flatten testimg2
+    rbd snap unprotect testimg1@snap1
+
+    remove_images
+}
+
+test_clone_v2() {
+    echo "testing clone v2..."
+    remove_images
+
+    rbd create $RBD_CREATE_ARGS -s 1 test1
+    rbd snap create test1@1
+    rbd clone --rbd-default-clone-format=1 test1@1 test2 && exit 1 || true
+    rbd clone --rbd-default-clone-format=2 test1@1 test2
+    rbd clone --rbd-default-clone-format=2 test1@1 test3
+
+    rbd snap protect test1@1
+    rbd clone --rbd-default-clone-format=1 test1@1 test4
+
+    rbd children test1@1 | sort | tr '\n' ' ' | grep -E "test2.*test3.*test4"
+    rbd children --descendants test1 | sort | tr '\n' ' ' | grep -E "test2.*test3.*test4"
+
+    rbd remove test4
+    rbd snap unprotect test1@1
+
+    rbd snap remove test1@1
+    rbd snap list --all test1 | grep -E "trash \(1\) *$"
+
+    rbd snap create test1@2
+    rbd rm test1 2>&1 | grep 'image has snapshots'
+
+    rbd snap rm test1@2
+    rbd rm test1 2>&1 | grep 'linked clones'
+
+    rbd rm test3
+    rbd rm test1 2>&1 | grep 'linked clones'
+
+    rbd flatten test2
+    rbd snap list --all test1 | wc -l | grep '^0$'
+    rbd rm test1
+    rbd rm test2
+}
+
+test_thick_provision() {
+    echo "testing thick provision..."
+    remove_images
+
+    # Try to create small and large thick-pro image and
+    # check actual size. (64M and 4G)
+
+    # Small thick-pro image test
+    rbd create $RBD_CREATE_ARGS --thick-provision -s 64M test1
+    count=0
+    ret=""
+    while [ $count -lt 10 ]
+    do
+        rbd du|grep test1|tr -s " "|cut -d " " -f 4-5|grep '^64 MiB' && ret=$?
+        if [ "$ret" = "0" ]
+        then
+            break;
+        fi
+        count=`expr $count + 1`
+        sleep 2
+    done
+    rbd du
+    if [ "$ret" != "0" ]
+    then
+        exit 1
+    fi
+    rbd rm test1
+    rbd ls | grep test1 | wc -l | grep '^0$'
+
+    # Large thick-pro image test
+    rbd create $RBD_CREATE_ARGS --thick-provision -s 4G test1
+    count=0
+    ret=""
+    while [ $count -lt 10 ]
+    do
+        rbd du|grep test1|tr -s " "|cut -d " " -f 4-5|grep '^4 GiB' && ret=$?
+        if [ "$ret" = "0" ]
+        then
+            break;
+        fi
+        count=`expr $count + 1`
+        sleep 2
+    done
+    rbd du
+    if [ "$ret" != "0" ]
+    then
+        exit 1
+    fi
+    rbd rm test1
+    rbd ls | grep test1 | wc -l | grep '^0$'
+}
+
+test_namespace() {
+    echo "testing namespace..."
+    remove_images
+
+    rbd namespace ls | wc -l | grep '^0$'
+    rbd namespace create rbd/test1
+    rbd namespace create --pool rbd --namespace test2
+    rbd namespace create --namespace test3
+    expect_fail rbd namespace create rbd/test3
+
+    rbd namespace list | grep 'test' | wc -l | grep '^3$'
+
+    expect_fail rbd namespace remove --pool rbd missing
+
+    rbd create $RBD_CREATE_ARGS --size 1G rbd/test1/image1
+
+    # default test1 ns to test2 ns clone
+    rbd bench --io-type write --io-pattern rand --io-total 32M --io-size 4K rbd/test1/image1
+    rbd snap create rbd/test1/image1@1
+    rbd clone --rbd-default-clone-format 2 rbd/test1/image1@1 rbd/test2/image1
+    rbd snap rm rbd/test1/image1@1
+    cmp <(rbd export rbd/test1/image1 -) <(rbd export rbd/test2/image1 -)
+    rbd rm rbd/test2/image1
+
+    # default ns to test1 ns clone
+    rbd create $RBD_CREATE_ARGS --size 1G rbd/image2
+    rbd bench --io-type write --io-pattern rand --io-total 32M --io-size 4K rbd/image2
+    rbd snap create rbd/image2@1
+    rbd clone --rbd-default-clone-format 2 rbd/image2@1 rbd/test2/image2
+    rbd snap rm rbd/image2@1
+    cmp <(rbd export rbd/image2 -) <(rbd export rbd/test2/image2 -)
+    expect_fail rbd rm rbd/image2
+    rbd rm rbd/test2/image2
+    rbd rm rbd/image2
+
+    # v1 clones are supported within the same namespace
+    rbd create $RBD_CREATE_ARGS --size 1G rbd/test1/image3
+    rbd snap create rbd/test1/image3@1
+    rbd snap protect rbd/test1/image3@1
+    rbd clone --rbd-default-clone-format 1 rbd/test1/image3@1 rbd/test1/image4
+    rbd rm rbd/test1/image4
+    rbd snap unprotect rbd/test1/image3@1
+    rbd snap rm rbd/test1/image3@1
+    rbd rm rbd/test1/image3
+
+    rbd create $RBD_CREATE_ARGS --size 1G --namespace test1 image2
+    expect_fail rbd namespace remove rbd/test1
+
+    rbd group create rbd/test1/group1
+    rbd group image add rbd/test1/group1 rbd/test1/image1
+    rbd group rm rbd/test1/group1
+
+    rbd trash move rbd/test1/image1
+    ID=`rbd trash --namespace test1 ls | cut -d ' ' -f 1`
+    rbd trash rm rbd/test1/${ID}
+
+    rbd remove rbd/test1/image2
+
+    rbd namespace remove --pool rbd --namespace test1
+    rbd namespace remove --namespace test3
+
+    rbd namespace list | grep 'test' | wc -l | grep '^1$'
+    rbd namespace remove rbd/test2
+}
+
+get_migration_state() {
+    local image=$1
+
+    rbd --format xml status $image |
+        $XMLSTARLET sel -t -v '//status/migration/state'
+}
+
+test_migration() {
+    echo "testing migration..."
+    remove_images
+    ceph osd pool create rbd2 8
+    rbd pool init rbd2
+
+    # Convert to new format
+    rbd create --image-format 1 -s 128M test1
+    rbd info test1 | grep 'format: 1'
+    rbd migration prepare test1 --image-format 2
+    test "$(get_migration_state test1)" = prepared
+    rbd info test1 | grep 'format: 2'
+    rbd rm test1 && exit 1 || true
+    rbd migration execute test1
+    test "$(get_migration_state test1)" = executed
+    rbd migration commit test1
+    get_migration_state test1 && exit 1 || true
+
+    # Enable layering (and some other features)
+    rbd info test1 | grep 'features: .*layering' && exit 1 || true
+    rbd migration prepare test1 --image-feature \
+        layering,exclusive-lock,object-map,fast-diff,deep-flatten
+    rbd info test1 | grep 'features: .*layering'
+    rbd migration execute test1
+    rbd migration commit test1
+
+    # Migration to other pool
+    rbd migration prepare test1 rbd2/test1
+    test "$(get_migration_state rbd2/test1)" = prepared
+    rbd ls | wc -l | grep '^0$'
+    rbd -p rbd2 ls | grep test1
+    rbd migration execute test1
+    test "$(get_migration_state rbd2/test1)" = executed
+    rbd rm rbd2/test1 && exit 1 || true
+    rbd migration commit test1
+
+    # Migration to other namespace
+    rbd namespace create rbd2/ns1
+    rbd namespace create rbd2/ns2
+    rbd migration prepare rbd2/test1 rbd2/ns1/test1
+    test "$(get_migration_state rbd2/ns1/test1)" = prepared
+    rbd migration execute rbd2/test1
+    test "$(get_migration_state rbd2/ns1/test1)" = executed
+    rbd migration commit rbd2/test1
+    rbd migration prepare rbd2/ns1/test1 rbd2/ns2/test1
+    rbd migration execute rbd2/ns2/test1
+    rbd migration commit rbd2/ns2/test1
+
+    # Enable data pool
+    rbd create -s 128M test1
+    rbd migration prepare test1 --data-pool rbd2
+    rbd info test1 | grep 'data_pool: rbd2'
+    rbd migration execute test1
+    rbd migration commit test1
+
+    # testing trash
+    rbd migration prepare test1
+    expect_fail rbd trash mv test1
+    ID=`rbd trash ls -a | cut -d ' ' -f 1`
+    expect_fail rbd trash rm $ID
+    expect_fail rbd trash restore $ID
+    rbd migration abort test1
+
+    # Migrate parent
+    rbd remove test1
+    dd if=/dev/urandom bs=1M count=1 | rbd --image-format 2 import - test1
+    md5sum=$(rbd export test1 - | md5sum)
+    rbd snap create test1@snap1
+    rbd snap protect test1@snap1
+    rbd snap create test1@snap2
+    rbd clone test1@snap1 clone_v1 --rbd_default_clone_format=1
+    rbd clone test1@snap2 clone_v2 --rbd_default_clone_format=2
+    rbd info clone_v1 | fgrep 'parent: rbd/test1@snap1'
+    rbd info clone_v2 | fgrep 'parent: rbd/test1@snap2'
+    rbd info clone_v2 |grep 'op_features: clone-child'
+    test "$(rbd export clone_v1 - | md5sum)" = "${md5sum}"
+    test "$(rbd export clone_v2 - | md5sum)" = "${md5sum}"
+    test "$(rbd children test1@snap1)" = "rbd/clone_v1"
+    test "$(rbd children test1@snap2)" = "rbd/clone_v2"
+    rbd migration prepare test1 rbd2/test2
+    rbd info clone_v1 | fgrep 'parent: rbd2/test2@snap1'
+    rbd info clone_v2 | fgrep 'parent: rbd2/test2@snap2'
+    rbd info clone_v2 | fgrep 'op_features: clone-child'
+    test "$(rbd children rbd2/test2@snap1)" = "rbd/clone_v1"
+    test "$(rbd children rbd2/test2@snap2)" = "rbd/clone_v2"
+    rbd migration execute test1
+    expect_fail rbd migration commit test1
+    rbd migration commit test1 --force
+    test "$(rbd export clone_v1 - | md5sum)" = "${md5sum}"
+    test "$(rbd export clone_v2 - | md5sum)" = "${md5sum}"
+    rbd migration prepare rbd2/test2 test1
+    rbd info clone_v1 | fgrep 'parent: rbd/test1@snap1'
+    rbd info clone_v2 | fgrep 'parent: rbd/test1@snap2'
+    rbd info clone_v2 | fgrep 'op_features: clone-child'
+    test "$(rbd children test1@snap1)" = "rbd/clone_v1"
+    test "$(rbd children test1@snap2)" = "rbd/clone_v2"
+    rbd migration execute test1
+    expect_fail rbd migration commit test1
+    rbd migration commit test1 --force
+    test "$(rbd export clone_v1 - | md5sum)" = "${md5sum}"
+    test "$(rbd export clone_v2 - | md5sum)" = "${md5sum}"
+    rbd remove clone_v1
+    rbd remove clone_v2
+    rbd snap unprotect test1@snap1
+    rbd snap purge test1
+    rbd rm test1
+
+    for format in 1 2; do
+        # Abort migration after successful prepare
+        rbd create -s 128M --image-format ${format} test2
+        rbd migration prepare test2 --data-pool rbd2
+        rbd bench --io-type write --io-size 1024 --io-total 1024 test2
+        rbd migration abort test2
+        rbd bench --io-type write --io-size 1024 --io-total 1024 test2
+        rbd rm test2
+
+        # Abort migration after successful execute
+        rbd create -s 128M --image-format ${format} test2
+        rbd migration prepare test2 --data-pool rbd2
+        rbd bench --io-type write --io-size 1024 --io-total 1024 test2
+        rbd migration execute test2
+        rbd migration abort test2
+        rbd bench --io-type write --io-size 1024 --io-total 1024 test2
+        rbd rm test2
+
+        # Migration is automatically aborted if prepare failed
+        rbd create -s 128M --image-format ${format} test2
+        rbd migration prepare test2 --data-pool INVALID_DATA_POOL && exit 1 || true
+        rbd bench --io-type write --io-size 1024 --io-total 1024 test2
+        rbd rm test2
+
+        # Abort migration to other pool
+        rbd create -s 128M --image-format ${format} test2
+        rbd migration prepare test2 rbd2/test2
+        rbd bench --io-type write --io-size 1024 --io-total 1024 rbd2/test2
+        rbd migration abort test2
+        rbd bench --io-type write --io-size 1024 --io-total 1024 test2
+        rbd rm test2
+
+        # The same but abort using destination image
+        rbd create -s 128M --image-format ${format} test2
+        rbd migration prepare test2 rbd2/test2
+        rbd migration abort rbd2/test2
+        rbd bench --io-type write --io-size 1024 --io-total 1024 test2
+        rbd rm test2
+
+        test $format = 1 && continue
+
+        # Abort migration to other namespace
+        rbd create -s 128M --image-format ${format} test2
+        rbd migration prepare test2 rbd2/ns1/test3
+        rbd bench --io-type write --io-size 1024 --io-total 1024 rbd2/ns1/test3
+        rbd migration abort test2
+        rbd bench --io-type write --io-size 1024 --io-total 1024 test2
+        rbd rm test2
+    done
+
+    remove_images
+    ceph osd pool rm rbd2 rbd2 --yes-i-really-really-mean-it
+}
+
+test_config() {
+    echo "testing config..."
+    remove_images
+
+    expect_fail rbd config global set osd rbd_cache true
+    expect_fail rbd config global set global debug_ms 10
+    expect_fail rbd config global set global rbd_UNKNOWN false
+    expect_fail rbd config global set global rbd_cache INVALID
+    rbd config global set global rbd_cache false
+    rbd config global set client rbd_cache true
+    rbd config global set client.123 rbd_cache false
+    rbd config global get global rbd_cache | grep '^false$'
+    rbd config global get client rbd_cache | grep '^true$'
+    rbd config global get client.123 rbd_cache | grep '^false$'
+    expect_fail rbd config global get client.UNKNOWN rbd_cache
+    rbd config global list global | grep '^rbd_cache * false * global *$'
+    rbd config global list client | grep '^rbd_cache * true * client *$'
+    rbd config global list client.123 | grep '^rbd_cache * false * client.123 *$'
+    rbd config global list client.UNKNOWN | grep '^rbd_cache * true * client *$'
+    rbd config global rm client rbd_cache
+    expect_fail rbd config global get client rbd_cache
+    rbd config global list client | grep '^rbd_cache * false * global *$'
+    rbd config global rm client.123 rbd_cache
+    rbd config global rm global rbd_cache
+
+    rbd config pool set rbd rbd_cache true
+    rbd config pool list rbd | grep '^rbd_cache * true * pool *$'
+    rbd config pool get rbd rbd_cache | grep '^true$'
+
+    rbd create $RBD_CREATE_ARGS -s 1 test1
+
+    rbd config image list rbd/test1 | grep '^rbd_cache * true * pool *$'
+    rbd config image set rbd/test1 rbd_cache false
+    rbd config image list rbd/test1 | grep '^rbd_cache * false * image *$'
+    rbd config image get rbd/test1 rbd_cache | grep '^false$'
+    rbd config image remove rbd/test1 rbd_cache
+    expect_fail rbd config image get rbd/test1 rbd_cache
+    rbd config image list rbd/test1 | grep '^rbd_cache * true * pool *$'
+
+    rbd config pool remove rbd rbd_cache
+    expect_fail rbd config pool get rbd rbd_cache
+    rbd config pool list rbd | grep '^rbd_cache * true * config *$'
+
+    rbd rm test1
+}
+
+test_pool_image_args
+test_rename
+test_ls
+test_remove
+test_migration
+test_config
+RBD_CREATE_ARGS=""
+test_others
+test_locking
+test_thick_provision
+RBD_CREATE_ARGS="--image-format 2"
+test_others
+test_locking
+test_clone
+test_trash
+test_purge
+test_deep_copy_clone
+test_clone_v2
+test_thick_provision
+test_namespace
+
+echo OK
diff --git a/qa/workunits/rbd/concurrent.sh b/qa/workunits/rbd/concurrent.sh
new file mode 100755
index 00000000..abaad75f
--- /dev/null
+++ b/qa/workunits/rbd/concurrent.sh
@@ -0,0 +1,375 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2013 Inktank Storage, Inc.
+#
+# This is free software; see the source for copying conditions.
+# There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.
+#
+# This is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as
+# published by the Free Software Foundation version 2.
+
+# Alex Elder <elder@inktank.com>
+# January 29, 2013
+
+################################################################
+
+# The purpose of this test is to exercise paths through the rbd
+# code, making sure no bad pointer references or invalid reference
+# count operations occur in the face of concurrent activity.
+#
+# Each pass of the test creates an rbd image, maps it, and writes
+# some data into the image.  It also reads some data from all of the
+# other images that exist at the time the pass executes.  Finally,
+# the image is unmapped and removed.  The image removal completes in
+# the background.
+#
+# An iteration of the test consists of performing some number of
+# passes, initating each pass as a background job, and finally
+# sleeping for a variable delay.  The delay is initially a specified
+# value, but each iteration shortens that proportionally, such that
+# the last iteration will not delay at all.
+#
+# The result exercises concurrent creates and deletes of rbd images,
+# writes to new images, reads from both written and unwritten image
+# data (including reads concurrent with writes), and attempts to
+# unmap images being read.
+
+# Usage: concurrent [-i <iter>] [-c <count>] [-d <delay>]
+#
+# Exit status:
+#     0:  success
+#     1:  usage error
+#     2:  other runtime error
+#    99:  argument count error (programming error)
+#   100:  getopt error (internal error)
+
+################################################################
+
+set -ex
+
+# Default flag values; RBD_CONCURRENT_ITER names are intended
+# to be used in yaml scripts to pass in alternate values, e.g.:
+#    env:
+#        RBD_CONCURRENT_ITER: 20
+#        RBD_CONCURRENT_COUNT: 5
+#        RBD_CONCURRENT_DELAY: 3
+ITER_DEFAULT=${RBD_CONCURRENT_ITER:-100}
+COUNT_DEFAULT=${RBD_CONCURRENT_COUNT:-5}
+DELAY_DEFAULT=${RBD_CONCURRENT_DELAY:-5}		# seconds
+
+CEPH_SECRET_FILE=${CEPH_SECRET_FILE:-}
+CEPH_ID=${CEPH_ID:-admin}
+SECRET_ARGS=""
+if [ "${CEPH_SECRET_FILE}" ]; then
+	SECRET_ARGS="--secret $CEPH_SECRET_FILE"
+fi
+
+################################################################
+
+function setup() {
+	ID_MAX_DIR=$(mktemp -d /tmp/image_max_id.XXXXX)
+	ID_COUNT_DIR=$(mktemp -d /tmp/image_ids.XXXXXX)
+	NAMES_DIR=$(mktemp -d /tmp/image_names.XXXXXX)
+	SOURCE_DATA=$(mktemp /tmp/source_data.XXXXXX)
+
+	# Use urandom to generate SOURCE_DATA
+        dd if=/dev/urandom of=${SOURCE_DATA} bs=2048 count=66 \
+               >/dev/null 2>&1
+
+	# List of rbd id's *not* created by this script
+	export INITIAL_RBD_IDS=$(ls /sys/bus/rbd/devices)
+
+	# Set up some environment for normal teuthology test setup.
+	# This really should not be necessary but I found it was.
+
+	export CEPH_ARGS=" --name client.0"
+}
+
+function cleanup() {
+	[ ! "${ID_MAX_DIR}" ] && return
+	local id
+	local image
+
+	# Unmap mapped devices
+	for id in $(rbd_ids); do
+		image=$(cat "/sys/bus/rbd/devices/${id}/name")
+		rbd_unmap_image "${id}"
+		rbd_destroy_image "${image}"
+	done
+	# Get any leftover images
+	for image in $(rbd ls 2>/dev/null); do
+		rbd_destroy_image "${image}"
+	done
+	wait
+	sync
+	rm -f "${SOURCE_DATA}"
+	[ -d "${NAMES_DIR}" ] && rmdir "${NAMES_DIR}"
+	echo "Max concurrent rbd image count was $(get_max "${ID_COUNT_DIR}")"
+	rm -rf "${ID_COUNT_DIR}"
+	echo "Max rbd image id was $(get_max "${ID_MAX_DIR}")"
+	rm -rf "${ID_MAX_DIR}"
+}
+
+function get_max() {
+	[ $# -eq 1 ] || exit 99
+	local dir="$1"
+
+	ls -U "${dir}" | sort -n | tail -1
+}
+
+trap cleanup HUP INT QUIT
+
+# print a usage message and quit
+#
+# if a message is supplied, print that first, and then exit
+# with non-zero status
+function usage() {
+	if [ $# -gt 0 ]; then
+		echo "" >&2
+		echo "$@" >&2
+	fi
+
+	echo "" >&2
+	echo "Usage: ${PROGNAME} <options> <tests>" >&2
+	echo "" >&2
+	echo "    options:" >&2
+	echo "        -h or --help" >&2
+	echo "            show this message" >&2
+	echo "        -i or --iterations" >&2
+	echo "            iteration count (1 or more)" >&2
+	echo "        -c or --count" >&2
+	echo "            images created per iteration (1 or more)" >&2
+	echo "        -d or --delay" >&2
+	echo "            maximum delay between iterations" >&2
+	echo "" >&2
+	echo "    defaults:" >&2
+	echo "        iterations: ${ITER_DEFAULT}"
+	echo "        count: ${COUNT_DEFAULT}"
+	echo "        delay: ${DELAY_DEFAULT} (seconds)"
+	echo "" >&2
+
+	[ $# -gt 0 ] && exit 1
+
+	exit 0		# This is used for a --help
+}
+
+# parse command line arguments
+function parseargs() {
+	ITER="${ITER_DEFAULT}"
+	COUNT="${COUNT_DEFAULT}"
+	DELAY="${DELAY_DEFAULT}"
+
+	# Short option flags
+	SHORT_OPTS=""
+	SHORT_OPTS="${SHORT_OPTS},h"
+	SHORT_OPTS="${SHORT_OPTS},i:"
+	SHORT_OPTS="${SHORT_OPTS},c:"
+	SHORT_OPTS="${SHORT_OPTS},d:"
+
+	# Short option flags
+	LONG_OPTS=""
+	LONG_OPTS="${LONG_OPTS},help"
+	LONG_OPTS="${LONG_OPTS},iterations:"
+	LONG_OPTS="${LONG_OPTS},count:"
+	LONG_OPTS="${LONG_OPTS},delay:"
+
+	TEMP=$(getopt --name "${PROGNAME}" \
+		--options "${SHORT_OPTS}" \
+		--longoptions "${LONG_OPTS}" \
+		-- "$@")
+	eval set -- "$TEMP"
+
+	while [ "$1" != "--" ]; do
+		case "$1" in
+			-h|--help)
+				usage
+				;;
+			-i|--iterations)
+				ITER="$2"
+				[ "${ITER}" -lt 1 ] &&
+					usage "bad iterations value"
+				shift
+				;;
+			-c|--count)
+				COUNT="$2"
+				[ "${COUNT}" -lt 1 ] &&
+					usage "bad count value"
+				shift
+				;;
+			-d|--delay)
+				DELAY="$2"
+				shift
+				;;
+			*)
+				exit 100	# Internal error
+				;;
+		esac
+		shift
+	done
+	shift
+}
+
+function rbd_ids() {
+	[ $# -eq 0 ] || exit 99
+	local ids
+	local i
+
+	[ -d /sys/bus/rbd ] || return
+	ids=" $(echo $(ls /sys/bus/rbd/devices)) "
+	for i in ${INITIAL_RBD_IDS}; do
+		ids=${ids/ ${i} / }
+	done
+	echo ${ids}
+}
+
+function update_maxes() {
+	local ids="$@"
+	local last_id
+	# These aren't 100% safe against concurrent updates but it
+	# should be pretty close
+	count=$(echo ${ids} | wc -w)
+	touch "${ID_COUNT_DIR}/${count}"
+	last_id=${ids% }
+	last_id=${last_id##* }
+	touch "${ID_MAX_DIR}/${last_id}"
+}
+
+function rbd_create_image() {
+	[ $# -eq 0 ] || exit 99
+	local image=$(basename $(mktemp "${NAMES_DIR}/image.XXXXXX"))
+
+	rbd create "${image}" --size=1024
+	echo "${image}"
+}
+
+function rbd_image_id() {
+	[ $# -eq 1 ] || exit 99
+	local image="$1"
+
+	grep -l "${image}" /sys/bus/rbd/devices/*/name 2>/dev/null |
+		cut -d / -f 6
+}
+
+function rbd_map_image() {
+	[ $# -eq 1 ] || exit 99
+	local image="$1"
+	local id
+
+	sudo rbd map "${image}" --user "${CEPH_ID}" ${SECRET_ARGS} \
+		> /dev/null 2>&1
+
+	id=$(rbd_image_id "${image}")
+	echo "${id}"
+}
+
+function rbd_write_image() {
+	[ $# -eq 1 ] || exit 99
+	local id="$1"
+
+	# Offset and size here are meant to ensure beginning and end
+	# cross both (4K or 64K) page and (4MB) rbd object boundaries.
+	# It assumes the SOURCE_DATA file has size 66 * 2048 bytes
+	dd if="${SOURCE_DATA}" of="/dev/rbd${id}" bs=2048 seek=2015 \
+		> /dev/null 2>&1
+}
+
+# All starting and ending offsets here are selected so they are not
+# aligned on a (4 KB or 64 KB) page boundary
+function rbd_read_image() {
+	[ $# -eq 1 ] || exit 99
+	local id="$1"
+
+	# First read starting and ending at an offset before any
+	# written data.  The osd zero-fills data read from an
+	# existing rbd object, but before any previously-written
+	# data.
+	dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=3 \
+		> /dev/null 2>&1
+	# Next read starting at an offset before any written data,
+	# but ending at an offset that includes data that's been
+	# written.  The osd zero-fills unwritten data at the
+	# beginning of a read.
+	dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=1983 \
+		> /dev/null 2>&1
+	# Read the data at offset 2015 * 2048 bytes (where it was
+	# written) and make sure it matches the original data.
+	cmp --quiet "${SOURCE_DATA}" "/dev/rbd${id}" 0 4126720 ||
+		echo "MISMATCH!!!"
+	# Now read starting within the pre-written data, but ending
+	# beyond it.  The rbd client zero-fills the unwritten
+	# portion at the end of a read.
+	dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=2079 \
+		> /dev/null 2>&1
+	# Now read starting from an unwritten range within a written
+	# rbd object.  The rbd client zero-fills this.
+	dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=2115 \
+		> /dev/null 2>&1
+	# Finally read from an unwritten region which would reside
+	# in a different (non-existent) osd object.  The osd client
+	# zero-fills unwritten data when the target object doesn't
+	# exist.
+	dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=4098 \
+		> /dev/null 2>&1
+}
+
+function rbd_unmap_image() {
+	[ $# -eq 1 ] || exit 99
+	local id="$1"
+
+	sudo rbd unmap "/dev/rbd${id}"
+}
+
+function rbd_destroy_image() {
+	[ $# -eq 1 ] || exit 99
+	local image="$1"
+
+	# Don't wait for it to complete, to increase concurrency
+	rbd rm "${image}" >/dev/null 2>&1 &
+	rm -f "${NAMES_DIR}/${image}"
+}
+
+function one_pass() {
+	[ $# -eq 0 ] || exit 99
+	local image
+	local id
+	local ids
+	local i
+
+	image=$(rbd_create_image)
+	id=$(rbd_map_image "${image}")
+	ids=$(rbd_ids)
+	update_maxes "${ids}"
+	for i in ${rbd_ids}; do
+		if [ "${i}" -eq "${id}" ]; then
+			rbd_write_image "${i}"
+		else
+			rbd_read_image "${i}"
+		fi
+	done
+	rbd_unmap_image "${id}"
+	rbd_destroy_image "${image}"
+}
+
+################################################################
+
+parseargs "$@"
+
+setup
+
+for iter in $(seq 1 "${ITER}"); do
+	for count in $(seq 1 "${COUNT}"); do
+		one_pass &
+	done
+	# Sleep longer at first, overlap iterations more later.
+	# Use awk to get sub-second granularity (see sleep(1)).
+	sleep $(echo "${DELAY}" "${iter}" "${ITER}" |
+		awk '{ printf("%.2f\n", $1 - $1 * $2 / $3);}')
+
+done
+wait
+
+cleanup
+
+exit 0
diff --git a/qa/workunits/rbd/diff.sh b/qa/workunits/rbd/diff.sh
new file mode 100755
index 00000000..fbd6e064
--- /dev/null
+++ b/qa/workunits/rbd/diff.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -ex
+
+function cleanup() {
+    rbd snap purge foo || :
+    rbd rm foo || :
+    rbd snap purge foo.copy || :
+    rbd rm foo.copy || :
+    rbd snap purge foo.copy2 || :
+    rbd rm foo.copy2 || :
+    rm -f foo.diff foo.out
+}
+
+cleanup
+
+rbd create foo --size 1000
+rbd bench --io-type write foo --io-size 4096 --io-threads 5 --io-total 4096000 --io-pattern rand
+
+#rbd cp foo foo.copy
+rbd create foo.copy --size 1000
+rbd export-diff foo - | rbd import-diff - foo.copy
+
+rbd snap create foo --snap=two
+rbd bench --io-type write foo --io-size 4096 --io-threads 5 --io-total 4096000 --io-pattern rand
+rbd snap create foo --snap=three
+rbd snap create foo.copy --snap=two
+
+rbd export-diff foo@two --from-snap three foo.diff && exit 1 || true  # wrong snap order
+rm -f foo.diff
+
+rbd export-diff foo@three --from-snap two foo.diff
+rbd import-diff foo.diff foo.copy
+rbd import-diff foo.diff foo.copy && exit 1 || true   # this should fail with EEXIST on the end snap
+rbd snap ls foo.copy | grep three
+
+rbd create foo.copy2 --size 1000
+rbd import-diff foo.diff foo.copy2 && exit 1 || true   # this should fail bc the start snap dne
+
+rbd export foo foo.out
+orig=`md5sum foo.out | awk '{print $1}'`
+rm foo.out
+rbd export foo.copy foo.out
+copy=`md5sum foo.out | awk '{print $1}'`
+
+if [ "$orig" != "$copy" ]; then
+    echo does not match
+    exit 1
+fi
+
+cleanup
+
+echo OK
+
diff --git a/qa/workunits/rbd/diff_continuous.sh b/qa/workunits/rbd/diff_continuous.sh
new file mode 100755
index 00000000..b8f7e8b7
--- /dev/null
+++ b/qa/workunits/rbd/diff_continuous.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -ex
+
+max=20
+size=1500
+
+iosize=16384
+iototal=16384000
+iothreads=16
+
+parent=`uuidgen`"-parent"
+src=`uuidgen`"-src";
+dst=`uuidgen`"-dst";
+
+function cleanup() {
+    rbd snap purge $src || :
+    rbd rm $src || :
+    rbd snap purge $dst || :
+    rbd rm $dst || :
+    rbd snap unprotect $parent --snap parent || :
+    rbd snap purge $parent || :
+    rbd rm $parent || :
+}
+trap cleanup EXIT
+
+# start from a clone
+rbd create $parent --size $size --image-format 2 --stripe-count 8 --stripe-unit 65536
+rbd bench --io-type write $parent --io-size $iosize --io-threads $iothreads --io-total $iototal --io-pattern rand
+rbd snap create $parent --snap parent
+rbd snap protect $parent --snap parent
+rbd clone $parent@parent $src --stripe-count 4 --stripe-unit 262144
+rbd create $dst --size $size --image-format 2 --order 19
+
+# mirror for a while
+for s in `seq 1 $max`; do
+    rbd snap create $src --snap=snap$s
+    rbd export-diff $src@snap$s - $lastsnap | rbd import-diff - $dst  &
+    rbd bench --io-type write $src --io-size $iosize --io-threads $iothreads --io-total $iototal --io-pattern rand  &
+    wait
+    lastsnap="--from-snap snap$s"
+done
+
+#trap "" EXIT
+#exit 0
+
+# validate
+for s in `seq 1 $max`; do
+    ssum=`rbd export $src@snap$s - | md5sum`
+    dsum=`rbd export $dst@snap$s - | md5sum`
+    if [ "$ssum" != "$dsum" ]; then
+	echo different sum at snap$s
+	exit 1
+    fi
+done
+
+cleanup
+trap "" EXIT
+
+echo OK
+
diff --git a/qa/workunits/rbd/huge-tickets.sh b/qa/workunits/rbd/huge-tickets.sh
new file mode 100755
index 00000000..22853c07
--- /dev/null
+++ b/qa/workunits/rbd/huge-tickets.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# This is a test for http://tracker.ceph.com/issues/8979 and the fallout
+# from triaging it.  #8979 itself was random crashes on corrupted memory
+# due to a buffer overflow (for tickets larger than 256 bytes), further
+# inspection showed that vmalloced tickets weren't handled correctly as
+# well.
+#
+# What we are doing here is generating three huge keyrings and feeding
+# them to libceph (through 'rbd map' on a scratch image).  Bad kernels
+# will crash reliably either on corrupted memory somewhere or a bad page
+# fault in scatterwalk_pagedone().
+
+set -ex
+
+function generate_keyring() {
+    local user=$1
+    local n=$2
+
+    ceph-authtool -C -n client.$user --cap mon 'allow *' --gen-key /tmp/keyring-$user
+
+    set +x # don't pollute trace with echos
+    echo -en "\tcaps osd = \"allow rwx pool=rbd" >>/tmp/keyring-$user
+    for i in $(seq 1 $n); do
+        echo -n ", allow rwx pool=pool$i" >>/tmp/keyring-$user
+    done
+    echo "\"" >>/tmp/keyring-$user
+    set -x
+}
+
+generate_keyring foo 1000 # ~25K, kmalloc
+generate_keyring bar 20000 # ~500K, vmalloc
+generate_keyring baz 300000 # ~8M, vmalloc + sg chaining
+
+rbd create --size 1 test
+
+for user in {foo,bar,baz}; do
+    ceph auth import -i /tmp/keyring-$user
+    DEV=$(sudo rbd map -n client.$user --keyring /tmp/keyring-$user test)
+    sudo rbd unmap $DEV
+done
diff --git a/qa/workunits/rbd/image_read.sh b/qa/workunits/rbd/image_read.sh
new file mode 100755
index 00000000..ddca8356
--- /dev/null
+++ b/qa/workunits/rbd/image_read.sh
@@ -0,0 +1,680 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2013 Inktank Storage, Inc.
+#
+# This is free software; see the source for copying conditions.
+# There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.
+#
+# This is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as
+# published by the Free Software Foundation version 2.
+
+# Alex Elder <elder@inktank.com>
+# April 10, 2013
+
+################################################################
+
+# The purpose of this test is to validate that data read from a
+# mapped rbd image is what it's expected to be.
+#
+# By default it creates an image and fills it with some data.  It
+# then reads back the data at a series of offsets known to cover
+# various situations (such as reading the beginning, end, or the
+# entirety of an object, or doing a read that spans multiple
+# objects), and stashes the results in a set of local files.
+#
+# It also creates and maps a snapshot of the original image after
+# it's been filled, and reads back the same ranges of data from the
+# snapshot.  It then compares the data read back with what was read
+# back from the original image, verifying they match.
+#
+# Clone functionality is tested as well, in which case a clone is
+# made of the snapshot, and the same ranges of data are again read
+# and compared with the original.  In addition, a snapshot of that
+# clone is created, and a clone of *that* snapshot is put through
+# the same set of tests.  (Clone testing can be optionally skipped.)
+
+################################################################
+
+# Default parameter values.  Environment variables, if set, will
+# supercede these defaults.  Such variables have names that begin
+# with "IMAGE_READ_", for e.g. use IMAGE_READ_PAGE_SIZE=65536
+# to use 65536 as the page size.
+set -e
+
+DEFAULT_VERBOSE=true
+DEFAULT_TEST_CLONES=true
+DEFAULT_LOCAL_FILES=false
+DEFAULT_FORMAT=2
+DEFAULT_DOUBLE_ORDER=true
+DEFAULT_HALF_ORDER=false
+DEFAULT_PAGE_SIZE=4096
+DEFAULT_OBJECT_ORDER=22
+MIN_OBJECT_ORDER=12	# technically 9, but the rbd CLI enforces 12
+MAX_OBJECT_ORDER=32
+
+RBD_FORCE_ALLOW_V1=1
+
+PROGNAME=$(basename $0)
+
+ORIGINAL=original-$$
+SNAP1=snap1-$$
+CLONE1=clone1-$$
+SNAP2=snap2-$$
+CLONE2=clone2-$$
+
+function err() {
+	if [ $# -gt 0 ]; then
+		echo "${PROGNAME}: $@" >&2
+	fi
+	exit 2
+}
+
+function usage() {
+	if [ $# -gt 0 ]; then
+		echo "" >&2
+		echo "${PROGNAME}: $@" >&2
+	fi
+	echo "" >&2
+	echo "Usage: ${PROGNAME} [<options>]" >&2
+	echo "" >&2
+	echo "options are:" >&2
+	echo "    -o object_order" >&2
+	echo "        must be ${MIN_OBJECT_ORDER}..${MAX_OBJECT_ORDER}" >&2
+	echo "    -p page_size    (in bytes)" >&2
+	echo "        note: there must be at least 4 pages per object" >&2
+	echo "    -1" >&2
+	echo "        test using format 1 rbd images (default)" >&2
+	echo "    -2" >&2
+	echo "        test using format 2 rbd images" >&2
+	echo "    -c" >&2
+	echo "        also test rbd clone images (implies format 2)" >&2
+	echo "    -d" >&2
+	echo "        clone object order double its parent's (format 2)" >&2
+	echo "    -h" >&2
+	echo "        clone object order half of its parent's (format 2)" >&2
+	echo "    -l" >&2
+	echo "        use local files rather than rbd images" >&2
+	echo "    -v" >&2
+	echo "        disable reporting of what's going on" >&2
+	echo "" >&2
+	exit 1
+}
+
+function verbose() {
+	[ "${VERBOSE}" = true ] && echo "$@"
+	true	# Don't let the verbose test spoil our return value
+}
+
+function quiet() {
+	"$@" 2> /dev/null
+}
+
+function boolean_toggle() {
+	[ $# -eq 1 ] || exit 99
+	test "$1" = "true" && echo false || echo true
+}
+
+function parseargs() {
+	local opts="o:p:12clv"
+	local lopts="order:,page_size:,local,clone,verbose"
+	local parsed
+	local clone_order_msg
+
+	# use values from environment if available
+	VERBOSE="${IMAGE_READ_VERBOSE:-${DEFAULT_VERBOSE}}"
+	TEST_CLONES="${IMAGE_READ_TEST_CLONES:-${DEFAULT_TEST_CLONES}}"
+	LOCAL_FILES="${IMAGE_READ_LOCAL_FILES:-${DEFAULT_LOCAL_FILES}}"
+	DOUBLE_ORDER="${IMAGE_READ_DOUBLE_ORDER:-${DEFAULT_DOUBLE_ORDER}}"
+	HALF_ORDER="${IMAGE_READ_HALF_ORDER:-${DEFAULT_HALF_ORDER}}"
+	FORMAT="${IMAGE_READ_FORMAT:-${DEFAULT_FORMAT}}"
+	PAGE_SIZE="${IMAGE_READ_PAGE_SIZE:-${DEFAULT_PAGE_SIZE}}"
+	OBJECT_ORDER="${IMAGE_READ_OBJECT_ORDER:-${DEFAULT_OBJECT_ORDER}}"
+
+	parsed=$(getopt -o "${opts}" -l "${lopts}" -n "${PROGNAME}" -- "$@") ||
+		usage
+	eval set -- "${parsed}"
+	while true; do
+		case "$1" in
+		-v|--verbose)
+			VERBOSE=$(boolean_toggle "${VERBOSE}");;
+		-c|--clone)
+			TEST_CLONES=$(boolean_toggle "${TEST_CLONES}");;
+		-d|--double)
+			DOUBLE_ORDER=$(boolean_toggle "${DOUBLE_ORDER}");;
+		-h|--half)
+			HALF_ORDER=$(boolean_toggle "${HALF_ORDER}");;
+		-l|--local)
+			LOCAL_FILES=$(boolean_toggle "${LOCAL_FILES}");;
+		-1|-2)
+			FORMAT="${1:1}";;
+		-p|--page_size)
+			PAGE_SIZE="$2"; shift;;
+		-o|--order)
+			OBJECT_ORDER="$2"; shift;;
+		--)
+			shift; break;;
+		*)
+			err "getopt internal error"
+		esac
+		shift
+	done
+	[ $# -gt 0 ] && usage "excess arguments ($*)"
+
+	if [ "${TEST_CLONES}" = true ]; then
+		# If we're using different object orders for clones,
+		# make sure the limits are updated accordingly.  If
+		# both "half" and "double" are specified, just
+		# ignore them both.
+		if [ "${DOUBLE_ORDER}" = true ]; then
+			if [ "${HALF_ORDER}" = true ]; then
+				DOUBLE_ORDER=false
+				HALF_ORDER=false
+			else
+				((MAX_OBJECT_ORDER -= 2))
+			fi
+		elif [ "${HALF_ORDER}" = true ]; then
+			((MIN_OBJECT_ORDER += 2))
+		fi
+	fi
+
+	[ "${OBJECT_ORDER}" -lt "${MIN_OBJECT_ORDER}" ] &&
+		usage "object order (${OBJECT_ORDER}) must be" \
+			"at least ${MIN_OBJECT_ORDER}"
+	[ "${OBJECT_ORDER}" -gt "${MAX_OBJECT_ORDER}" ] &&
+		usage "object order (${OBJECT_ORDER}) must be" \
+			"at most ${MAX_OBJECT_ORDER}"
+
+	if [ "${TEST_CLONES}" = true ]; then
+		if [ "${DOUBLE_ORDER}" = true ]; then
+			((CLONE1_ORDER = OBJECT_ORDER + 1))
+			((CLONE2_ORDER = OBJECT_ORDER + 2))
+			clone_order_msg="double"
+		elif [ "${HALF_ORDER}" = true ]; then
+			((CLONE1_ORDER = OBJECT_ORDER - 1))
+			((CLONE2_ORDER = OBJECT_ORDER - 2))
+			clone_order_msg="half of"
+		else
+			CLONE1_ORDER="${OBJECT_ORDER}"
+			CLONE2_ORDER="${OBJECT_ORDER}"
+			clone_order_msg="the same as"
+		fi
+	fi
+
+	[ "${TEST_CLONES}" != true ] || FORMAT=2
+
+	OBJECT_SIZE=$(echo "2 ^ ${OBJECT_ORDER}" | bc)
+	OBJECT_PAGES=$(echo "${OBJECT_SIZE} / ${PAGE_SIZE}" | bc)
+	IMAGE_SIZE=$((2 * 16 * OBJECT_SIZE / (1024 * 1024)))
+	[ "${IMAGE_SIZE}" -lt 1 ] && IMAGE_SIZE=1
+	IMAGE_OBJECTS=$((IMAGE_SIZE * (1024 * 1024) / OBJECT_SIZE))
+
+	[ "${OBJECT_PAGES}" -lt 4 ] &&
+		usage "object size (${OBJECT_SIZE}) must be" \
+			"at least 4 * page size (${PAGE_SIZE})"
+
+	echo "parameters for this run:"
+	echo "    format ${FORMAT} images will be tested"
+	echo "    object order is ${OBJECT_ORDER}, so" \
+		"objects are ${OBJECT_SIZE} bytes"
+	echo "    page size is ${PAGE_SIZE} bytes, so" \
+		"there are are ${OBJECT_PAGES} pages in an object"
+	echo "    derived image size is ${IMAGE_SIZE} MB, so" \
+		"there are ${IMAGE_OBJECTS} objects in an image"
+	if [ "${TEST_CLONES}" = true ]; then
+		echo "    clone functionality will be tested"
+		echo "    object size for a clone will be ${clone_order_msg}"
+		echo "        the object size of its parent image"
+	fi
+
+	true	# Don't let the clones test spoil our return value
+}
+
+function image_dev_path() {
+	[ $# -eq 1 ] || exit 99
+	local image_name="$1"
+
+	if [ "${LOCAL_FILES}" = true ]; then
+		echo "${TEMP}/${image_name}"
+		return
+	fi
+
+	echo "/dev/rbd/rbd/${image_name}"
+}
+
+function out_data_dir() {
+	[ $# -lt 2 ] || exit 99
+	local out_data="${TEMP}/data"
+	local image_name
+
+	if [ $# -eq 1 ]; then
+		image_name="$1"
+		echo "${out_data}/${image_name}"
+	else
+		echo "${out_data}"
+	fi
+}
+
+function setup() {
+	verbose "===== setting up ====="
+	TEMP=$(mktemp -d /tmp/rbd_image_read.XXXXX)
+	mkdir -p $(out_data_dir)
+
+	# create and fill the original image with some data
+	create_image "${ORIGINAL}"
+	map_image "${ORIGINAL}"
+	fill_original
+
+	# create a snapshot of the original
+	create_image_snap "${ORIGINAL}" "${SNAP1}"
+	map_image_snap "${ORIGINAL}" "${SNAP1}"
+
+	if [ "${TEST_CLONES}" = true ]; then
+		# create a clone of the original snapshot
+		create_snap_clone "${ORIGINAL}" "${SNAP1}" \
+			"${CLONE1}" "${CLONE1_ORDER}"
+		map_image "${CLONE1}"
+
+		# create a snapshot of that clone
+		create_image_snap "${CLONE1}" "${SNAP2}"
+		map_image_snap "${CLONE1}" "${SNAP2}"
+
+		# create a clone of that clone's snapshot
+		create_snap_clone "${CLONE1}" "${SNAP2}" \
+			"${CLONE2}" "${CLONE2_ORDER}"
+		map_image "${CLONE2}"
+	fi
+}
+
+function teardown() {
+	verbose "===== cleaning up ====="
+	if [ "${TEST_CLONES}" = true ]; then
+		unmap_image "${CLONE2}"					|| true
+		destroy_snap_clone "${CLONE1}" "${SNAP2}" "${CLONE2}"	|| true
+
+		unmap_image_snap "${CLONE1}" "${SNAP2}"			|| true
+		destroy_image_snap "${CLONE1}" "${SNAP2}"		|| true
+
+		unmap_image "${CLONE1}"					|| true
+		destroy_snap_clone "${ORIGINAL}" "${SNAP1}" "${CLONE1}"	|| true
+	fi
+	unmap_image_snap "${ORIGINAL}" "${SNAP1}"			|| true
+	destroy_image_snap "${ORIGINAL}" "${SNAP1}"			|| true
+	unmap_image "${ORIGINAL}"					|| true
+	destroy_image "${ORIGINAL}"					|| true
+
+	rm -rf $(out_data_dir)
+	rmdir "${TEMP}"
+}
+
+function create_image() {
+	[ $# -eq 1 ] || exit 99
+	local image_name="$1"
+	local image_path
+	local bytes
+
+	verbose "creating image \"${image_name}\""
+	if [ "${LOCAL_FILES}" = true ]; then
+		image_path=$(image_dev_path "${image_name}")
+		bytes=$(echo "${IMAGE_SIZE} * 1024 * 1024 - 1" | bc)
+		quiet dd if=/dev/zero bs=1 count=1 seek="${bytes}" \
+			of="${image_path}"
+		return
+	fi
+
+	rbd create "${image_name}" --image-format "${FORMAT}" \
+		--size "${IMAGE_SIZE}" --order "${OBJECT_ORDER}" \
+		--image-shared
+}
+
+function destroy_image() {
+	[ $# -eq 1 ] || exit 99
+	local image_name="$1"
+	local image_path
+
+	verbose "destroying image \"${image_name}\""
+	if [ "${LOCAL_FILES}" = true ]; then
+		image_path=$(image_dev_path "${image_name}")
+		rm -f "${image_path}"
+		return
+	fi
+
+	rbd rm "${image_name}"
+}
+
+function map_image() {
+	[ $# -eq 1 ] || exit 99
+	local image_name="$1"		# can be image@snap too
+
+	if [ "${LOCAL_FILES}" = true ]; then
+		return
+	fi
+
+	sudo rbd map "${image_name}"
+}
+
+function unmap_image() {
+	[ $# -eq 1 ] || exit 99
+	local image_name="$1"		# can be image@snap too
+	local image_path
+
+	if [ "${LOCAL_FILES}" = true ]; then
+		return
+	fi
+	image_path=$(image_dev_path "${image_name}")
+
+	if [ -e "${image_path}" ]; then
+		sudo rbd unmap "${image_path}"
+	fi
+}
+
+function map_image_snap() {
+	[ $# -eq 2 ] || exit 99
+	local image_name="$1"
+	local snap_name="$2"
+	local image_snap
+
+	if [ "${LOCAL_FILES}" = true ]; then
+		return
+	fi
+
+	image_snap="${image_name}@${snap_name}"
+	map_image "${image_snap}"
+}
+
+function unmap_image_snap() {
+	[ $# -eq 2 ] || exit 99
+	local image_name="$1"
+	local snap_name="$2"
+	local image_snap
+
+	if [ "${LOCAL_FILES}" = true ]; then
+		return
+	fi
+
+	image_snap="${image_name}@${snap_name}"
+	unmap_image "${image_snap}"
+}
+
+function create_image_snap() {
+	[ $# -eq 2 ] || exit 99
+	local image_name="$1"
+	local snap_name="$2"
+	local image_snap="${image_name}@${snap_name}"
+	local image_path
+	local snap_path
+
+	verbose "creating snapshot \"${snap_name}\"" \
+		"of image \"${image_name}\""
+	if [ "${LOCAL_FILES}" = true ]; then
+		image_path=$(image_dev_path "${image_name}")
+		snap_path=$(image_dev_path "${image_snap}")
+
+		cp "${image_path}" "${snap_path}"
+		return
+	fi
+
+	rbd snap create "${image_snap}"
+}
+
+function destroy_image_snap() {
+	[ $# -eq 2 ] || exit 99
+	local image_name="$1"
+	local snap_name="$2"
+	local image_snap="${image_name}@${snap_name}"
+	local snap_path
+
+	verbose "destroying snapshot \"${snap_name}\"" \
+		"of image \"${image_name}\""
+	if [ "${LOCAL_FILES}" = true ]; then
+		snap_path=$(image_dev_path "${image_snap}")
+		rm -rf "${snap_path}"
+		return
+	fi
+
+	rbd snap rm "${image_snap}"
+}
+
+function create_snap_clone() {
+	[ $# -eq 4 ] || exit 99
+	local image_name="$1"
+	local snap_name="$2"
+	local clone_name="$3"
+	local clone_order="$4"
+	local image_snap="${image_name}@${snap_name}"
+	local snap_path
+	local clone_path
+
+	verbose "creating clone image \"${clone_name}\"" \
+		"of image snapshot \"${image_name}@${snap_name}\""
+	if [ "${LOCAL_FILES}" = true ]; then
+		snap_path=$(image_dev_path "${image_name}@${snap_name}")
+		clone_path=$(image_dev_path "${clone_name}")
+
+		cp "${snap_path}" "${clone_path}"
+		return
+	fi
+
+	rbd snap protect "${image_snap}"
+	rbd clone --order "${clone_order}" --image-shared \
+		"${image_snap}" "${clone_name}"
+}
+
+function destroy_snap_clone() {
+	[ $# -eq 3 ] || exit 99
+	local image_name="$1"
+	local snap_name="$2"
+	local clone_name="$3"
+	local image_snap="${image_name}@${snap_name}"
+	local clone_path
+
+	verbose "destroying clone image \"${clone_name}\""
+	if [ "${LOCAL_FILES}" = true ]; then
+		clone_path=$(image_dev_path "${clone_name}")
+
+		rm -rf "${clone_path}"
+		return
+	fi
+
+	rbd rm "${clone_name}"
+	rbd snap unprotect "${image_snap}"
+}
+
+# function that produces "random" data with which to fill the image
+function source_data() {
+	while quiet dd if=/bin/bash skip=$(($$ % 199)) bs="${PAGE_SIZE}"; do
+		:	# Just do the dd
+	done
+}
+
+function fill_original() {
+	local image_path=$(image_dev_path "${ORIGINAL}")
+
+	verbose "filling original image"
+	# Fill 16 objects worth of "random" data
+	source_data |
+	quiet dd bs="${PAGE_SIZE}" count=$((16 * OBJECT_PAGES)) \
+		of="${image_path}"
+}
+
+function do_read() {
+	[ $# -eq 3 -o $# -eq 4 ] || exit 99
+	local image_name="$1"
+	local offset="$2"
+	local length="$3"
+	[ "${length}" -gt 0 ] || err "do_read: length must be non-zero"
+	local image_path=$(image_dev_path "${image_name}")
+	local out_data=$(out_data_dir "${image_name}")
+	local range=$(printf "%06u~%04u" "${offset}" "${length}")
+	local out_file
+
+	[ $# -eq 4 ] && offset=$((offset + 16 * OBJECT_PAGES))
+
+	verbose "reading \"${image_name}\" pages ${range}"
+
+	out_file="${out_data}/pages_${range}"
+
+	quiet dd bs="${PAGE_SIZE}" skip="${offset}" count="${length}" \
+		if="${image_path}" of="${out_file}"
+}
+
+function one_pass() {
+	[ $# -eq 1 -o $# -eq 2 ] || exit 99
+	local image_name="$1"
+	local extended
+	[ $# -eq 2 ] && extended="true"
+	local offset
+	local length
+
+	offset=0
+
+	# +-----------+-----------+---
+	# |X:X:X...X:X| : : ... : | :
+	# +-----------+-----------+---
+	length="${OBJECT_PAGES}"
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# ---+-----------+---
+	#  : |X: : ... : | :
+	# ---+-----------+---
+	length=1
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# ---+-----------+---
+	#  : | :X: ... : | :
+	# ---+-----------+---
+	length=1
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# ---+-----------+---
+	#  : | : :X...X: | :
+	# ---+-----------+---
+	length=$((OBJECT_PAGES - 3))
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# ---+-----------+---
+	#  : | : : ... :X| :
+	# ---+-----------+---
+	length=1
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# ---+-----------+---
+	#  : |X:X:X...X:X| :
+	# ---+-----------+---
+	length="${OBJECT_PAGES}"
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	offset=$((offset + 1))		# skip 1
+
+	# ---+-----------+---
+	#  : | :X:X...X:X| :
+	# ---+-----------+---
+	length=$((OBJECT_PAGES - 1))
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# ---+-----------+-----------+---
+	#  : |X:X:X...X:X|X: : ... : | :
+	# ---+-----------+-----------+---
+	length=$((OBJECT_PAGES + 1))
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# ---+-----------+-----------+---
+	#  : | :X:X...X:X|X: : ... : | :
+	# ---+-----------+-----------+---
+	length="${OBJECT_PAGES}"
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# ---+-----------+-----------+---
+	#  : | :X:X...X:X|X:X: ... : | :
+	# ---+-----------+-----------+---
+	length=$((OBJECT_PAGES + 1))
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# ---+-----------+-----------+---
+	#  : | : :X...X:X|X:X:X...X:X| :
+	# ---+-----------+-----------+---
+	length=$((2 * OBJECT_PAGES + 2))
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	offset=$((offset + 1))		# skip 1
+
+	# ---+-----------+-----------+-----
+	#  : | :X:X...X:X|X:X:X...X:X|X: :
+	# ---+-----------+-----------+-----
+	length=$((2 * OBJECT_PAGES))
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	offset=$((offset + length))
+
+	# --+-----------+-----------+--------
+	#  : | :X:X...X:X|X:X:X...X:X|X:X: :
+	# --+-----------+-----------+--------
+	length=2049
+	length=$((2 * OBJECT_PAGES + 1))
+	do_read "${image_name}" "${offset}" "${length}" ${extended}
+	# offset=$((offset + length))
+}
+
+function run_using() {
+	[ $# -eq 1 ] || exit 99
+	local image_name="$1"
+	local out_data=$(out_data_dir "${image_name}")
+
+	verbose "===== running using \"${image_name}\" ====="
+	mkdir -p "${out_data}"
+	one_pass "${image_name}"
+	one_pass "${image_name}" extended
+}
+
+function compare() {
+	[ $# -eq 1 ] || exit 99
+	local image_name="$1"
+	local out_data=$(out_data_dir "${image_name}")
+	local original=$(out_data_dir "${ORIGINAL}")
+
+	verbose "===== comparing \"${image_name}\" ====="
+	for i in $(ls "${original}"); do
+		verbose compare "\"${image_name}\" \"${i}\""
+		cmp "${original}/${i}" "${out_data}/${i}"
+	done
+	[ "${image_name}" = "${ORIGINAL}" ] || rm -rf "${out_data}"
+}
+
+function doit() {
+	[ $# -eq 1 ] || exit 99
+	local image_name="$1"
+
+	run_using "${image_name}"
+	compare "${image_name}"
+}
+
+########## Start
+
+parseargs "$@"
+
+trap teardown EXIT HUP INT
+setup
+
+run_using "${ORIGINAL}"
+doit "${ORIGINAL}@${SNAP1}"
+if [ "${TEST_CLONES}" = true ]; then
+	doit "${CLONE1}"
+	doit "${CLONE1}@${SNAP2}"
+	doit "${CLONE2}"
+fi
+rm -rf $(out_data_dir "${ORIGINAL}")
+
+echo "Success!"
+
+exit 0
diff --git a/qa/workunits/rbd/import_export.sh b/qa/workunits/rbd/import_export.sh
new file mode 100755
index 00000000..89e8d35c
--- /dev/null
+++ b/qa/workunits/rbd/import_export.sh
@@ -0,0 +1,259 @@
+#!/bin/sh -ex
+
+# V1 image unsupported but required for testing purposes
+export RBD_FORCE_ALLOW_V1=1
+
+# returns data pool for a given image
+get_image_data_pool () {
+    image=$1
+    data_pool=$(rbd info $image | grep "data_pool: " | awk -F':' '{ print $NF }')
+    if [ -z $data_pool ]; then
+       data_pool='rbd'
+    fi
+
+    echo $data_pool
+}
+
+# return list of object numbers populated in image
+objects () {
+   image=$1
+   prefix=$(rbd info $image | grep block_name_prefix | awk '{print $NF;}')
+
+   # strip off prefix and leading zeros from objects; sort, although
+   # it doesn't necessarily make sense as they're hex, at least it makes
+   # the list repeatable and comparable
+   objects=$(rados ls -p $(get_image_data_pool $image) | grep $prefix | \
+       sed -e 's/'$prefix'\.//' -e 's/^0*\([0-9a-f]\)/\1/' | sort -u)
+   echo $objects
+}
+
+# return false if either files don't compare or their ondisk
+# sizes don't compare
+
+compare_files_and_ondisk_sizes () {
+    cmp -l $1 $2 || return 1
+    origsize=$(stat $1 --format %b)
+    exportsize=$(stat $2 --format %b)
+    difference=$(($exportsize - $origsize))
+    difference=${difference#-} # absolute value
+    test $difference -ge 0 -a $difference -lt 4096
+}
+
+TMPDIR=/tmp/rbd_import_export_$$
+rm -rf $TMPDIR
+mkdir $TMPDIR
+trap "rm -rf $TMPDIR" INT TERM EXIT
+
+# cannot import a dir
+mkdir foo.$$
+rbd import foo.$$ foo.dir && exit 1 || true   # should fail
+rmdir foo.$$
+
+# create a sparse file
+dd if=/bin/sh of=${TMPDIR}/img bs=1k count=1 seek=10
+dd if=/bin/dd of=${TMPDIR}/img bs=1k count=10 seek=100
+dd if=/bin/rm of=${TMPDIR}/img bs=1k count=100 seek=1000
+dd if=/bin/ls of=${TMPDIR}/img bs=1k seek=10000
+dd if=/bin/ln of=${TMPDIR}/img bs=1k seek=100000
+dd if=/bin/grep of=${TMPDIR}/img bs=1k seek=1000000
+
+rbd rm testimg || true
+
+rbd import $RBD_CREATE_ARGS ${TMPDIR}/img testimg
+rbd export testimg ${TMPDIR}/img2
+rbd export testimg - > ${TMPDIR}/img3
+rbd rm testimg
+cmp ${TMPDIR}/img ${TMPDIR}/img2
+cmp ${TMPDIR}/img ${TMPDIR}/img3
+rm ${TMPDIR}/img2 ${TMPDIR}/img3
+
+# try again, importing from stdin
+rbd import $RBD_CREATE_ARGS - testimg < ${TMPDIR}/img
+rbd export testimg ${TMPDIR}/img2
+rbd export testimg - > ${TMPDIR}/img3
+rbd rm testimg
+cmp ${TMPDIR}/img ${TMPDIR}/img2
+cmp ${TMPDIR}/img ${TMPDIR}/img3
+
+rm ${TMPDIR}/img ${TMPDIR}/img2 ${TMPDIR}/img3
+
+if rbd help export | grep -q export-format; then
+    # try with --export-format for snapshots
+    dd if=/bin/dd of=${TMPDIR}/img bs=1k count=10 seek=100
+    rbd import $RBD_CREATE_ARGS ${TMPDIR}/img testimg
+    rbd snap create testimg@snap
+    rbd image-meta set testimg key1 value1
+    IMAGEMETA_BEFORE=`rbd image-meta list testimg`
+    rbd export --export-format 2 testimg ${TMPDIR}/img_v2
+    rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import
+    rbd info testimg_import
+    rbd info testimg_import@snap
+    IMAGEMETA_AFTER=`rbd image-meta list testimg_import`
+    [ "$IMAGEMETA_BEFORE" = "$IMAGEMETA_AFTER" ]
+
+    # compare the contents between testimg and testimg_import
+    rbd export testimg_import ${TMPDIR}/img_import
+    compare_files_and_ondisk_sizes ${TMPDIR}/img ${TMPDIR}/img_import
+
+    rbd export testimg@snap ${TMPDIR}/img_snap
+    rbd export testimg_import@snap ${TMPDIR}/img_snap_import
+    compare_files_and_ondisk_sizes ${TMPDIR}/img_snap ${TMPDIR}/img_snap_import
+
+    rm ${TMPDIR}/img_v2
+    rm ${TMPDIR}/img_import
+    rm ${TMPDIR}/img_snap
+    rm ${TMPDIR}/img_snap_import
+
+    rbd snap rm testimg_import@snap
+    rbd remove testimg_import
+    rbd snap rm testimg@snap
+    rbd rm testimg
+
+    # order
+    rbd import --order 20 ${TMPDIR}/img testimg
+    rbd export --export-format 2 testimg ${TMPDIR}/img_v2
+    rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import
+    rbd info testimg_import|grep order|awk '{print $2}'|grep 20
+    
+    rm ${TMPDIR}/img_v2
+
+    rbd remove testimg_import
+    rbd remove testimg
+
+    # features
+    rbd import --image-feature layering ${TMPDIR}/img testimg
+    FEATURES_BEFORE=`rbd info testimg|grep features`
+    rbd export --export-format 2 testimg ${TMPDIR}/img_v2
+    rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import
+    FEATURES_AFTER=`rbd info testimg_import|grep features`
+    if [ "$FEATURES_BEFORE" != "$FEATURES_AFTER" ]; then
+        false
+    fi
+
+    rm ${TMPDIR}/img_v2
+
+    rbd remove testimg_import
+    rbd remove testimg
+
+    # stripe
+    rbd import --stripe-count 1000 --stripe-unit 4096 ${TMPDIR}/img testimg
+    rbd export --export-format 2 testimg ${TMPDIR}/img_v2
+    rbd import --export-format 2 ${TMPDIR}/img_v2 testimg_import
+    rbd info testimg_import|grep "stripe unit"|grep -Ei '(4 KiB|4096)'
+    rbd info testimg_import|grep "stripe count"|awk '{print $3}'|grep 1000
+
+    rm ${TMPDIR}/img_v2
+
+    rbd remove testimg_import
+    rbd remove testimg
+
+    # snap protect
+    rbd import --image-format=2 ${TMPDIR}/img testimg
+    rbd snap create testimg@snap1
+    rbd snap create testimg@snap2
+    rbd snap protect testimg@snap2
+    rbd export --export-format 2 testimg ${TMPDIR}/snap_protect
+    rbd import --export-format 2 ${TMPDIR}/snap_protect testimg_import
+    rbd info testimg_import@snap1 | grep 'protected: False'
+    rbd info testimg_import@snap2 | grep 'protected: True'
+
+    rm ${TMPDIR}/snap_protect
+
+    rbd snap unprotect testimg@snap2
+    rbd snap unprotect testimg_import@snap2
+    rbd snap purge testimg
+    rbd snap purge testimg_import
+    rbd remove testimg
+    rbd remove testimg_import
+fi
+
+tiered=0
+if ceph osd dump | grep ^pool | grep "'rbd'" | grep tier; then
+    tiered=1
+fi
+
+# create specifically sparse files
+# 1 1M block of sparse, 1 1M block of random
+dd if=/dev/urandom bs=1M seek=1 count=1 of=${TMPDIR}/sparse1
+
+# 1 1M block of random, 1 1M block of sparse
+dd if=/dev/urandom bs=1M count=1 of=${TMPDIR}/sparse2; truncate ${TMPDIR}/sparse2 -s 2M
+
+# 1M-block images; validate resulting blocks
+
+# 1M sparse, 1M data
+rbd rm sparse1 || true
+rbd import $RBD_CREATE_ARGS --order 20 ${TMPDIR}/sparse1
+rbd ls -l | grep sparse1 | grep -Ei '(2 MiB|2048k)'
+[ $tiered -eq 1 -o "$(objects sparse1)" = '1' ]
+
+# export, compare contents and on-disk size
+rbd export sparse1 ${TMPDIR}/sparse1.out
+compare_files_and_ondisk_sizes ${TMPDIR}/sparse1 ${TMPDIR}/sparse1.out
+rm ${TMPDIR}/sparse1.out
+rbd rm sparse1
+
+# 1M data, 1M sparse
+rbd rm sparse2 || true
+rbd import $RBD_CREATE_ARGS --order 20 ${TMPDIR}/sparse2
+rbd ls -l | grep sparse2 | grep -Ei '(2 MiB|2048k)'
+[ $tiered -eq 1 -o "$(objects sparse2)" = '0' ]
+rbd export sparse2 ${TMPDIR}/sparse2.out
+compare_files_and_ondisk_sizes ${TMPDIR}/sparse2 ${TMPDIR}/sparse2.out
+rm ${TMPDIR}/sparse2.out
+rbd rm sparse2
+
+# extend sparse1 to 10 1M blocks, sparse at the end
+truncate ${TMPDIR}/sparse1 -s 10M
+# import from stdin just for fun, verify still sparse
+rbd import $RBD_CREATE_ARGS --order 20 - sparse1 < ${TMPDIR}/sparse1
+rbd ls -l | grep sparse1 | grep -Ei '(10 MiB|10240k)'
+[ $tiered -eq 1 -o "$(objects sparse1)" = '1' ]
+rbd export sparse1 ${TMPDIR}/sparse1.out
+compare_files_and_ondisk_sizes ${TMPDIR}/sparse1 ${TMPDIR}/sparse1.out
+rm ${TMPDIR}/sparse1.out
+rbd rm sparse1
+
+# extend sparse2 to 4M total with two more nonsparse megs
+dd if=/dev/urandom bs=2M count=1 of=${TMPDIR}/sparse2 oflag=append conv=notrunc
+# again from stding
+rbd import $RBD_CREATE_ARGS --order 20 - sparse2 < ${TMPDIR}/sparse2
+rbd ls -l | grep sparse2 | grep -Ei '(4 MiB|4096k)'
+[ $tiered -eq 1 -o "$(objects sparse2)" = '0 2 3' ]
+rbd export sparse2 ${TMPDIR}/sparse2.out
+compare_files_and_ondisk_sizes ${TMPDIR}/sparse2 ${TMPDIR}/sparse2.out
+rm ${TMPDIR}/sparse2.out
+rbd rm sparse2
+
+# zeros import to a sparse image.  Note: all zeros currently
+# doesn't work right now due to the way we handle 'empty' fiemaps;
+# the image ends up zero-filled.
+
+echo "partially-sparse file imports to partially-sparse image"
+rbd import $RBD_CREATE_ARGS --order 20 ${TMPDIR}/sparse1 sparse
+[ $tiered -eq 1 -o "$(objects sparse)" = '1' ]
+rbd rm sparse
+
+echo "zeros import through stdin to sparse image"
+# stdin
+dd if=/dev/zero bs=1M count=4 | rbd import $RBD_CREATE_ARGS - sparse
+[ $tiered -eq 1 -o "$(objects sparse)" = '' ]
+rbd rm sparse
+
+echo "zeros export to sparse file"
+#  Must be tricky to make image "by hand" ; import won't create a zero image
+rbd create $RBD_CREATE_ARGS sparse --size 4
+prefix=$(rbd info sparse | grep block_name_prefix | awk '{print $NF;}')
+# drop in 0 object directly
+dd if=/dev/zero bs=4M count=1 | rados -p $(get_image_data_pool sparse) \
+                                      put ${prefix}.000000000000 -
+[ $tiered -eq 1 -o "$(objects sparse)" = '0' ]
+# 1 object full of zeros; export should still create 0-disk-usage file
+rm ${TMPDIR}/sparse || true
+rbd export sparse ${TMPDIR}/sparse
+[ $(stat ${TMPDIR}/sparse --format=%b) = '0' ]
+rbd rm sparse
+
+rm ${TMPDIR}/sparse ${TMPDIR}/sparse1 ${TMPDIR}/sparse2 ${TMPDIR}/sparse3 || true
+
+echo OK
diff --git a/qa/workunits/rbd/issue-20295.sh b/qa/workunits/rbd/issue-20295.sh
new file mode 100755
index 00000000..3d617a06
--- /dev/null
+++ b/qa/workunits/rbd/issue-20295.sh
@@ -0,0 +1,18 @@
+#!/bin/sh -ex
+
+TEST_POOL=ecpool
+TEST_IMAGE=test1
+PGS=12
+
+ceph osd pool create $TEST_POOL $PGS $PGS erasure
+ceph osd pool application enable $TEST_POOL rbd
+ceph osd pool set $TEST_POOL allow_ec_overwrites true
+rbd --data-pool $TEST_POOL create --size 1024G $TEST_IMAGE
+rbd bench \
+    --io-type write \
+    --io-size 4096 \
+    --io-pattern=rand \
+    --io-total 100M \
+    $TEST_IMAGE
+
+echo "OK"
diff --git a/qa/workunits/rbd/journal.sh b/qa/workunits/rbd/journal.sh
new file mode 100755
index 00000000..ba89e75c
--- /dev/null
+++ b/qa/workunits/rbd/journal.sh
@@ -0,0 +1,326 @@
+#!/usr/bin/env bash
+set -e
+
+. $(dirname $0)/../../standalone/ceph-helpers.sh
+
+function list_tests()
+{
+  echo "AVAILABLE TESTS"
+  for i in $TESTS; do
+    echo "  $i"
+  done
+}
+
+function usage()
+{
+  echo "usage: $0 [-h|-l|-t <testname> [-t <testname>...] [--no-cleanup]]"
+}
+
+function expect_false()
+{
+    set -x
+    if "$@"; then return 1; else return 0; fi
+}
+
+function save_commit_position()
+{
+    local journal=$1
+
+    rados -p rbd getomapval journal.${journal} client_ \
+	  $TMPDIR/${journal}.client_.omap
+}
+
+function restore_commit_position()
+{
+    local journal=$1
+
+    rados -p rbd setomapval journal.${journal} client_ \
+	  < $TMPDIR/${journal}.client_.omap
+}
+
+test_rbd_journal()
+{
+    local image=testrbdjournal$$
+
+    rbd create --image-feature exclusive-lock --image-feature journaling \
+	--size 128 ${image}
+    local journal=$(rbd info ${image} --format=xml 2>/dev/null |
+			   $XMLSTARLET sel -t -v "//image/journal")
+    test -n "${journal}"
+    rbd journal info ${journal}
+    rbd journal info --journal ${journal}
+    rbd journal info --image ${image}
+
+    rbd feature disable ${image} journaling
+
+    rbd info ${image} --format=xml 2>/dev/null |
+	expect_false $XMLSTARLET sel -t -v "//image/journal"
+    expect_false rbd journal info ${journal}
+    expect_false rbd journal info --image ${image}
+
+    rbd feature enable ${image} journaling
+
+    local journal1=$(rbd info ${image} --format=xml 2>/dev/null |
+			    $XMLSTARLET sel -t -v "//image/journal")
+    test "${journal}" = "${journal1}"
+
+    rbd journal info ${journal}
+
+    rbd journal status ${journal}
+
+    local count=10
+    save_commit_position ${journal}
+    rbd bench --io-type write ${image} --io-size 4096 --io-threads 1 \
+	--io-total $((4096 * count)) --io-pattern seq
+    rbd journal status --image ${image} | fgrep "tid=$((count - 1))"
+    restore_commit_position ${journal}
+    rbd journal status --image ${image} | fgrep "positions=[]"
+    local count1=$(rbd journal inspect --verbose ${journal} |
+			  grep -c 'event_type.*AioWrite')
+    test "${count}" -eq "${count1}"
+
+    rbd journal export ${journal} $TMPDIR/journal.export
+    local size=$(stat -c "%s" $TMPDIR/journal.export)
+    test "${size}" -gt 0
+
+    rbd export ${image} $TMPDIR/${image}.export
+
+    local image1=${image}1
+    rbd create --image-feature exclusive-lock --image-feature journaling \
+	--size 128 ${image1}
+    journal1=$(rbd info ${image1} --format=xml 2>/dev/null |
+		      $XMLSTARLET sel -t -v "//image/journal")
+
+    save_commit_position ${journal1}
+    rbd journal import --dest ${image1} $TMPDIR/journal.export
+    rbd snap create ${image1}@test
+    restore_commit_position ${journal1}
+    # check that commit position is properly updated: the journal should contain
+    # 14 entries (2 AioFlush + 10 AioWrite + 1 SnapCreate + 1 OpFinish) and
+    # commit position set to tid=14
+    rbd journal inspect --image ${image1} --verbose | awk '
+      /AioFlush/          {a++}         # match: "event_type": "AioFlush",
+      /AioWrite/          {w++}         # match: "event_type": "AioWrite",
+      /SnapCreate/        {s++}         # match: "event_type": "SnapCreate",
+      /OpFinish/          {f++}         # match: "event_type": "OpFinish",
+      /entries inspected/ {t=$1; e=$4}  # match: 14 entries inspected, 0 errors
+                          {print}       # for diagnostic
+      END                 {
+        if (a != 2 || w != 10 || s != 1 || f != 1 || t != 14 || e != 0) exit(1)
+      }
+    '
+
+    rbd export ${image1}@test $TMPDIR/${image1}.export
+    cmp $TMPDIR/${image}.export $TMPDIR/${image1}.export
+
+    rbd journal reset ${journal}
+
+    rbd journal inspect --verbose ${journal} | expect_false grep 'event_type'
+
+    rbd snap purge ${image1}
+    rbd remove ${image1}
+    rbd remove ${image}
+}
+
+
+rbd_assert_eq() {
+    local image=$1
+    local cmd=$2
+    local param=$3
+    local expected_val=$4
+
+    local val=$(rbd --format xml ${cmd} --image ${image} |
+		       $XMLSTARLET sel -t -v "${param}")
+    test "${val}" = "${expected_val}"
+}
+
+test_rbd_create()
+{
+    local image=testrbdcreate$$
+
+    rbd create --image-feature exclusive-lock --image-feature journaling \
+	--journal-pool rbd \
+	--journal-object-size 20M \
+	--journal-splay-width 6 \
+	--size 256 ${image}
+
+    rbd_assert_eq ${image} 'journal info' '//journal/order' 25
+    rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6
+    rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd
+
+    rbd remove ${image}
+}
+
+test_rbd_copy()
+{
+    local src=testrbdcopys$$
+    rbd create --size 256 ${src}
+
+    local image=testrbdcopy$$
+    rbd copy --image-feature exclusive-lock --image-feature journaling \
+	--journal-pool rbd \
+	--journal-object-size 20M \
+	--journal-splay-width 6 \
+	${src} ${image}
+
+    rbd remove ${src}
+
+    rbd_assert_eq ${image} 'journal info' '//journal/order' 25
+    rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6
+    rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd
+
+    rbd remove ${image}
+}
+
+test_rbd_deep_copy()
+{
+    local src=testrbdcopys$$
+    rbd create --size 256 ${src}
+    rbd snap create ${src}@snap1
+
+    local dest=testrbdcopy$$
+    rbd deep copy --image-feature exclusive-lock --image-feature journaling \
+        --journal-pool rbd \
+        --journal-object-size 20M \
+        --journal-splay-width 6 \
+        ${src} ${dest}
+
+    rbd snap purge ${src}
+    rbd remove ${src}
+
+    rbd_assert_eq ${dest} 'journal info' '//journal/order' 25
+    rbd_assert_eq ${dest} 'journal info' '//journal/splay_width' 6
+    rbd_assert_eq ${dest} 'journal info' '//journal/object_pool' rbd
+
+    rbd snap purge ${dest}
+    rbd remove ${dest}
+}
+
+test_rbd_clone()
+{
+    local parent=testrbdclonep$$
+    rbd create --image-feature layering --size 256 ${parent}
+    rbd snap create ${parent}@snap
+    rbd snap protect ${parent}@snap
+
+    local image=testrbdclone$$
+    rbd clone --image-feature layering --image-feature exclusive-lock --image-feature journaling \
+	--journal-pool rbd \
+	--journal-object-size 20M \
+	--journal-splay-width 6 \
+	${parent}@snap ${image}
+
+    rbd_assert_eq ${image} 'journal info' '//journal/order' 25
+    rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6
+    rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd
+
+    rbd remove ${image}
+    rbd snap unprotect ${parent}@snap
+    rbd snap purge ${parent}
+    rbd remove ${parent}
+}
+
+test_rbd_import()
+{
+    local src=testrbdimports$$
+    rbd create --size 256 ${src}
+
+    rbd export ${src} $TMPDIR/${src}.export
+    rbd remove ${src}
+
+    local image=testrbdimport$$
+    rbd import --image-feature exclusive-lock --image-feature journaling \
+	--journal-pool rbd \
+	--journal-object-size 20M \
+	--journal-splay-width 6 \
+	$TMPDIR/${src}.export ${image}
+
+    rbd_assert_eq ${image} 'journal info' '//journal/order' 25
+    rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6
+    rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd
+
+    rbd remove ${image}
+}
+
+test_rbd_feature()
+{
+    local image=testrbdfeature$$
+
+    rbd create --image-feature exclusive-lock --size 256 ${image}
+
+    rbd feature enable ${image} journaling \
+	--journal-pool rbd \
+	--journal-object-size 20M \
+	--journal-splay-width 6
+
+    rbd_assert_eq ${image} 'journal info' '//journal/order' 25
+    rbd_assert_eq ${image} 'journal info' '//journal/splay_width' 6
+    rbd_assert_eq ${image} 'journal info' '//journal/object_pool' rbd
+
+    rbd remove ${image}
+}
+
+TESTS+=" rbd_journal"
+TESTS+=" rbd_create"
+TESTS+=" rbd_copy"
+TESTS+=" rbd_clone"
+TESTS+=" rbd_import"
+TESTS+=" rbd_feature"
+
+#
+# "main" follows
+#
+
+tests_to_run=()
+
+cleanup=true
+
+while [[ $# -gt 0 ]]; do
+    opt=$1
+
+    case "$opt" in
+	"-l" )
+	    do_list=1
+	    ;;
+	"--no-cleanup" )
+	    cleanup=false
+	    ;;
+	"-t" )
+	    shift
+	    if [[ -z "$1" ]]; then
+		echo "missing argument to '-t'"
+		usage ;
+		exit 1
+	    fi
+	    tests_to_run+=" $1"
+	    ;;
+	"-h" )
+	    usage ;
+	    exit 0
+	    ;;
+    esac
+    shift
+done
+
+if [[ $do_list -eq 1 ]]; then
+    list_tests ;
+    exit 0
+fi
+
+TMPDIR=/tmp/rbd_journal$$
+mkdir $TMPDIR
+if $cleanup; then
+    trap "rm -fr $TMPDIR" 0
+fi
+
+if test -z "$tests_to_run" ; then
+    tests_to_run="$TESTS"
+fi
+
+for i in $tests_to_run; do
+    set -x
+    test_${i}
+    set +x
+done
+
+echo OK
diff --git a/qa/workunits/rbd/kernel.sh b/qa/workunits/rbd/kernel.sh
new file mode 100755
index 00000000..faa5760e
--- /dev/null
+++ b/qa/workunits/rbd/kernel.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set -ex
+
+CEPH_SECRET_FILE=${CEPH_SECRET_FILE:-}
+CEPH_ID=${CEPH_ID:-admin}
+SECRET_ARGS=''
+if [ ! -z $CEPH_SECRET_FILE ]; then
+	SECRET_ARGS="--secret $CEPH_SECRET_FILE"
+fi
+
+TMP_FILES="/tmp/img1 /tmp/img1.small /tmp/img1.snap1 /tmp/img1.export /tmp/img1.trunc"
+
+function expect_false() {
+	if "$@"; then return 1; else return 0; fi
+}
+
+function get_device_dir {
+	local POOL=$1
+	local IMAGE=$2
+	local SNAP=$3
+	rbd device list | tail -n +2 | egrep "\s+$POOL\s+$IMAGE\s+$SNAP\s+" |
+	    awk '{print $1;}'
+}
+
+function clean_up {
+	[ -e /dev/rbd/rbd/testimg1@snap1 ] &&
+		sudo rbd device unmap /dev/rbd/rbd/testimg1@snap1
+	if [ -e /dev/rbd/rbd/testimg1 ]; then
+		sudo rbd device unmap /dev/rbd/rbd/testimg1
+		rbd snap purge testimg1 || true
+	fi
+	rbd ls | grep testimg1 > /dev/null && rbd rm testimg1 || true
+	sudo rm -f $TMP_FILES
+}
+
+clean_up
+
+trap clean_up INT TERM EXIT
+
+# create an image
+dd if=/bin/sh of=/tmp/img1 bs=1k count=1 seek=10
+dd if=/bin/dd of=/tmp/img1 bs=1k count=10 seek=100
+dd if=/bin/rm of=/tmp/img1 bs=1k count=100 seek=1000
+dd if=/bin/ls of=/tmp/img1 bs=1k seek=10000
+dd if=/bin/ln of=/tmp/img1 bs=1k seek=100000
+dd if=/dev/zero of=/tmp/img1 count=0 seek=150000
+
+# import
+rbd import /tmp/img1 testimg1
+sudo rbd device map testimg1 --user $CEPH_ID $SECRET_ARGS
+
+DEV_ID1=$(get_device_dir rbd testimg1 -)
+echo "dev_id1 = $DEV_ID1"
+cat /sys/bus/rbd/devices/$DEV_ID1/size
+cat /sys/bus/rbd/devices/$DEV_ID1/size | grep 76800000
+
+sudo dd if=/dev/rbd/rbd/testimg1 of=/tmp/img1.export
+cmp /tmp/img1 /tmp/img1.export
+
+# snapshot
+rbd snap create testimg1 --snap=snap1
+sudo rbd device map --snap=snap1 testimg1 --user $CEPH_ID $SECRET_ARGS
+
+DEV_ID2=$(get_device_dir rbd testimg1 snap1)
+cat /sys/bus/rbd/devices/$DEV_ID2/size | grep 76800000
+
+sudo dd if=/dev/rbd/rbd/testimg1@snap1 of=/tmp/img1.snap1
+cmp /tmp/img1 /tmp/img1.snap1
+
+# resize
+rbd resize testimg1 --size=40 --allow-shrink
+cat /sys/bus/rbd/devices/$DEV_ID1/size | grep 41943040
+cat /sys/bus/rbd/devices/$DEV_ID2/size | grep 76800000
+
+sudo dd if=/dev/rbd/rbd/testimg1 of=/tmp/img1.small
+cp /tmp/img1 /tmp/img1.trunc
+truncate -s 41943040 /tmp/img1.trunc
+cmp /tmp/img1.trunc /tmp/img1.small
+
+# rollback expects an unlocked image
+# (acquire and) release the lock as a side effect
+rbd bench --io-type read --io-size 1 --io-threads 1 --io-total 1 testimg1
+
+# rollback and check data again
+rbd snap rollback --snap=snap1 testimg1
+cat /sys/bus/rbd/devices/$DEV_ID1/size | grep 76800000
+cat /sys/bus/rbd/devices/$DEV_ID2/size | grep 76800000
+sudo rm -f /tmp/img1.snap1 /tmp/img1.export
+
+sudo dd if=/dev/rbd/rbd/testimg1@snap1 of=/tmp/img1.snap1
+cmp /tmp/img1 /tmp/img1.snap1
+sudo dd if=/dev/rbd/rbd/testimg1 of=/tmp/img1.export
+cmp /tmp/img1 /tmp/img1.export
+
+# zeros are returned if an image or a snapshot is removed
+expect_false cmp -n 76800000 /dev/rbd/rbd/testimg1@snap1 /dev/zero
+rbd snap rm --snap=snap1 testimg1
+cmp -n 76800000 /dev/rbd/rbd/testimg1@snap1 /dev/zero
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_data_pool.sh b/qa/workunits/rbd/krbd_data_pool.sh
new file mode 100755
index 00000000..e8fc8348
--- /dev/null
+++ b/qa/workunits/rbd/krbd_data_pool.sh
@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+
+set -ex
+
+export RBD_FORCE_ALLOW_V1=1
+
+function fill_image() {
+    local spec=$1
+
+    local dev
+    dev=$(sudo rbd map $spec)
+    xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 -W 0 $IMAGE_SIZE" $dev
+    sudo rbd unmap $dev
+}
+
+function create_clones() {
+    local spec=$1
+
+    rbd snap create $spec@snap
+    rbd snap protect $spec@snap
+
+    local pool=${spec%/*}  # pool/image is assumed
+    local image=${spec#*/}
+    local child_pool
+    for child_pool in $pool clonesonly; do
+        rbd clone $spec@snap $child_pool/$pool-$image-clone1
+        rbd clone $spec@snap --data-pool repdata $child_pool/$pool-$image-clone2
+        rbd clone $spec@snap --data-pool ecdata $child_pool/$pool-$image-clone3
+    done
+}
+
+function trigger_copyup() {
+    local spec=$1
+
+    local dev
+    dev=$(sudo rbd map $spec)
+    local i
+    {
+        for ((i = 0; i < $NUM_OBJECTS; i++)); do
+            echo pwrite -b $OBJECT_SIZE -S 0x59 $((i * OBJECT_SIZE + OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))
+        done
+        echo fsync
+        echo quit
+    } | xfs_io $dev
+    sudo rbd unmap $dev
+}
+
+function compare() {
+    local spec=$1
+    local object=$2
+
+    local dev
+    dev=$(sudo rbd map $spec)
+    local i
+    for ((i = 0; i < $NUM_OBJECTS; i++)); do
+        dd if=$dev bs=$OBJECT_SIZE count=1 skip=$i | cmp $object -
+    done
+    sudo rbd unmap $dev
+}
+
+function mkfs_and_mount() {
+    local spec=$1
+
+    local dev
+    dev=$(sudo rbd map $spec)
+    blkdiscard $dev
+    mkfs.ext4 -q -E nodiscard $dev
+    sudo mount $dev /mnt
+    sudo umount /mnt
+    sudo rbd unmap $dev
+}
+
+function list_HEADs() {
+    local pool=$1
+
+    rados -p $pool ls | while read obj; do
+        if rados -p $pool stat $obj >/dev/null 2>&1; then
+            echo $obj
+        fi
+    done
+}
+
+function count_data_objects() {
+    local spec=$1
+
+    local pool
+    pool=$(rbd info $spec | grep 'data_pool: ' | awk '{ print $NF }')
+    if [[ -z $pool ]]; then
+        pool=${spec%/*}  # pool/image is assumed
+    fi
+
+    local prefix
+    prefix=$(rbd info $spec | grep 'block_name_prefix: ' | awk '{ print $NF }')
+    rados -p $pool ls | grep -c $prefix
+}
+
+function get_num_clones() {
+    local pool=$1
+
+    rados -p $pool --format=json df |
+        python -c 'import sys, json; print json.load(sys.stdin)["pools"][0]["num_object_clones"]'
+}
+
+ceph osd pool create repdata 24 24
+rbd pool init repdata
+ceph osd erasure-code-profile set teuthologyprofile crush-failure-domain=osd m=1 k=2
+ceph osd pool create ecdata 24 24 erasure teuthologyprofile
+rbd pool init ecdata
+ceph osd pool set ecdata allow_ec_overwrites true
+ceph osd pool create rbdnonzero 24 24
+rbd pool init rbdnonzero
+ceph osd pool create clonesonly 24 24
+rbd pool init clonesonly
+
+for pool in rbd rbdnonzero; do
+    rbd create --size 200 --image-format 1 $pool/img0
+    rbd create --size 200 $pool/img1
+    rbd create --size 200 --data-pool repdata $pool/img2
+    rbd create --size 200 --data-pool ecdata $pool/img3
+done
+
+IMAGE_SIZE=$(rbd info --format=json img1 | python -c 'import sys, json; print json.load(sys.stdin)["size"]')
+OBJECT_SIZE=$(rbd info --format=json img1 | python -c 'import sys, json; print json.load(sys.stdin)["object_size"]')
+NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE))
+[[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]]
+
+OBJECT_X=$(mktemp)   # xxxx
+xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $OBJECT_SIZE" $OBJECT_X
+
+OBJECT_XY=$(mktemp)  # xxYY
+xfs_io -c "pwrite -b $OBJECT_SIZE -S 0x78 0 $((OBJECT_SIZE / 2))" \
+       -c "pwrite -b $OBJECT_SIZE -S 0x59 $((OBJECT_SIZE / 2)) $((OBJECT_SIZE / 2))" \
+       $OBJECT_XY
+
+for pool in rbd rbdnonzero; do
+    for i in {0..3}; do
+        fill_image $pool/img$i
+        if [[ $i -ne 0 ]]; then
+            create_clones $pool/img$i
+            for child_pool in $pool clonesonly; do
+                for j in {1..3}; do
+                    trigger_copyup $child_pool/$pool-img$i-clone$j
+                done
+            done
+        fi
+    done
+done
+
+# rbd_directory, rbd_children, rbd_info + img0 header + ...
+NUM_META_RBDS=$((3 + 1 + 3 * (1*2 + 3*2)))
+# rbd_directory, rbd_children, rbd_info + ...
+NUM_META_CLONESONLY=$((3 + 2 * 3 * (3*2)))
+
+[[ $(rados -p rbd ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
+[[ $(rados -p repdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
+[[ $(rados -p ecdata ls | wc -l) -eq $((1 + 14 * NUM_OBJECTS)) ]]
+[[ $(rados -p rbdnonzero ls | wc -l) -eq $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
+[[ $(rados -p clonesonly ls | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
+
+for pool in rbd rbdnonzero; do
+    for i in {0..3}; do
+        [[ $(count_data_objects $pool/img$i) -eq $NUM_OBJECTS ]]
+        if [[ $i -ne 0 ]]; then
+            for child_pool in $pool clonesonly; do
+                for j in {1..3}; do
+                    [[ $(count_data_objects $child_pool/$pool-img$i-clone$j) -eq $NUM_OBJECTS ]]
+                done
+            done
+        fi
+    done
+done
+
+[[ $(get_num_clones rbd) -eq 0 ]]
+[[ $(get_num_clones repdata) -eq 0 ]]
+[[ $(get_num_clones ecdata) -eq 0 ]]
+[[ $(get_num_clones rbdnonzero) -eq 0 ]]
+[[ $(get_num_clones clonesonly) -eq 0 ]]
+
+for pool in rbd rbdnonzero; do
+    for i in {0..3}; do
+        compare $pool/img$i $OBJECT_X
+        mkfs_and_mount $pool/img$i
+        if [[ $i -ne 0 ]]; then
+            for child_pool in $pool clonesonly; do
+                for j in {1..3}; do
+                    compare $child_pool/$pool-img$i-clone$j $OBJECT_XY
+                done
+            done
+        fi
+    done
+done
+
+# mkfs_and_mount should discard some objects everywhere but in clonesonly
+[[ $(list_HEADs rbd | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
+[[ $(list_HEADs repdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
+[[ $(list_HEADs ecdata | wc -l) -lt $((1 + 14 * NUM_OBJECTS)) ]]
+[[ $(list_HEADs rbdnonzero | wc -l) -lt $((NUM_META_RBDS + 5 * NUM_OBJECTS)) ]]
+[[ $(list_HEADs clonesonly | wc -l) -eq $((NUM_META_CLONESONLY + 6 * NUM_OBJECTS)) ]]
+
+[[ $(get_num_clones rbd) -eq $NUM_OBJECTS ]]
+[[ $(get_num_clones repdata) -eq $((2 * NUM_OBJECTS)) ]]
+[[ $(get_num_clones ecdata) -eq $((2 * NUM_OBJECTS)) ]]
+[[ $(get_num_clones rbdnonzero) -eq $NUM_OBJECTS ]]
+[[ $(get_num_clones clonesonly) -eq 0 ]]
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_exclusive_option.sh b/qa/workunits/rbd/krbd_exclusive_option.sh
new file mode 100755
index 00000000..d7bcbb6d
--- /dev/null
+++ b/qa/workunits/rbd/krbd_exclusive_option.sh
@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+set -ex
+
+function expect_false() {
+    if "$@"; then return 1; else return 0; fi
+}
+
+function assert_locked() {
+    local dev_id="${1#/dev/rbd}"
+
+    local client_addr
+    client_addr="$(< $SYSFS_DIR/$dev_id/client_addr)"
+
+    local client_id
+    client_id="$(< $SYSFS_DIR/$dev_id/client_id)"
+    # client4324 -> client.4324
+    client_id="client.${client_id#client}"
+
+    local watch_cookie
+    watch_cookie="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID |
+        grep $client_id | cut -d ' ' -f 3 | cut -d '=' -f 2)"
+    [[ $(echo -n "$watch_cookie" | grep -c '^') -eq 1 ]]
+
+    local actual
+    actual="$(rados -p rbd --format=json lock info rbd_header.$IMAGE_ID rbd_lock |
+        python -m json.tool)"
+
+    local expected
+    expected="$(cat <<EOF | python -m json.tool
+{
+    "lockers": [
+        {
+            "addr": "$client_addr",
+            "cookie": "auto $watch_cookie",
+            "description": "",
+            "expiration": "0.000000",
+            "name": "$client_id"
+        }
+    ],
+    "name": "rbd_lock",
+    "tag": "internal",
+    "type": "exclusive"
+}
+EOF
+    )"
+
+    [ "$actual" = "$expected" ]
+}
+
+function assert_unlocked() {
+    rados -p rbd --format=json lock info rbd_header.$IMAGE_ID rbd_lock |
+        grep '"lockers":\[\]'
+}
+
+function blacklist_add() {
+    local dev_id="${1#/dev/rbd}"
+
+    local client_addr
+    client_addr="$(< $SYSFS_DIR/$dev_id/client_addr)"
+
+    ceph osd blacklist add $client_addr
+}
+
+SYSFS_DIR="/sys/bus/rbd/devices"
+IMAGE_NAME="exclusive-option-test"
+
+rbd create --size 1 --image-feature '' $IMAGE_NAME
+
+IMAGE_ID="$(rbd info --format=json $IMAGE_NAME |
+    python -c "import sys, json; print json.load(sys.stdin)['block_name_prefix'].split('.')[1]")"
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+assert_unlocked
+sudo rbd unmap $DEV
+assert_unlocked
+
+expect_false sudo rbd map -o exclusive $IMAGE_NAME
+assert_unlocked
+
+expect_false sudo rbd map -o lock_on_read $IMAGE_NAME
+assert_unlocked
+
+rbd feature enable $IMAGE_NAME exclusive-lock
+rbd snap create $IMAGE_NAME@snap
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+assert_locked $DEV
+[[ $(blockdev --getro $DEV) -eq 0 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map $IMAGE_NAME@snap)
+assert_unlocked
+[[ $(blockdev --getro $DEV) -eq 1 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o ro $IMAGE_NAME)
+assert_unlocked
+[[ $(blockdev --getro $DEV) -eq 1 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
+assert_locked $DEV
+[[ $(blockdev --getro $DEV) -eq 0 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME@snap)
+assert_unlocked
+[[ $(blockdev --getro $DEV) -eq 1 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive,ro $IMAGE_NAME)
+assert_unlocked
+[[ $(blockdev --getro $DEV) -eq 1 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+# alternate syntax
+DEV=$(sudo rbd map --exclusive --read-only $IMAGE_NAME)
+assert_unlocked
+[[ $(blockdev --getro $DEV) -eq 1 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+assert_locked $DEV
+OTHER_DEV=$(sudo rbd map -o noshare $IMAGE_NAME)
+assert_locked $OTHER_DEV
+dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
+assert_locked $DEV
+dd if=/dev/urandom of=$OTHER_DEV bs=4k count=10 oflag=direct
+assert_locked $OTHER_DEV
+sudo rbd unmap $DEV
+sudo rbd unmap $OTHER_DEV
+assert_unlocked
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+assert_locked $DEV
+OTHER_DEV=$(sudo rbd map -o noshare,exclusive $IMAGE_NAME)
+assert_locked $OTHER_DEV
+dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct
+expect_false dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
+assert_locked $OTHER_DEV
+sudo rbd unmap $OTHER_DEV
+assert_unlocked
+dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct
+assert_unlocked
+dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
+assert_locked $DEV
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o lock_on_read $IMAGE_NAME)
+assert_locked $DEV
+OTHER_DEV=$(sudo rbd map -o noshare,exclusive $IMAGE_NAME)
+assert_locked $OTHER_DEV
+expect_false dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct
+expect_false dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
+sudo udevadm settle
+assert_locked $OTHER_DEV
+sudo rbd unmap $OTHER_DEV
+assert_unlocked
+dd if=$DEV of=/dev/null bs=4k count=10 iflag=direct
+assert_locked $DEV
+dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
+assert_locked $DEV
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
+assert_locked $DEV
+expect_false sudo rbd map -o noshare $IMAGE_NAME
+assert_locked $DEV
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
+assert_locked $DEV
+expect_false sudo rbd map -o noshare,exclusive $IMAGE_NAME
+assert_locked $DEV
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+assert_locked $DEV
+rbd resize --size 1G $IMAGE_NAME
+assert_unlocked
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
+assert_locked $DEV
+expect_false rbd resize --size 2G $IMAGE_NAME
+assert_locked $DEV
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+assert_locked $DEV
+dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
+{ sleep 10; blacklist_add $DEV; } &
+PID=$!
+expect_false dd if=/dev/urandom of=$DEV bs=4k count=200000 oflag=direct
+wait $PID
+# break lock
+OTHER_DEV=$(sudo rbd map -o noshare $IMAGE_NAME)
+assert_locked $OTHER_DEV
+sudo rbd unmap $DEV
+assert_locked $OTHER_DEV
+sudo rbd unmap $OTHER_DEV
+assert_unlocked
+
+# induce a watch error after 30 seconds
+DEV=$(sudo rbd map -o exclusive,osdkeepalive=60 $IMAGE_NAME)
+assert_locked $DEV
+OLD_WATCHER="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID)"
+sleep 40
+assert_locked $DEV
+NEW_WATCHER="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID)"
+# same client_id, old cookie < new cookie
+[ "$(echo "$OLD_WATCHER" | cut -d ' ' -f 2)" = \
+    "$(echo "$NEW_WATCHER" | cut -d ' ' -f 2)" ]
+[[ $(echo "$OLD_WATCHER" | cut -d ' ' -f 3 | cut -d '=' -f 2) -lt \
+    $(echo "$NEW_WATCHER" | cut -d ' ' -f 3 | cut -d '=' -f 2) ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_fallocate.sh b/qa/workunits/rbd/krbd_fallocate.sh
new file mode 100755
index 00000000..f6e80656
--- /dev/null
+++ b/qa/workunits/rbd/krbd_fallocate.sh
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+# - fallocate -z deallocates because BLKDEV_ZERO_NOUNMAP hint is ignored by
+# krbd
+#
+# - big unaligned blkdiscard and fallocate -z/-p leave the objects in place
+
+set -ex
+
+# no blkdiscard(8) in trusty
+function py_blkdiscard() {
+    local offset=$1
+
+    python <<EOF
+import fcntl, struct
+BLKDISCARD = 0x1277
+with open('$DEV', 'w') as dev:
+    fcntl.ioctl(dev, BLKDISCARD, struct.pack('QQ', $offset, $IMAGE_SIZE - $offset))
+EOF
+}
+
+# fallocate(1) in trusty doesn't support -z/-p
+function py_fallocate() {
+    local mode=$1
+    local offset=$2
+
+    python <<EOF
+import os, ctypes, ctypes.util
+FALLOC_FL_KEEP_SIZE = 0x01
+FALLOC_FL_PUNCH_HOLE = 0x02
+FALLOC_FL_ZERO_RANGE = 0x10
+libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
+with open('$DEV', 'w') as dev:
+    if libc.fallocate(dev.fileno(), ctypes.c_int($mode), ctypes.c_long($offset), ctypes.c_long($IMAGE_SIZE - $offset)):
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+EOF
+}
+
+function allocate() {
+    xfs_io -c "pwrite -b $OBJECT_SIZE -W 0 $IMAGE_SIZE" $DEV
+    assert_allocated
+}
+
+function assert_allocated() {
+    cmp <(od -xAx $DEV) - <<EOF
+000000 cdcd cdcd cdcd cdcd cdcd cdcd cdcd cdcd
+*
+$(printf %x $IMAGE_SIZE)
+EOF
+    [[ $(rados -p rbd ls | grep -c rbd_data.$IMAGE_ID) -eq $NUM_OBJECTS ]]
+}
+
+function assert_zeroes() {
+    local num_objects_expected=$1
+
+    cmp <(od -xAx $DEV) - <<EOF
+000000 0000 0000 0000 0000 0000 0000 0000 0000
+*
+$(printf %x $IMAGE_SIZE)
+EOF
+    [[ $(rados -p rbd ls | grep -c rbd_data.$IMAGE_ID) -eq $num_objects_expected ]]
+}
+
+function assert_zeroes_unaligned() {
+    local num_objects_expected=$1
+
+    cmp <(od -xAx $DEV) - <<EOF
+000000 cdcd cdcd cdcd cdcd cdcd cdcd cdcd cdcd
+*
+$(printf %x $((OBJECT_SIZE / 2))) 0000 0000 0000 0000 0000 0000 0000 0000
+*
+$(printf %x $IMAGE_SIZE)
+EOF
+    [[ $(rados -p rbd ls | grep -c rbd_data.$IMAGE_ID) -eq $num_objects_expected ]]
+    for ((i = 0; i < $num_objects_expected; i++)); do
+        rados -p rbd stat rbd_data.$IMAGE_ID.$(printf %016x $i) | egrep "(size $((OBJECT_SIZE / 2)))|(size 0)"
+    done
+}
+
+IMAGE_NAME="fallocate-test"
+
+rbd create --size 200 $IMAGE_NAME
+
+IMAGE_SIZE=$(rbd info --format=json $IMAGE_NAME | python -c 'import sys, json; print json.load(sys.stdin)["size"]')
+OBJECT_SIZE=$(rbd info --format=json $IMAGE_NAME | python -c 'import sys, json; print json.load(sys.stdin)["object_size"]')
+NUM_OBJECTS=$((IMAGE_SIZE / OBJECT_SIZE))
+[[ $((IMAGE_SIZE % OBJECT_SIZE)) -eq 0 ]]
+
+IMAGE_ID="$(rbd info --format=json $IMAGE_NAME |
+    python -c "import sys, json; print json.load(sys.stdin)['block_name_prefix'].split('.')[1]")"
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+
+# make sure -ENOENT is hidden
+assert_zeroes 0
+py_blkdiscard 0
+assert_zeroes 0
+
+# blkdev_issue_discard
+allocate
+py_blkdiscard 0
+assert_zeroes 0
+
+# blkdev_issue_zeroout w/ BLKDEV_ZERO_NOUNMAP
+allocate
+py_fallocate FALLOC_FL_ZERO_RANGE\|FALLOC_FL_KEEP_SIZE 0
+assert_zeroes 0
+
+# blkdev_issue_zeroout w/ BLKDEV_ZERO_NOFALLBACK
+allocate
+py_fallocate FALLOC_FL_PUNCH_HOLE\|FALLOC_FL_KEEP_SIZE 0
+assert_zeroes 0
+
+# unaligned blkdev_issue_discard
+allocate
+py_blkdiscard $((OBJECT_SIZE / 2))
+assert_zeroes_unaligned $NUM_OBJECTS
+
+# unaligned blkdev_issue_zeroout w/ BLKDEV_ZERO_NOUNMAP
+allocate
+py_fallocate FALLOC_FL_ZERO_RANGE\|FALLOC_FL_KEEP_SIZE $((OBJECT_SIZE / 2))
+assert_zeroes_unaligned $NUM_OBJECTS
+
+# unaligned blkdev_issue_zeroout w/ BLKDEV_ZERO_NOFALLBACK
+allocate
+py_fallocate FALLOC_FL_PUNCH_HOLE\|FALLOC_FL_KEEP_SIZE $((OBJECT_SIZE / 2))
+assert_zeroes_unaligned $NUM_OBJECTS
+
+sudo rbd unmap $DEV
+
+DEV=$(sudo rbd map -o notrim $IMAGE_NAME)
+
+# blkdev_issue_discard
+allocate
+py_blkdiscard 0 |& grep 'Operation not supported'
+assert_allocated
+
+# blkdev_issue_zeroout w/ BLKDEV_ZERO_NOUNMAP
+allocate
+py_fallocate FALLOC_FL_ZERO_RANGE\|FALLOC_FL_KEEP_SIZE 0
+assert_zeroes $NUM_OBJECTS
+
+# blkdev_issue_zeroout w/ BLKDEV_ZERO_NOFALLBACK
+allocate
+py_fallocate FALLOC_FL_PUNCH_HOLE\|FALLOC_FL_KEEP_SIZE 0 |& grep 'Operation not supported'
+assert_allocated
+
+sudo rbd unmap $DEV
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_latest_osdmap_on_map.sh b/qa/workunits/rbd/krbd_latest_osdmap_on_map.sh
new file mode 100755
index 00000000..f70f3863
--- /dev/null
+++ b/qa/workunits/rbd/krbd_latest_osdmap_on_map.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -ex
+
+function run_test() {
+    ceph osd pool create foo 12
+    rbd pool init foo
+    rbd create --size 1 foo/img
+
+    local dev
+    dev=$(sudo rbd map foo/img)
+    sudo rbd unmap $dev
+
+    ceph osd pool delete foo foo --yes-i-really-really-mean-it
+}
+
+NUM_ITER=20
+
+for ((i = 0; i < $NUM_ITER; i++)); do
+    run_test
+done
+
+rbd create --size 1 img
+DEV=$(sudo rbd map img)
+for ((i = 0; i < $NUM_ITER; i++)); do
+    run_test
+done
+sudo rbd unmap $DEV
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_namespaces.sh b/qa/workunits/rbd/krbd_namespaces.sh
new file mode 100755
index 00000000..22b05b50
--- /dev/null
+++ b/qa/workunits/rbd/krbd_namespaces.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+
+set -ex
+
+function get_block_name_prefix() {
+    rbd info --format=json $1 | python -c "import sys, json; print json.load(sys.stdin)['block_name_prefix']"
+}
+
+function do_pwrite() {
+    local spec=$1
+    local old_byte=$2
+    local new_byte=$3
+
+    local dev
+    dev=$(sudo rbd map $spec)
+    cmp <(dd if=/dev/zero bs=1M count=10 | tr \\000 \\$old_byte) $dev
+    xfs_io -c "pwrite -b 1M -S $new_byte 0 10M" $dev
+    sudo rbd unmap $dev
+}
+
+function do_cmp() {
+    local spec=$1
+    local byte=$2
+
+    local dev
+    dev=$(sudo rbd map $spec)
+    cmp <(dd if=/dev/zero bs=1M count=10 | tr \\000 \\$byte) $dev
+    sudo rbd unmap $dev
+}
+
+function gen_child_specs() {
+    local i=$1
+
+    local child_specs="foo/img$i-clone1 foo/img$i-clone2 foo/ns1/img$i-clone1 foo/ns1/img$i-clone2"
+    if [[ $i -ge 3 ]]; then
+        child_specs="$child_specs foo/ns2/img$i-clone1 foo/ns2/img$i-clone2"
+    fi
+    echo $child_specs
+}
+
+ceph osd pool create foo 12
+rbd pool init foo
+ceph osd pool create bar 12
+rbd pool init bar
+
+ceph osd set-require-min-compat-client nautilus
+rbd namespace create foo/ns1
+rbd namespace create foo/ns2
+
+SPECS=(foo/img1 foo/img2 foo/ns1/img3 foo/ns1/img4)
+
+COUNT=1
+for spec in "${SPECS[@]}"; do
+    if [[ $spec =~ img1|img3 ]]; then
+        rbd create --size 10 $spec
+    else
+        rbd create --size 10 --data-pool bar $spec
+    fi
+    do_pwrite $spec 000 $(printf %03d $COUNT)
+    rbd snap create $spec@snap
+    COUNT=$((COUNT + 1))
+done
+for i in {1..4}; do
+    for child_spec in $(gen_child_specs $i); do
+        if [[ $child_spec =~ clone1 ]]; then
+            rbd clone ${SPECS[i - 1]}@snap $child_spec
+        else
+            rbd clone --data-pool bar ${SPECS[i - 1]}@snap $child_spec
+        fi
+        do_pwrite $child_spec $(printf %03d $i) $(printf %03d $COUNT)
+        COUNT=$((COUNT + 1))
+    done
+done
+
+[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img1)) -eq 3 ]]
+[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img2)) -eq 3 ]]
+[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img3)) -eq 3 ]]
+[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img4)) -eq 3 ]]
+
+[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img1-clone1)) -eq 3 ]]
+[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img1-clone2)) -eq 3 ]]
+[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img1-clone1)) -eq 3 ]]
+[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img1-clone2)) -eq 3 ]]
+
+[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img2-clone1)) -eq 3 ]]
+[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img2-clone2)) -eq 3 ]]
+[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img2-clone1)) -eq 3 ]]
+[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img2-clone2)) -eq 3 ]]
+
+[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img3-clone1)) -eq 3 ]]
+[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img3-clone2)) -eq 3 ]]
+[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img3-clone1)) -eq 3 ]]
+[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img3-clone2)) -eq 3 ]]
+[[ $(rados -p foo -N ns2 ls | grep -c $(get_block_name_prefix foo/ns2/img3-clone1)) -eq 3 ]]
+[[ $(rados -p bar -N ns2 ls | grep -c $(get_block_name_prefix foo/ns2/img3-clone2)) -eq 3 ]]
+
+[[ $(rados -p foo ls | grep -c $(get_block_name_prefix foo/img4-clone1)) -eq 3 ]]
+[[ $(rados -p bar ls | grep -c $(get_block_name_prefix foo/img4-clone2)) -eq 3 ]]
+[[ $(rados -p foo -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img4-clone1)) -eq 3 ]]
+[[ $(rados -p bar -N ns1 ls | grep -c $(get_block_name_prefix foo/ns1/img4-clone2)) -eq 3 ]]
+[[ $(rados -p foo -N ns2 ls | grep -c $(get_block_name_prefix foo/ns2/img4-clone1)) -eq 3 ]]
+[[ $(rados -p bar -N ns2 ls | grep -c $(get_block_name_prefix foo/ns2/img4-clone2)) -eq 3 ]]
+
+COUNT=1
+for spec in "${SPECS[@]}"; do
+    do_cmp $spec $(printf %03d $COUNT)
+    COUNT=$((COUNT + 1))
+done
+for i in {1..4}; do
+    for child_spec in $(gen_child_specs $i); do
+        do_cmp $child_spec $(printf %03d $COUNT)
+        COUNT=$((COUNT + 1))
+    done
+done
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_stable_writes.sh b/qa/workunits/rbd/krbd_stable_writes.sh
new file mode 100755
index 00000000..45e04123
--- /dev/null
+++ b/qa/workunits/rbd/krbd_stable_writes.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -ex
+
+IMAGE_NAME="stable-writes-test"
+
+rbd create --size 1 $IMAGE_NAME
+DEV=$(sudo rbd map $IMAGE_NAME)
+[[ $(blockdev --getsize64 $DEV) -eq 1048576 ]]
+grep -q 1 /sys/block/${DEV#/dev/}/queue/stable_writes
+
+rbd resize --size 2 $IMAGE_NAME
+[[ $(blockdev --getsize64 $DEV) -eq 2097152 ]]
+grep -q 1 /sys/block/${DEV#/dev/}/queue/stable_writes
+
+sudo rbd unmap $DEV
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_udev_enumerate.sh b/qa/workunits/rbd/krbd_udev_enumerate.sh
new file mode 100755
index 00000000..494f958f
--- /dev/null
+++ b/qa/workunits/rbd/krbd_udev_enumerate.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+# This is a test for https://tracker.ceph.com/issues/41036, but it also
+# triggers https://tracker.ceph.com/issues/41404 in some environments.
+
+set -ex
+
+function assert_exit_codes() {
+    declare -a pids=($@)
+
+    for pid in ${pids[@]}; do
+       wait $pid
+    done
+}
+
+function run_map() {
+    declare -a pids
+
+    for i in {1..300}; do
+        sudo rbd map img$i &
+        pids+=($!)
+    done
+
+    assert_exit_codes ${pids[@]}
+    [[ $(rbd showmapped | wc -l) -eq 301 ]]
+}
+
+function run_unmap_by_dev() {
+    declare -a pids
+
+    run_map
+    for i in {0..299}; do
+        sudo rbd unmap /dev/rbd$i &
+        pids+=($!)
+    done
+
+    assert_exit_codes ${pids[@]}
+    [[ $(rbd showmapped | wc -l) -eq 0 ]]
+}
+
+function run_unmap_by_spec() {
+    declare -a pids
+
+    run_map
+    for i in {1..300}; do
+        sudo rbd unmap img$i &
+        pids+=($!)
+    done
+
+    assert_exit_codes ${pids[@]}
+    [[ $(rbd showmapped | wc -l) -eq 0 ]]
+}
+
+# Can't test with exclusive-lock, don't bother enabling deep-flatten.
+# See https://tracker.ceph.com/issues/42492.
+for i in {1..300}; do
+    rbd create --size 1 --image-feature '' img$i
+done
+
+for i in {1..30}; do
+    echo Iteration $i
+    run_unmap_by_dev
+    run_unmap_by_spec
+done
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_udev_netlink_enobufs.sh b/qa/workunits/rbd/krbd_udev_netlink_enobufs.sh
new file mode 100755
index 00000000..7c9c53a2
--- /dev/null
+++ b/qa/workunits/rbd/krbd_udev_netlink_enobufs.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# This is a test for https://tracker.ceph.com/issues/41404, verifying that udev
+# events are properly reaped while the image is being (un)mapped in the kernel.
+# UDEV_BUF_SIZE is 1M (giving us a 2M socket receive buffer), but modprobe +
+# modprobe -r generate ~28M worth of "block" events.
+
+set -ex
+
+rbd create --size 1 img
+
+ceph osd pause
+sudo rbd map img &
+PID=$!
+sudo modprobe scsi_debug max_luns=16 add_host=16 num_parts=1 num_tgts=16
+sudo udevadm settle
+sudo modprobe -r scsi_debug
+[[ $(rbd showmapped | wc -l) -eq 0 ]]
+ceph osd unpause
+wait $PID
+[[ $(rbd showmapped | wc -l) -eq 2 ]]
+sudo rbd unmap img
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_udev_netns.sh b/qa/workunits/rbd/krbd_udev_netns.sh
new file mode 100755
index 00000000..e746a682
--- /dev/null
+++ b/qa/workunits/rbd/krbd_udev_netns.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -ex
+
+sudo ip netns add ns1
+sudo ip link add veth1-ext type veth peer name veth1-int
+sudo ip link set veth1-int netns ns1
+
+sudo ip netns exec ns1 ip link set dev lo up
+sudo ip netns exec ns1 ip addr add 192.168.1.2/24 dev veth1-int
+sudo ip netns exec ns1 ip link set veth1-int up
+sudo ip netns exec ns1 ip route add default via 192.168.1.1
+
+sudo ip addr add 192.168.1.1/24 dev veth1-ext
+sudo ip link set veth1-ext up
+
+# Enable forwarding between the namespace and the default route
+# interface and set up NAT.  In case of multiple default routes,
+# just pick the first one.
+if [[ $(sysctl -n net.ipv4.ip_forward) -eq 0 ]]; then
+    sudo iptables -P FORWARD DROP
+    sudo sysctl -w net.ipv4.ip_forward=1
+fi
+IFACE="$(ip route list 0.0.0.0/0 | head -n 1 | cut -d ' ' -f 5)"
+sudo iptables -A FORWARD -i veth1-ext -o "$IFACE" -j ACCEPT
+sudo iptables -A FORWARD -i "$IFACE" -o veth1-ext -j ACCEPT
+sudo iptables -t nat -A POSTROUTING -s 192.168.1.2 -o "$IFACE" -j MASQUERADE
+
+rbd create --size 300 img
+
+DEV="$(sudo rbd map img)"
+mkfs.ext4 "$DEV"
+sudo mount "$DEV" /mnt
+sudo umount /mnt
+sudo rbd unmap "$DEV"
+
+sudo ip netns exec ns1 bash <<'EOF'
+
+set -ex
+
+DEV="/dev/rbd/rbd/img"
+[[ ! -e "$DEV" ]]
+
+# In a network namespace, "rbd map" maps the device and hangs waiting
+# for udev add uevents.  udev runs as usual (in particular creating the
+# symlink which is used here because the device node is never printed),
+# but the uevents it sends out never come because they don't cross
+# network namespace boundaries.
+set +e
+timeout 30s rbd map img
+RET=$?
+set -e
+[[ $RET -eq 124 ]]
+[[ -L "$DEV" ]]
+mkfs.ext4 -F "$DEV"
+mount "$DEV" /mnt
+umount /mnt
+
+# In a network namespace, "rbd unmap" unmaps the device and hangs
+# waiting for udev remove uevents.  udev runs as usual (removing the
+# symlink), but the uevents it sends out never come because they don't
+# cross network namespace boundaries.
+set +e
+timeout 30s rbd unmap "$DEV"
+RET=$?
+set -e
+[[ $RET -eq 124 ]]
+[[ ! -e "$DEV" ]]
+
+# Skip waiting for udev uevents with "-o noudev".
+DEV="$(rbd map -o noudev img)"
+mkfs.ext4 -F "$DEV"
+mount "$DEV" /mnt
+umount /mnt
+rbd unmap -o noudev "$DEV"
+
+EOF
+
+rbd rm img
+
+sudo iptables -t nat -D POSTROUTING -s 192.168.1.2 -o "$IFACE" -j MASQUERADE
+sudo iptables -D FORWARD -i "$IFACE" -o veth1-ext -j ACCEPT
+sudo iptables -D FORWARD -i veth1-ext -o "$IFACE" -j ACCEPT
+sudo ip netns delete ns1
+
+echo OK
diff --git a/qa/workunits/rbd/krbd_udev_symlinks.sh b/qa/workunits/rbd/krbd_udev_symlinks.sh
new file mode 100755
index 00000000..27147652
--- /dev/null
+++ b/qa/workunits/rbd/krbd_udev_symlinks.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+
+set -ex
+
+SPECS=(
+rbd/img1
+rbd/img2
+rbd/img2@snap1
+rbd/img3
+rbd/img3@snap1
+rbd/img3@snap2
+rbd/ns1/img1
+rbd/ns1/img2
+rbd/ns1/img2@snap1
+rbd/ns1/img3
+rbd/ns1/img3@snap1
+rbd/ns1/img3@snap2
+rbd/ns2/img1
+rbd/ns2/img2
+rbd/ns2/img2@snap1
+rbd/ns2/img3
+rbd/ns2/img3@snap1
+rbd/ns2/img3@snap2
+custom/img1
+custom/img1@snap1
+custom/img2
+custom/img2@snap1
+custom/img2@snap2
+custom/img3
+custom/ns1/img1
+custom/ns1/img1@snap1
+custom/ns1/img2
+custom/ns1/img2@snap1
+custom/ns1/img2@snap2
+custom/ns1/img3
+custom/ns2/img1
+custom/ns2/img1@snap1
+custom/ns2/img2
+custom/ns2/img2@snap1
+custom/ns2/img2@snap2
+custom/ns2/img3
+)
+
+ceph osd pool create custom 8
+rbd pool init custom
+
+ceph osd set-require-min-compat-client nautilus
+rbd namespace create rbd/ns1
+rbd namespace create rbd/ns2
+rbd namespace create custom/ns1
+rbd namespace create custom/ns2
+
+# create in order, images before snapshots
+for spec in "${SPECS[@]}"; do
+    if [[ "$spec" =~ snap ]]; then
+        rbd snap create "$spec"
+    else
+        rbd create --size 10 "$spec"
+        DEV="$(sudo rbd map "$spec")"
+        sudo sfdisk "$DEV" <<EOF
+unit: sectors
+${DEV}p1 : start=        2048, size=           2, type=83
+${DEV}p2 : start=        4096, size=           2, type=83
+EOF
+        sudo rbd unmap "$DEV"
+    fi
+done
+
+[[ ! -e /dev/rbd ]]
+
+# map in random order
+COUNT=${#SPECS[@]}
+read -r -a INDEXES < <(python3 <<EOF
+import random
+l = list(range($COUNT))
+random.shuffle(l)
+print(*l)
+EOF
+)
+
+DEVS=()
+for idx in "${INDEXES[@]}"; do
+    DEVS+=("$(sudo rbd map "${SPECS[idx]}")")
+done
+
+[[ $(rbd showmapped | wc -l) -eq $((COUNT + 1)) ]]
+
+for ((i = 0; i < COUNT; i++)); do
+    [[ "$(readlink -e "/dev/rbd/${SPECS[INDEXES[i]]}")" == "${DEVS[i]}" ]]
+    [[ "$(readlink -e "/dev/rbd/${SPECS[INDEXES[i]]}-part1")" == "${DEVS[i]}p1" ]]
+    [[ "$(readlink -e "/dev/rbd/${SPECS[INDEXES[i]]}-part2")" == "${DEVS[i]}p2" ]]
+done
+
+for idx in "${INDEXES[@]}"; do
+    sudo rbd unmap "/dev/rbd/${SPECS[idx]}"
+done
+
+[[ ! -e /dev/rbd ]]
+
+# remove in reverse order, snapshots before images
+for ((i = COUNT - 1; i >= 0; i--)); do
+    if [[ "${SPECS[i]}" =~ snap ]]; then
+        rbd snap rm "${SPECS[i]}"
+    else
+        rbd rm "${SPECS[i]}"
+    fi
+done
+
+rbd namespace rm custom/ns2
+rbd namespace rm custom/ns1
+rbd namespace rm rbd/ns2
+rbd namespace rm rbd/ns1
+
+ceph osd pool delete custom custom --yes-i-really-really-mean-it
+
+echo OK
diff --git a/qa/workunits/rbd/map-snapshot-io.sh b/qa/workunits/rbd/map-snapshot-io.sh
new file mode 100755
index 00000000..a69d8482
--- /dev/null
+++ b/qa/workunits/rbd/map-snapshot-io.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# http://tracker.ceph.com/issues/3964
+
+set -ex
+
+rbd create image -s 100
+DEV=$(sudo rbd map image)
+dd if=/dev/zero of=$DEV oflag=direct count=10
+rbd snap create image@s1
+dd if=/dev/zero of=$DEV oflag=direct count=10   # used to fail
+rbd snap rm image@s1
+dd if=/dev/zero of=$DEV oflag=direct count=10
+sudo rbd unmap $DEV
+rbd rm image
+
+echo OK
diff --git a/qa/workunits/rbd/map-unmap.sh b/qa/workunits/rbd/map-unmap.sh
new file mode 100755
index 00000000..99863849
--- /dev/null
+++ b/qa/workunits/rbd/map-unmap.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -ex
+
+RUN_TIME=300		# approximate duration of run (seconds)
+
+[ $# -eq 1 ] && RUN_TIME="$1"
+
+IMAGE_NAME="image-$$"
+IMAGE_SIZE="1024"	# MB
+
+function get_time() {
+	date '+%s'
+}
+
+function times_up() {
+	local end_time="$1"
+
+	test $(get_time) -ge "${end_time}"
+}
+
+function map_unmap() {
+	[ $# -eq 1 ] || exit 99
+	local image_name="$1"
+
+	local dev
+	dev="$(sudo rbd map "${image_name}")"
+	sudo rbd unmap "${dev}"
+}
+
+#### Start
+
+rbd create "${IMAGE_NAME}" --size="${IMAGE_SIZE}"
+
+COUNT=0
+START_TIME=$(get_time)
+END_TIME=$(expr $(get_time) + ${RUN_TIME})
+while ! times_up "${END_TIME}"; do
+	map_unmap "${IMAGE_NAME}"
+	COUNT=$(expr $COUNT + 1)
+done
+ELAPSED=$(expr "$(get_time)" - "${START_TIME}")
+
+rbd rm "${IMAGE_NAME}"
+
+echo "${COUNT} iterations completed in ${ELAPSED} seconds"
diff --git a/qa/workunits/rbd/merge_diff.sh b/qa/workunits/rbd/merge_diff.sh
new file mode 100755
index 00000000..eb859730
--- /dev/null
+++ b/qa/workunits/rbd/merge_diff.sh
@@ -0,0 +1,477 @@
+#!/usr/bin/env bash
+set -ex
+
+export RBD_FORCE_ALLOW_V1=1
+
+pool=rbd
+gen=$pool/gen
+out=$pool/out
+testno=1
+
+mkdir -p merge_diff_test
+pushd merge_diff_test
+
+function expect_false()
+{
+  if "$@"; then return 1; else return 0; fi
+}
+
+function clear_all()
+{
+  fusermount -u mnt || true
+
+  rbd snap purge --no-progress $gen || true
+  rbd rm --no-progress $gen || true
+  rbd snap purge --no-progress $out || true
+  rbd rm --no-progress $out || true
+
+  rm -rf diffs || true
+}
+
+function rebuild()
+{
+  clear_all
+  echo Starting test $testno
+  ((testno++))
+  if [[ "$2" -lt "$1" ]] && [[ "$3" -gt "1" ]]; then
+    rbd create $gen --size 100 --object-size $1 --stripe-unit $2 --stripe-count $3 --image-format $4
+  else
+    rbd create $gen --size 100 --object-size $1 --image-format $4
+  fi
+  rbd create $out --size 1 --object-size 524288
+  mkdir -p mnt diffs
+  # lttng has atexit handlers that need to be fork/clone aware
+  LD_PRELOAD=liblttng-ust-fork.so.0 rbd-fuse -p $pool mnt
+}
+
+function write()
+{
+  dd if=/dev/urandom of=mnt/gen bs=1M conv=notrunc seek=$1 count=$2
+}
+
+function snap()
+{
+  rbd snap create $gen@$1
+}
+
+function resize()
+{
+  rbd resize --no-progress $gen --size $1 --allow-shrink
+}
+
+function export_diff()
+{
+  if [ $2 == "head" ]; then
+    target="$gen"
+  else
+    target="$gen@$2"
+  fi
+  if [ $1 == "null" ]; then
+    rbd export-diff --no-progress $target diffs/$1.$2
+  else
+    rbd export-diff --no-progress $target --from-snap $1 diffs/$1.$2
+  fi
+}
+
+function merge_diff()
+{
+  rbd merge-diff diffs/$1.$2 diffs/$2.$3 diffs/$1.$3
+}
+
+function check()
+{
+  rbd import-diff --no-progress diffs/$1.$2 $out || return -1
+  if [ "$2" == "head" ]; then
+    sum1=`rbd export $gen - | md5sum`
+  else
+    sum1=`rbd export $gen@$2 - | md5sum`
+  fi
+  sum2=`rbd export $out - | md5sum`
+  if [ "$sum1" != "$sum2" ]; then
+    exit -1
+  fi
+  if [ "$2" != "head" ]; then
+    rbd snap ls $out | awk '{print $2}' | grep "^$2\$" || return -1
+  fi
+}
+
+#test f/t header
+rebuild 4194304 4194304 1 2
+write 0 1
+snap a
+write 1 1
+export_diff null a
+export_diff a head
+merge_diff null a head
+check null head
+
+rebuild 4194304 4194304 1 2
+write 0 1
+snap a
+write 1 1
+snap b
+write 2 1
+export_diff null a
+export_diff a b
+export_diff b head
+merge_diff null a b
+check null b
+
+rebuild 4194304 4194304 1 2
+write 0 1
+snap a
+write 1 1
+snap b
+write 2 1
+export_diff null a
+export_diff a b
+export_diff b head
+merge_diff a b head
+check null a
+check a head
+
+rebuild 4194304 4194304 1 2
+write 0 1
+snap a
+write 1 1
+snap b
+write 2 1
+export_diff null a
+export_diff a b
+export_diff b head
+rbd merge-diff diffs/null.a diffs/a.b - | rbd merge-diff - diffs/b.head - > diffs/null.head
+check null head
+
+#data test
+rebuild 4194304 4194304 1 2
+write 4 2
+snap s101
+write 0 3
+write 8 2
+snap s102
+export_diff null s101
+export_diff s101 s102
+merge_diff null s101 s102
+check null s102
+
+rebuild 4194304 4194304 1 2
+write 0 3
+write 2 5
+write 8 2
+snap s201
+write 0 2
+write 6 3
+snap s202
+export_diff null s201
+export_diff s201 s202
+merge_diff null s201 s202
+check null s202
+
+rebuild 4194304 4194304 1 2
+write 0 4
+write 12 6
+snap s301
+write 0 6
+write 10 5
+write 16 4
+snap s302
+export_diff null s301
+export_diff s301 s302
+merge_diff null s301 s302
+check null s302
+
+rebuild 4194304 4194304 1 2
+write 0 12
+write 14 2
+write 18 2
+snap s401
+write 1 2
+write 5 6
+write 13 3
+write 18 2
+snap s402
+export_diff null s401
+export_diff s401 s402
+merge_diff null s401 s402
+check null s402
+
+rebuild 4194304 4194304 1 2
+write 2 4
+write 10 12
+write 27 6
+write 36 4
+snap s501
+write 0 24
+write 28 4
+write 36 4
+snap s502
+export_diff null s501
+export_diff s501 s502
+merge_diff null s501 s502
+check null s502
+
+rebuild 4194304 4194304 1 2
+write 0 8
+resize 5
+snap r1
+resize 20
+write 12 8
+snap r2
+resize 8
+write 4 4
+snap r3
+export_diff null r1
+export_diff r1 r2
+export_diff r2 r3
+merge_diff null r1 r2
+merge_diff null r2 r3
+check null r3
+
+rebuild 4194304 4194304 1 2
+write 0 8
+resize 5
+snap r1
+resize 20
+write 12 8
+snap r2
+resize 8
+write 4 4
+snap r3
+resize 10
+snap r4
+export_diff null r1
+export_diff r1 r2
+export_diff r2 r3
+export_diff r3 r4
+merge_diff null r1 r2
+merge_diff null r2 r3
+merge_diff null r3 r4
+check null r4
+
+# merge diff doesn't yet support fancy striping
+# rebuild 4194304 65536 8 2
+# write 0 32
+# snap r1
+# write 16 32
+# snap r2
+# export_diff null r1
+# export_diff r1 r2
+# expect_false merge_diff null r1 r2
+
+rebuild 4194304 4194304 1 2
+write 0 1
+write 2 1
+write 4 1
+write 6 1
+snap s1
+write 1 1
+write 3 1
+write 5 1
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 1 1
+write 3 1
+write 5 1
+snap s1
+write 0 1
+write 2 1
+write 4 1
+write 6 1
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 0 3
+write 6 3
+write 12 3
+snap s1
+write 1 1
+write 7 1
+write 13 1
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 0 3
+write 6 3
+write 12 3
+snap s1
+write 0 1
+write 6 1
+write 12 1
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 0 3
+write 6 3
+write 12 3
+snap s1
+write 2 1
+write 8 1
+write 14 1
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 1 1
+write 7 1
+write 13 1
+snap s1
+write 0 3
+write 6 3
+write 12 3
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 0 1
+write 6 1
+write 12 1
+snap s1
+write 0 3
+write 6 3
+write 12 3
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 2 1
+write 8 1
+write 14 1
+snap s1
+write 0 3
+write 6 3
+write 12 3
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 0 3
+write 6 3
+write 12 3
+snap s1
+write 0 3
+write 6 3
+write 12 3
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 2 4
+write 8 4
+write 14 4
+snap s1
+write 0 3
+write 6 3
+write 12 3
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 0 4
+write 6 4
+write 12 4
+snap s1
+write 0 3
+write 6 3
+write 12 3
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 0 6
+write 6 6
+write 12 6
+snap s1
+write 0 3
+write 6 3
+write 12 3
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 3 6
+write 9 6
+write 15 6
+snap s1
+write 0 3
+write 6 3
+write 12 3
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 0 8
+snap s1
+resize 2
+resize 100
+snap s2
+export_diff null s1
+export_diff s1 s2
+merge_diff null s1 s2
+check null s2
+
+rebuild 4194304 4194304 1 2
+write 0 8
+snap s1
+resize 2
+resize 100
+snap s2
+write 20 2
+snap s3
+export_diff null s1
+export_diff s1 s2
+export_diff s2 s3
+merge_diff s1 s2 s3
+check null s1
+check s1 s3
+
+#addme
+
+clear_all
+popd
+rm -rf merge_diff_test
+
+echo OK
diff --git a/qa/workunits/rbd/notify_master.sh b/qa/workunits/rbd/notify_master.sh
new file mode 100755
index 00000000..6ebea31e
--- /dev/null
+++ b/qa/workunits/rbd/notify_master.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -ex
+
+relpath=$(dirname $0)/../../../src/test/librbd
+python $relpath/test_notify.py master
+exit 0
diff --git a/qa/workunits/rbd/notify_slave.sh b/qa/workunits/rbd/notify_slave.sh
new file mode 100755
index 00000000..ea66161a
--- /dev/null
+++ b/qa/workunits/rbd/notify_slave.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -ex
+
+relpath=$(dirname $0)/../../../src/test/librbd
+python $relpath/test_notify.py slave
+exit 0
diff --git a/qa/workunits/rbd/permissions.sh b/qa/workunits/rbd/permissions.sh
new file mode 100755
index 00000000..cf836848
--- /dev/null
+++ b/qa/workunits/rbd/permissions.sh
@@ -0,0 +1,272 @@
+#!/usr/bin/env bash
+set -ex
+
+IMAGE_FEATURES="layering,exclusive-lock,object-map,fast-diff"
+
+clone_v2_enabled() {
+    image_spec=$1
+    rbd info $image_spec | grep "clone-parent"
+}
+
+create_pools() {
+    ceph osd pool create images 32
+    rbd pool init images
+    ceph osd pool create volumes 32
+    rbd pool init volumes
+}
+
+delete_pools() {
+    (ceph osd pool delete images images --yes-i-really-really-mean-it || true) >/dev/null 2>&1
+    (ceph osd pool delete volumes volumes --yes-i-really-really-mean-it || true) >/dev/null 2>&1
+
+}
+
+recreate_pools() {
+    delete_pools
+    create_pools
+}
+
+delete_users() {
+    (ceph auth del client.volumes || true) >/dev/null 2>&1
+    (ceph auth del client.images || true) >/dev/null 2>&1
+
+    (ceph auth del client.snap_none || true) >/dev/null 2>&1
+    (ceph auth del client.snap_all || true) >/dev/null 2>&1
+    (ceph auth del client.snap_pool || true) >/dev/null 2>&1
+    (ceph auth del client.snap_profile_all || true) >/dev/null 2>&1
+    (ceph auth del client.snap_profile_pool || true) >/dev/null 2>&1
+
+    (ceph auth del client.mon_write || true) >/dev/null 2>&1
+}
+
+create_users() {
+    ceph auth get-or-create client.volumes \
+	mon 'profile rbd' \
+	osd 'profile rbd pool=volumes, profile rbd-read-only pool=images' \
+	mgr 'profile rbd pool=volumes, profile rbd-read-only pool=images' >> $KEYRING
+    ceph auth get-or-create client.images mon 'profile rbd' osd 'profile rbd pool=images' >> $KEYRING
+
+    ceph auth get-or-create client.snap_none mon 'allow r' >> $KEYRING
+    ceph auth get-or-create client.snap_all mon 'allow r' osd 'allow w' >> $KEYRING
+    ceph auth get-or-create client.snap_pool mon 'allow r' osd 'allow w pool=images' >> $KEYRING
+    ceph auth get-or-create client.snap_profile_all mon 'allow r' osd 'profile rbd' >> $KEYRING
+    ceph auth get-or-create client.snap_profile_pool mon 'allow r' osd 'profile rbd pool=images' >> $KEYRING
+
+    ceph auth get-or-create client.mon_write mon 'allow *' >> $KEYRING
+}
+
+expect() {
+
+  set +e
+
+  local expected_ret=$1
+  local ret
+
+  shift
+  cmd=$@
+
+  eval $cmd
+  ret=$?
+
+  set -e
+
+  if [[ $ret -ne $expected_ret ]]; then
+    echo "ERROR: running \'$cmd\': expected $expected_ret got $ret"
+    return 1
+  fi
+
+  return 0
+}
+
+test_images_access() {
+    rbd -k $KEYRING --id images create --image-format 2 --image-feature $IMAGE_FEATURES -s 1 images/foo
+    rbd -k $KEYRING --id images snap create images/foo@snap
+    rbd -k $KEYRING --id images snap protect images/foo@snap
+    rbd -k $KEYRING --id images snap unprotect images/foo@snap
+    rbd -k $KEYRING --id images snap protect images/foo@snap
+    rbd -k $KEYRING --id images export images/foo@snap - >/dev/null
+    expect 16 rbd -k $KEYRING --id images snap rm images/foo@snap
+
+    rbd -k $KEYRING --id volumes clone --image-feature $IMAGE_FEATURES images/foo@snap volumes/child
+
+    if ! clone_v2_enabled images/foo; then
+        expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
+    fi
+
+    expect 1 rbd -k $KEYRING --id volumes snap unprotect images/foo@snap
+    expect 1 rbd -k $KEYRING --id images flatten volumes/child
+    rbd -k $KEYRING --id volumes flatten volumes/child
+    expect 1 rbd -k $KEYRING --id volumes snap unprotect images/foo@snap
+    rbd -k $KEYRING --id images snap unprotect images/foo@snap
+
+    expect 39 rbd -k $KEYRING --id images rm images/foo
+    rbd -k $KEYRING --id images snap rm images/foo@snap
+    rbd -k $KEYRING --id images rm images/foo
+    rbd -k $KEYRING --id volumes rm volumes/child
+}
+
+test_volumes_access() {
+    rbd -k $KEYRING --id images create --image-format 2 --image-feature $IMAGE_FEATURES -s 1 images/foo
+    rbd -k $KEYRING --id images snap create images/foo@snap
+    rbd -k $KEYRING --id images snap protect images/foo@snap
+
+    # commands that work with read-only access
+    rbd -k $KEYRING --id volumes info images/foo@snap
+    rbd -k $KEYRING --id volumes snap ls images/foo
+    rbd -k $KEYRING --id volumes export images/foo - >/dev/null
+    rbd -k $KEYRING --id volumes cp images/foo volumes/foo_copy
+    rbd -k $KEYRING --id volumes rm volumes/foo_copy
+    rbd -k $KEYRING --id volumes children images/foo@snap
+    rbd -k $KEYRING --id volumes lock list images/foo
+
+    # commands that fail with read-only access
+    expect 1 rbd -k $KEYRING --id volumes resize -s 2 images/foo --allow-shrink
+    expect 1 rbd -k $KEYRING --id volumes snap create images/foo@2
+    expect 1 rbd -k $KEYRING --id volumes snap rollback images/foo@snap
+    expect 1 rbd -k $KEYRING --id volumes snap remove images/foo@snap
+    expect 1 rbd -k $KEYRING --id volumes snap purge images/foo
+    expect 1 rbd -k $KEYRING --id volumes snap unprotect images/foo@snap
+    expect 1 rbd -k $KEYRING --id volumes flatten images/foo
+    expect 1 rbd -k $KEYRING --id volumes lock add images/foo test
+    expect 1 rbd -k $KEYRING --id volumes lock remove images/foo test locker
+    expect 1 rbd -k $KEYRING --id volumes ls rbd
+
+    # create clone and snapshot
+    rbd -k $KEYRING --id volumes clone --image-feature $IMAGE_FEATURES images/foo@snap volumes/child
+    rbd -k $KEYRING --id volumes snap create volumes/child@snap1
+    rbd -k $KEYRING --id volumes snap protect volumes/child@snap1
+    rbd -k $KEYRING --id volumes snap create volumes/child@snap2
+
+    # make sure original snapshot stays protected
+    if clone_v2_enabled images/foo; then
+        rbd -k $KEYRING --id volumes flatten volumes/child
+        rbd -k $KEYRING --id volumes snap rm volumes/child@snap2
+        rbd -k $KEYRING --id volumes snap unprotect volumes/child@snap1
+    else
+        expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
+        rbd -k $KEYRING --id volumes flatten volumes/child
+        expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
+        rbd -k $KEYRING --id volumes snap rm volumes/child@snap2
+        expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
+        expect 2 rbd -k $KEYRING --id volumes snap rm volumes/child@snap2
+        rbd -k $KEYRING --id volumes snap unprotect volumes/child@snap1
+        expect 16 rbd -k $KEYRING --id images snap unprotect images/foo@snap
+    fi
+
+    # clean up
+    rbd -k $KEYRING --id volumes snap rm volumes/child@snap1
+    rbd -k $KEYRING --id images snap unprotect images/foo@snap
+    rbd -k $KEYRING --id images snap rm images/foo@snap
+    rbd -k $KEYRING --id images rm images/foo
+    rbd -k $KEYRING --id volumes rm volumes/child
+}
+
+create_self_managed_snapshot() {
+  ID=$1
+  POOL=$2
+
+  cat << EOF | CEPH_ARGS="-k $KEYRING" python
+import rados
+
+cluster = rados.Rados(conffile="", rados_id="${ID}")
+cluster.connect()
+ioctx = cluster.open_ioctx("${POOL}")
+
+snap_id = ioctx.create_self_managed_snap()
+print ("Created snap id {}".format(snap_id))
+EOF
+}
+
+remove_self_managed_snapshot() {
+  ID=$1
+  POOL=$2
+
+  cat << EOF | CEPH_ARGS="-k $KEYRING" python
+import rados
+
+cluster1 = rados.Rados(conffile="", rados_id="mon_write")
+cluster1.connect()
+ioctx1 = cluster1.open_ioctx("${POOL}")
+
+snap_id = ioctx1.create_self_managed_snap()
+print ("Created snap id {}".format(snap_id))
+
+cluster2 = rados.Rados(conffile="", rados_id="${ID}")
+cluster2.connect()
+ioctx2 = cluster2.open_ioctx("${POOL}")
+
+ioctx2.remove_self_managed_snap(snap_id)
+print ("Removed snap id {}".format(snap_id))
+EOF
+}
+
+test_remove_self_managed_snapshots() {
+    # Ensure users cannot create self-managed snapshots w/o permissions
+    expect 1 create_self_managed_snapshot snap_none images
+    expect 1 create_self_managed_snapshot snap_none volumes
+
+    create_self_managed_snapshot snap_all images
+    create_self_managed_snapshot snap_all volumes
+
+    create_self_managed_snapshot snap_pool images
+    expect 1 create_self_managed_snapshot snap_pool volumes
+
+    create_self_managed_snapshot snap_profile_all images
+    create_self_managed_snapshot snap_profile_all volumes
+
+    create_self_managed_snapshot snap_profile_pool images
+    expect 1 create_self_managed_snapshot snap_profile_pool volumes
+
+    # Ensure users cannot delete self-managed snapshots w/o permissions
+    expect 1 remove_self_managed_snapshot snap_none images
+    expect 1 remove_self_managed_snapshot snap_none volumes
+
+    remove_self_managed_snapshot snap_all images
+    remove_self_managed_snapshot snap_all volumes
+
+    remove_self_managed_snapshot snap_pool images
+    expect 1 remove_self_managed_snapshot snap_pool volumes
+
+    remove_self_managed_snapshot snap_profile_all images
+    remove_self_managed_snapshot snap_profile_all volumes
+
+    remove_self_managed_snapshot snap_profile_pool images
+    expect 1 remove_self_managed_snapshot snap_profile_pool volumes
+}
+
+test_rbd_support() {
+    # read-only commands should work on both pools
+    ceph -k $KEYRING --id volumes rbd perf image stats volumes
+    ceph -k $KEYRING --id volumes rbd perf image stats images
+
+    # read/write commands should only work on 'volumes'
+    rbd -k $KEYRING --id volumes create --image-format 2 --image-feature $IMAGE_FEATURES -s 1 volumes/foo
+    ceph -k $KEYRING --id volumes rbd task add remove volumes/foo
+    expect 13 ceph -k $KEYRING --id volumes rbd task add remove images/foo
+}
+
+cleanup() {
+    rm -f $KEYRING
+}
+
+KEYRING=$(mktemp)
+trap cleanup EXIT ERR HUP INT QUIT
+
+delete_users
+create_users
+
+recreate_pools
+test_images_access
+
+recreate_pools
+test_volumes_access
+
+test_remove_self_managed_snapshots
+
+test_rbd_support
+
+delete_pools
+delete_users
+
+echo OK
+exit 0
diff --git a/qa/workunits/rbd/qemu-iotests.sh b/qa/workunits/rbd/qemu-iotests.sh
new file mode 100755
index 00000000..1cd9c4e0
--- /dev/null
+++ b/qa/workunits/rbd/qemu-iotests.sh
@@ -0,0 +1,53 @@
+#!/bin/sh -ex
+
+# Run qemu-iotests against rbd. These are block-level tests that go
+# through qemu but do not involve running a full vm. Note that these
+# require the admin ceph user, as there's no way to pass the ceph user
+# to qemu-iotests currently.
+
+testlist='001 002 003 004 005 008 009 010 011 021 025 032 033 055'
+
+git clone https://github.com/qemu/qemu.git
+cd qemu
+if lsb_release -da 2>&1 | grep -iq 'bionic'; then
+    # Bionic requires a matching test harness
+    git checkout v2.11.0
+elif lsb_release -da 2>&1 | grep -iq 'xenial'; then
+    # Xenial requires a recent test harness
+    git checkout v2.3.0
+else
+    # use v2.2.0-rc3 (last released version that handles all the tests
+    git checkout 2528043f1f299e0e88cb026f1ca7c40bbb4e1f80
+fi
+
+cd tests/qemu-iotests
+mkdir bin
+# qemu-iotests expects a binary called just 'qemu' to be available
+if [ -x '/usr/bin/qemu-system-x86_64' ]
+then
+    QEMU='/usr/bin/qemu-system-x86_64'
+
+    # Bionic (v2.11.0) tests expect all tools in current directory
+    ln -s $QEMU qemu
+    ln -s /usr/bin/qemu-img
+    ln -s /usr/bin/qemu-io
+    ln -s /usr/bin/qemu-nbd
+else
+    QEMU='/usr/libexec/qemu-kvm'
+
+    # disable test 055 since qemu-kvm (RHEL/CentOS) doesn't support the
+    # required QMP commands
+    testlist=$(echo ${testlist} | sed "s/ 055//g")
+fi
+ln -s $QEMU bin/qemu
+
+# this is normally generated by configure, but has nothing but a python
+# binary definition, which we don't care about.  for some reason it is
+# not present on trusty.
+touch common.env
+
+# TEST_DIR is the pool for rbd
+TEST_DIR=rbd PATH="$PATH:$PWD/bin" ./check -rbd $testlist
+
+cd ../../..
+rm -rf qemu
diff --git a/qa/workunits/rbd/qemu_dynamic_features.sh b/qa/workunits/rbd/qemu_dynamic_features.sh
new file mode 100755
index 00000000..70e9fbb3
--- /dev/null
+++ b/qa/workunits/rbd/qemu_dynamic_features.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -x
+
+if [[ -z "${IMAGE_NAME}" ]]; then
+  echo image name must be provided
+  exit 1
+fi
+
+is_qemu_running() {
+  rbd status ${IMAGE_NAME} | grep -v "Watchers: none"
+}
+
+wait_for_qemu() {
+  while ! is_qemu_running ; do
+    echo "*** Waiting for QEMU"
+    sleep 30
+  done
+}
+
+wait_for_qemu
+rbd feature disable ${IMAGE_NAME} journaling
+rbd feature disable ${IMAGE_NAME} object-map
+rbd feature disable ${IMAGE_NAME} exclusive-lock
+
+while is_qemu_running ; do
+  echo "*** Enabling all features"
+  rbd feature enable ${IMAGE_NAME} exclusive-lock || break
+  rbd feature enable ${IMAGE_NAME} journaling || break
+  rbd feature enable ${IMAGE_NAME} object-map || break
+  if is_qemu_running ; then
+    sleep 60
+  fi
+
+  echo "*** Disabling all features"
+  rbd feature disable ${IMAGE_NAME} journaling || break
+  rbd feature disable ${IMAGE_NAME} object-map || break
+  rbd feature disable ${IMAGE_NAME} exclusive-lock || break
+  if is_qemu_running ; then
+    sleep 60
+  fi
+done
+
+if is_qemu_running ; then
+    echo "RBD command failed on alive QEMU"
+    exit 1
+fi
diff --git a/qa/workunits/rbd/qemu_rebuild_object_map.sh b/qa/workunits/rbd/qemu_rebuild_object_map.sh
new file mode 100755
index 00000000..2647dcdc
--- /dev/null
+++ b/qa/workunits/rbd/qemu_rebuild_object_map.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -ex
+
+if [[ -z "${IMAGE_NAME}" ]]; then
+  echo image name must be provided
+  exit 1
+fi
+
+is_qemu_running() {
+  rbd status ${IMAGE_NAME} | grep -v "Watchers: none"
+}
+
+wait_for_qemu() {
+  while ! is_qemu_running ; do
+    echo "*** Waiting for QEMU"
+    sleep 30
+  done
+}
+
+wait_for_qemu
+rbd feature disable ${IMAGE_NAME} journaling || true
+rbd feature disable ${IMAGE_NAME} fast-diff || true
+rbd feature disable ${IMAGE_NAME} object-map || true
+rbd feature disable ${IMAGE_NAME} exclusive-lock || true
+
+rbd feature enable ${IMAGE_NAME} exclusive-lock
+rbd feature enable ${IMAGE_NAME} object-map
+
+while is_qemu_running ; do
+  echo "*** Rebuilding object map"
+  rbd object-map rebuild ${IMAGE_NAME}
+
+  if is_qemu_running ; then
+    sleep 60
+  fi
+done
+
diff --git a/qa/workunits/rbd/rbd-ggate.sh b/qa/workunits/rbd/rbd-ggate.sh
new file mode 100755
index 00000000..9b60158d
--- /dev/null
+++ b/qa/workunits/rbd/rbd-ggate.sh
@@ -0,0 +1,242 @@
+#!/bin/sh -ex
+
+POOL=testrbdggate$$
+NS=ns
+IMAGE=test
+SIZE=64
+DATA=
+DEV=
+
+if which xmlstarlet > /dev/null 2>&1; then
+  XMLSTARLET=xmlstarlet
+elif which xml > /dev/null 2>&1; then
+  XMLSTARLET=xml
+else
+  echo "Missing xmlstarlet binary!"
+  exit 1
+fi
+
+if [ `uname -K` -ge 1200078 ] ; then
+    RBD_GGATE_RESIZE_SUPPORTED=1
+fi
+
+_sudo()
+{
+    local cmd
+
+    if [ `id -u` -eq 0 ]
+    then
+	"$@"
+	return $?
+    fi
+
+    # Look for the command in the user path. If it fails run it as is,
+    # supposing it is in sudo path.
+    cmd=`which $1 2>/dev/null` || cmd=$1
+    shift
+    sudo -nE "${cmd}" "$@"
+}
+
+check_geom_gate() 
+{
+    # See if geom_date is load, or can be loaded.
+    # Otherwise the tests can not run
+    if ! kldstat -q -n geom_gate ; then 
+        # See if we can load it
+        if ! _sudo kldload geom_gate ; then
+	    echo Not able to load geom_gate
+	    echo check /var/log/messages as to why 
+	    exit 1
+    	fi
+    fi
+}
+
+setup()
+{
+    local ns x
+
+    if [ -e CMakeCache.txt ]; then
+	# running under cmake build dir
+
+	CEPH_SRC=$(readlink -f $(dirname $0)/../../../src)
+	CEPH_ROOT=${PWD}
+	CEPH_BIN=${CEPH_ROOT}/bin
+
+	export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH}
+	export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind
+	for x in ${CEPH_ROOT}/lib/cython_modules/lib* ; do
+            PYTHONPATH="${PYTHONPATH}:${x}"
+	done
+	PATH=${CEPH_BIN}:${PATH}
+    fi
+
+    _sudo echo test sudo
+    check_geom_gate
+
+    trap cleanup INT TERM EXIT
+    TEMPDIR=`mktemp -d`
+    DATA=${TEMPDIR}/data
+    dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE}
+    ceph osd pool create ${POOL} 32
+
+    rbd namespace create ${POOL}/${NS}
+    for ns in '' ${NS}; do
+        rbd --dest-pool ${POOL} --dest-namespace "${ns}" --no-progress import \
+            ${DATA} ${IMAGE}
+    done
+}
+
+cleanup()
+{
+    local ns s
+
+    set +e
+    rm -Rf ${TEMPDIR}
+    if [ -n "${DEV}" ]
+    then
+	_sudo rbd-ggate unmap ${DEV}
+    fi
+
+    ceph osd pool delete ${POOL} ${POOL} --yes-i-really-really-mean-it
+}
+
+expect_false()
+{
+  if "$@"; then return 1; else return 0; fi
+}
+
+#
+# main
+#
+
+setup
+
+echo  exit status test
+expect_false rbd-ggate
+expect_false rbd-ggate INVALIDCMD
+if [ `id -u` -ne 0 ]
+then
+    expect_false rbd-ggate map ${IMAGE}
+fi
+expect_false _sudo rbd-ggate map INVALIDIMAGE
+
+echo  map test using the first unused device
+DEV=`_sudo rbd-ggate map ${POOL}/${IMAGE}`
+rbd-ggate list | grep " ${DEV} *$"
+
+echo  map test specifying the device
+expect_false _sudo rbd-ggate --device ${DEV} map ${POOL}/${IMAGE}
+dev1=${DEV}
+_sudo rbd-ggate unmap ${DEV}
+rbd-ggate list | expect_false grep " ${DEV} *$"
+DEV=
+# XXX: race possible when the device is reused by other process
+DEV=`_sudo rbd-ggate --device ${dev1} map ${POOL}/${IMAGE}`
+[ "${DEV}" = "${dev1}" ]
+rbd-ggate list | grep " ${DEV} *$"
+
+echo  list format test
+expect_false _sudo rbd-ggate --format INVALID list
+rbd-ggate --format json --pretty-format list
+rbd-ggate --format xml list
+
+echo  read test
+[ "`dd if=${DATA} bs=1M | md5`" = "`_sudo dd if=${DEV} bs=1M | md5`" ]
+
+echo  write test
+dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE}
+_sudo dd if=${DATA} of=${DEV} bs=1M
+_sudo sync
+[ "`dd if=${DATA} bs=1M | md5`" = "`rbd -p ${POOL} --no-progress export ${IMAGE} - | md5`" ]
+
+echo  trim test
+provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} |
+  $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .`
+used=`rbd -p ${POOL} --format xml du ${IMAGE} |
+  $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .`
+[ "${used}" -eq "${provisioned}" ]
+_sudo newfs -E ${DEV}
+_sudo sync
+provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} |
+  $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .`
+used=`rbd -p ${POOL} --format xml du ${IMAGE} |
+  $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .`
+[ "${used}" -lt "${provisioned}" ]
+
+echo  resize test
+devname=$(basename ${DEV})
+size=$(geom gate list ${devname} | awk '$1 ~ /Mediasize:/ {print $2}')
+test -n "${size}"
+rbd resize ${POOL}/${IMAGE} --size $((SIZE * 2))M
+rbd info ${POOL}/${IMAGE}
+if [ -z "$RBD_GGATE_RESIZE_SUPPORTED" ]; then
+    # when resizing is not supported:
+    # resizing the underlying image for a GEOM ggate will stop the
+    # ggate process servicing the device. So we can resize and test 
+    # the disappearance of the device
+    rbd-ggate list | expect_false grep " ${DEV} *$" 
+else
+    rbd-ggate list | grep " ${DEV} *$"
+    size2=$(geom gate list ${devname} | awk '$1 ~ /Mediasize:/ {print $2}')
+    test -n "${size2}"
+    test ${size2} -eq $((size * 2))
+    dd if=/dev/urandom of=${DATA} bs=1M count=$((SIZE * 2))
+    _sudo dd if=${DATA} of=${DEV} bs=1M
+    _sudo sync
+    [ "`dd if=${DATA} bs=1M | md5`" = "`rbd -p ${POOL} --no-progress export ${IMAGE} - | md5`" ]
+    rbd resize ${POOL}/${IMAGE} --allow-shrink --size ${SIZE}M
+    rbd info ${POOL}/${IMAGE}
+    size2=$(geom gate list ${devname} | awk '$1 ~ /Mediasize:/ {print $2}')
+    test -n "${size2}"
+    test ${size2} -eq ${size}
+    truncate -s ${SIZE}M ${DATA}
+    [ "`dd if=${DATA} bs=1M | md5`" = "`rbd -p ${POOL} --no-progress export ${IMAGE} - | md5`" ]
+    _sudo rbd-ggate unmap ${DEV}
+fi
+DEV=
+
+echo  read-only option test
+DEV=`_sudo rbd-ggate map --read-only ${POOL}/${IMAGE}`
+devname=$(basename ${DEV})
+rbd-ggate list | grep " ${DEV} *$"
+access=$(geom gate list ${devname} | awk '$1 == "access:" {print $2}')
+test "${access}" = "read-only"
+_sudo dd if=${DEV} of=/dev/null bs=1M
+expect_false _sudo dd if=${DATA} of=${DEV} bs=1M
+_sudo rbd-ggate unmap ${DEV}
+
+echo  exclusive option test
+DEV=`_sudo rbd-ggate map --exclusive ${POOL}/${IMAGE}`
+rbd-ggate list | grep " ${DEV} *$"
+_sudo dd if=${DATA} of=${DEV} bs=1M
+_sudo sync
+expect_false timeout 10 \
+    rbd -p ${POOL} bench ${IMAGE} --io-type=write --io-size=1024 --io-total=1024
+_sudo rbd-ggate unmap ${DEV}
+DEV=
+rbd bench -p ${POOL} ${IMAGE} --io-type=write --io-size=1024 --io-total=1024
+
+echo  unmap by image name test
+DEV=`_sudo rbd-ggate map ${POOL}/${IMAGE}`
+rbd-ggate list | grep " ${DEV} *$"
+_sudo rbd-ggate unmap "${POOL}/${IMAGE}"
+rbd-ggate list | expect_false grep " ${DEV} *$"
+DEV=
+
+echo  map/unmap snap test
+rbd snap create ${POOL}/${IMAGE}@snap
+DEV=`_sudo rbd-ggate map ${POOL}/${IMAGE}@snap`
+rbd-ggate list | grep " ${DEV} *$"
+_sudo rbd-ggate unmap "${POOL}/${IMAGE}@snap"
+rbd-ggate list | expect_false grep " ${DEV} *$"
+DEV=
+
+echo map/unmap namespace test
+rbd snap create ${POOL}/${NS}/${IMAGE}@snap
+DEV=`_sudo rbd-ggate map ${POOL}/${NS}/${IMAGE}@snap`
+rbd-ggate list | grep " ${DEV} *$"
+_sudo rbd-ggate unmap "${POOL}/${NS}/${IMAGE}@snap"
+rbd-ggate list | expect_false grep "${DEV} $"
+DEV=
+
+echo OK
diff --git a/qa/workunits/rbd/rbd-nbd.sh b/qa/workunits/rbd/rbd-nbd.sh
new file mode 100755
index 00000000..63728e06
--- /dev/null
+++ b/qa/workunits/rbd/rbd-nbd.sh
@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+set -ex
+
+. $(dirname $0)/../../standalone/ceph-helpers.sh
+
+POOL=rbd
+NS=ns
+IMAGE=testrbdnbd$$
+SIZE=64
+DATA=
+DEV=
+
+_sudo()
+{
+    local cmd
+
+    if [ `id -u` -eq 0 ]
+    then
+	"$@"
+	return $?
+    fi
+
+    # Look for the command in the user path. If it fails run it as is,
+    # supposing it is in sudo path.
+    cmd=`which $1 2>/dev/null` || cmd=$1
+    shift
+    sudo -nE "${cmd}" "$@"
+}
+
+setup()
+{
+    local ns x
+
+    if [ -e CMakeCache.txt ]; then
+	# running under cmake build dir
+
+	CEPH_SRC=$(readlink -f $(dirname $0)/../../../src)
+	CEPH_ROOT=${PWD}
+	CEPH_BIN=${CEPH_ROOT}/bin
+
+	export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH}
+	export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind
+	for x in ${CEPH_ROOT}/lib/cython_modules/lib* ; do
+            PYTHONPATH="${PYTHONPATH}:${x}"
+	done
+	PATH=${CEPH_BIN}:${PATH}
+    fi
+
+    _sudo echo test sudo
+
+    trap cleanup INT TERM EXIT
+    TEMPDIR=`mktemp -d`
+    DATA=${TEMPDIR}/data
+    dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE}
+
+    rbd namespace create ${POOL}/${NS}
+
+    for ns in '' ${NS}; do
+        rbd --dest-pool ${POOL} --dest-namespace "${ns}" --no-progress import \
+            ${DATA} ${IMAGE}
+    done
+}
+
+function cleanup()
+{
+    local ns s
+
+    set +e
+    rm -Rf ${TEMPDIR}
+    if [ -n "${DEV}" ]
+    then
+	_sudo rbd-nbd unmap ${DEV}
+    fi
+
+    for ns in '' ${NS}; do
+        if rbd -p ${POOL} --namespace "${ns}" status ${IMAGE} 2>/dev/null; then
+	    for s in 0.5 1 2 4 8 16 32; do
+	        sleep $s
+	        rbd -p ${POOL} --namespace "${ns}" status ${IMAGE} |
+                    grep 'Watchers: none' && break
+	    done
+	    rbd -p ${POOL} --namespace "${ns}" snap purge ${IMAGE}
+	    rbd -p ${POOL} --namespace "${ns}" remove ${IMAGE}
+        fi
+    done
+    rbd namespace remove ${POOL}/${NS}
+}
+
+function expect_false()
+{
+  if "$@"; then return 1; else return 0; fi
+}
+
+function get_pid()
+{
+    local ns=$1
+
+    PID=$(rbd-nbd --format xml list-mapped | $XMLSTARLET sel -t -v \
+      "//devices/device[pool='${POOL}'][namespace='${ns}'][image='${IMAGE}'][device='${DEV}']/id")
+    test -n "${PID}"
+    ps -p ${PID} -C rbd-nbd
+}
+
+unmap_device()
+{
+    local dev=$1
+    local pid=$2
+    _sudo rbd-nbd unmap ${dev}
+
+    for s in 0.5 1 2 4 8 16 32; do
+	sleep ${s}
+        rbd-nbd list-mapped | expect_false grep "^${pid}\\b" &&
+            ps -C rbd-nbd | expect_false grep "^${pid}\\b" &&
+            return 0
+    done
+    return 1
+}
+
+#
+# main
+#
+
+setup
+
+# exit status test
+expect_false rbd-nbd
+expect_false rbd-nbd INVALIDCMD
+if [ `id -u` -ne 0 ]
+then
+    expect_false rbd-nbd map ${IMAGE}
+fi
+expect_false _sudo rbd-nbd map INVALIDIMAGE
+expect_false _sudo rbd-nbd --device INVALIDDEV map ${IMAGE}
+
+# list format test
+expect_false rbd-nbd --format INVALID list-mapped
+rbd-nbd --format json --pretty-format list-mapped
+rbd-nbd --format xml list-mapped
+
+# map test using the first unused device
+DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}`
+get_pid
+# map test specifying the device
+expect_false _sudo rbd-nbd --device ${DEV} map ${POOL}/${IMAGE}
+dev1=${DEV}
+unmap_device ${DEV} ${PID}
+DEV=
+# XXX: race possible when the device is reused by other process
+DEV=`_sudo rbd-nbd --device ${dev1} map ${POOL}/${IMAGE}`
+[ "${DEV}" = "${dev1}" ]
+rbd-nbd list-mapped | grep "${IMAGE}"
+get_pid
+
+# read test
+[ "`dd if=${DATA} bs=1M | md5sum`" = "`_sudo dd if=${DEV} bs=1M | md5sum`" ]
+
+# write test
+dd if=/dev/urandom of=${DATA} bs=1M count=${SIZE}
+_sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct
+[ "`dd if=${DATA} bs=1M | md5sum`" = "`rbd -p ${POOL} --no-progress export ${IMAGE} - | md5sum`" ]
+
+# trim test
+provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} |
+  $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .`
+used=`rbd -p ${POOL} --format xml du ${IMAGE} |
+  $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .`
+[ "${used}" -eq "${provisioned}" ]
+_sudo mkfs.ext4 -E discard ${DEV} # better idea?
+sync
+provisioned=`rbd -p ${POOL} --format xml du ${IMAGE} |
+  $XMLSTARLET sel -t -m "//stats/images/image/provisioned_size" -v .`
+used=`rbd -p ${POOL} --format xml du ${IMAGE} |
+  $XMLSTARLET sel -t -m "//stats/images/image/used_size" -v .`
+[ "${used}" -lt "${provisioned}" ]
+
+# resize test
+devname=$(basename ${DEV})
+blocks=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions)
+test -n "${blocks}"
+rbd resize ${POOL}/${IMAGE} --size $((SIZE * 2))M
+rbd info ${POOL}/${IMAGE}
+blocks2=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions)
+test -n "${blocks2}"
+test ${blocks2} -eq $((blocks * 2))
+rbd resize ${POOL}/${IMAGE} --allow-shrink --size ${SIZE}M
+blocks2=$(awk -v dev=${devname} '$4 == dev {print $3}' /proc/partitions)
+test -n "${blocks2}"
+test ${blocks2} -eq ${blocks}
+
+# read-only option test
+unmap_device ${DEV} ${PID}
+DEV=`_sudo rbd-nbd map --read-only ${POOL}/${IMAGE}`
+PID=$(rbd-nbd list-mapped | awk -v pool=${POOL} -v img=${IMAGE} -v dev=${DEV} \
+    '$2 == pool && $3 == img && $5 == dev {print $1}')
+test -n "${PID}"
+ps -p ${PID} -C rbd-nbd
+
+_sudo dd if=${DEV} of=/dev/null bs=1M
+expect_false _sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct
+unmap_device ${DEV} ${PID}
+
+# exclusive option test
+DEV=`_sudo rbd-nbd map --exclusive ${POOL}/${IMAGE}`
+get_pid
+
+_sudo dd if=${DATA} of=${DEV} bs=1M oflag=direct
+expect_false timeout 10 \
+	rbd bench ${IMAGE} --io-type write --io-size=1024 --io-total=1024
+unmap_device ${DEV} ${PID}
+DEV=
+rbd bench ${IMAGE} --io-type write --io-size=1024 --io-total=1024
+
+# unmap by image name test
+DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}`
+get_pid
+unmap_device ${IMAGE} ${PID}
+DEV=
+
+# map/unmap snap test
+rbd snap create ${POOL}/${IMAGE}@snap
+DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}@snap`
+get_pid
+unmap_device "${IMAGE}@snap" ${PID}
+DEV=
+
+# map/unmap namespace test
+rbd snap create ${POOL}/${NS}/${IMAGE}@snap
+DEV=`_sudo rbd-nbd map ${POOL}/${NS}/${IMAGE}@snap`
+get_pid ${NS}
+unmap_device "${POOL}/${NS}/${IMAGE}@snap" ${PID}
+DEV=
+
+# unmap by image name test 2
+DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}`
+get_pid
+pid=$PID
+DEV=`_sudo rbd-nbd map ${POOL}/${NS}/${IMAGE}`
+get_pid ${NS}
+unmap_device ${POOL}/${NS}/${IMAGE} ${PID}
+DEV=
+unmap_device ${POOL}/${IMAGE} ${pid}
+
+# auto unmap test
+DEV=`_sudo rbd-nbd map ${POOL}/${IMAGE}`
+get_pid
+_sudo kill ${PID}
+for i in `seq 10`; do
+  rbd-nbd list-mapped | expect_false grep "^${PID} *${POOL} *${IMAGE}" && break
+  sleep 1
+done
+rbd-nbd list-mapped | expect_false grep "^${PID} *${POOL} *${IMAGE}"
+
+echo OK
diff --git a/qa/workunits/rbd/rbd_groups.sh b/qa/workunits/rbd/rbd_groups.sh
new file mode 100755
index 00000000..2c9fd318
--- /dev/null
+++ b/qa/workunits/rbd/rbd_groups.sh
@@ -0,0 +1,209 @@
+#!/bin/sh -ex
+
+#
+# rbd_consistency_groups.sh - test consistency groups cli commands
+#
+
+#
+# Functions
+#
+
+create_group()
+{
+    local group_name=$1
+
+    rbd group create $group_name
+}
+
+list_groups()
+{
+    rbd group list
+}
+
+check_group_exists()
+{
+    local group_name=$1
+    list_groups | grep $group_name
+}
+
+remove_group()
+{
+    local group_name=$1
+
+    rbd group remove $group_name
+}
+
+rename_group()
+{
+    local src_name=$1
+    local dest_name=$2
+
+    rbd group rename $src_name $dest_name
+}
+
+check_group_does_not_exist()
+{
+    local group_name=$1
+    for v in $(list_groups); do
+        if [ "$v" == "$group_name" ]; then
+            return 1
+        fi
+    done
+    return 0
+}
+
+create_image()
+{
+    local image_name=$1
+    rbd create --size 10M $image_name
+}
+
+remove_image()
+{
+    local image_name=$1
+    rbd remove $image_name
+}
+
+add_image_to_group()
+{
+    local image_name=$1
+    local group_name=$2
+    rbd group image add $group_name $image_name
+}
+
+remove_image_from_group()
+{
+    local image_name=$1
+    local group_name=$2
+    rbd group image remove $group_name $image_name
+}
+
+check_image_in_group()
+{
+    local image_name=$1
+    local group_name=$2
+    for v in $(rbd group image list $group_name); do
+        local vtrimmed=${v#*/}
+        if [ "$vtrimmed" = "$image_name" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+check_image_not_in_group()
+{
+    local image_name=$1
+    local group_name=$2
+    for v in $(rbd group image list $group_name); do
+        local vtrimmed=${v#*/}
+        if [ "$vtrimmed" = "$image_name" ]; then
+            return 1
+        fi
+    done
+    return 0
+}
+
+create_snapshot()
+{
+    local group_name=$1
+    local snap_name=$2
+    rbd group snap create $group_name@$snap_name
+}
+
+remove_snapshot()
+{
+    local group_name=$1
+    local snap_name=$2
+    rbd group snap remove $group_name@$snap_name
+}
+
+rename_snapshot()
+{
+    local group_name=$1
+    local snap_name=$2
+    local new_snap_name=$3
+    rbd group snap rename $group_name@$snap_name $new_snap_name
+}
+
+list_snapshots()
+{
+    local group_name=$1
+    rbd group snap list $group_name
+}
+
+rollback_snapshot()
+{
+    local group_name=$1
+    local snap_name=$2
+    rbd group snap rollback $group_name@$snap_name
+}
+
+check_snapshot_in_group()
+{
+    local group_name=$1
+    local snap_name=$2
+    list_snapshots $group_name | grep $snap_name
+}
+
+check_snapshot_not_in_group()
+{
+    local group_name=$1
+    local snap_name=$2
+    for v in $(list_snapshots $group_name | awk '{print $1}'); do
+        if [ "$v" = "$snap_name" ]; then
+            return 1
+        fi
+    done
+    return 0
+}
+
+echo "TEST: create remove consistency group"
+group="test_consistency_group"
+new_group="test_new_consistency_group"
+create_group $group
+check_group_exists $group
+rename_group $group $new_group
+check_group_exists $new_group
+remove_group $new_group
+check_group_does_not_exist $new_group
+echo "PASSED"
+
+echo "TEST: add remove images to consistency group"
+image="test_image"
+group="test_consistency_group"
+create_image $image
+create_group $group
+add_image_to_group $image $group
+check_image_in_group $image $group
+remove_image_from_group $image $group
+check_image_not_in_group $image $group
+remove_group $group
+remove_image $image
+echo "PASSED"
+
+echo "TEST: create remove snapshots of consistency group"
+image="test_image"
+group="test_consistency_group"
+snap="group_snap"
+new_snap="new_group_snap"
+sec_snap="group_snap2"
+create_image $image
+create_group $group
+add_image_to_group $image $group
+create_snapshot $group $snap
+check_snapshot_in_group $group $snap
+rename_snapshot $group $snap $new_snap
+check_snapshot_not_in_group $group $snap
+create_snapshot $group $sec_snap
+check_snapshot_in_group $group $sec_snap
+rollback_snapshot $group $new_snap
+remove_snapshot $group $new_snap
+check_snapshot_not_in_group $group $new_snap
+remove_snapshot $group $sec_snap
+check_snapshot_not_in_group $group $sec_snap
+remove_group $group
+remove_image $image
+echo "PASSED"
+
+echo "OK"
diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh
new file mode 100755
index 00000000..45e93e65
--- /dev/null
+++ b/qa/workunits/rbd/rbd_mirror.sh
@@ -0,0 +1,516 @@
+#!/bin/sh -ex
+#
+# rbd_mirror.sh - test rbd-mirror daemon
+#
+# The scripts starts two ("local" and "remote") clusters using mstart.sh script,
+# creates a temporary directory, used for cluster configs, daemon logs, admin
+# socket, temporary files, and launches rbd-mirror daemon.
+#
+
+. $(dirname $0)/rbd_mirror_helpers.sh
+
+setup
+
+testlog "TEST: add image and test replay"
+start_mirrors ${CLUSTER1}
+image=test
+create_image ${CLUSTER2} ${POOL} ${image}
+set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1"
+set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2"
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+write_image ${CLUSTER2} ${POOL} ${image} 100
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
+fi
+compare_images ${POOL} ${image}
+compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1"
+compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2"
+
+testlog "TEST: stop mirror, add image, start mirror and test replay"
+stop_mirrors ${CLUSTER1}
+image1=test1
+create_image ${CLUSTER2} ${POOL} ${image1}
+write_image ${CLUSTER2} ${POOL} ${image1} 100
+start_mirrors ${CLUSTER1}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
+fi
+compare_images ${POOL} ${image1}
+
+testlog "TEST: test the first image is replaying after restart"
+write_image ${CLUSTER2} ${POOL} ${image} 100
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+compare_images ${POOL} ${image}
+
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  testlog "TEST: stop/start/restart mirror via admin socket"
+  all_admin_daemons ${CLUSTER1} rbd mirror stop
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror start
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror restart
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror stop
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror restart
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
+
+  admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+
+  all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
+
+  all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
+
+  flush ${CLUSTER1}
+  all_admin_daemons ${CLUSTER1} rbd mirror status
+fi
+
+testlog "TEST: test image rename"
+new_name="${image}_RENAMED"
+rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
+admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
+admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
+rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+
+testlog "TEST: test trash move restore"
+image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image})
+trash_move ${CLUSTER2} ${POOL} ${image}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
+trash_restore ${CLUSTER2} ${POOL} ${image_id}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+
+testlog "TEST: failover and failback"
+start_mirrors ${CLUSTER2}
+
+# demote and promote same cluster
+demote_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+promote_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+write_image ${CLUSTER2} ${POOL} ${image} 100
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+compare_images ${POOL} ${image}
+
+# failover (unmodified)
+demote_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+promote_image ${CLUSTER1} ${POOL} ${image}
+wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
+
+# failback (unmodified)
+demote_image ${CLUSTER1} ${POOL} ${image}
+wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+promote_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+compare_images ${POOL} ${image}
+
+# failover
+demote_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+promote_image ${CLUSTER1} ${POOL} ${image}
+wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
+write_image ${CLUSTER1} ${POOL} ${image} 100
+wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position'
+compare_images ${POOL} ${image}
+
+# failback
+demote_image ${CLUSTER1} ${POOL} ${image}
+wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+promote_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+write_image ${CLUSTER2} ${POOL} ${image} 100
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+compare_images ${POOL} ${image}
+
+# force promote
+force_promote_image=test_force_promote
+create_image ${CLUSTER2} ${POOL} ${force_promote_image}
+write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
+wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
+promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
+wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
+wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
+wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
+write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
+write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
+
+testlog "TEST: cloned images"
+testlog " - default"
+parent_image=test_parent
+parent_snap=snap
+create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
+write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
+create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+
+clone_image=test_clone
+clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
+write_image ${CLUSTER2} ${POOL} ${clone_image} 100
+
+enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image}
+wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position'
+compare_images ${PARENT_POOL} ${parent_image}
+
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position'
+compare_images ${POOL} ${clone_image}
+
+testlog " - clone v1"
+clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
+
+clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
+            ${clone_image}_v1 --rbd-default-clone-format 1
+test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1
+test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1
+
+testlog " - clone v2"
+parent_snap=snap_v2
+create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
+            ${clone_image}_v2 --rbd-default-clone-format 2
+test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2
+test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2
+
+remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
+remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2
+wait_for_image_present ${CLUSTER1} ${POOL} ${clone_image}_v2 'deleted'
+test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
+
+testlog " - clone v2 non-primary"
+create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
+clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \
+            ${clone_image}_v2 --rbd-default-clone-format 2
+remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
+wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
+remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2
+wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap}
+
+testlog "TEST: data pool"
+dp_image=test_data_pool
+create_image ${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL}
+data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image})
+test "${data_pool}" = "${PARENT_POOL}"
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image}
+data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image})
+test "${data_pool}" = "${PARENT_POOL}"
+create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1'
+write_image ${CLUSTER2} ${POOL} ${dp_image} 100
+create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2'
+write_image ${CLUSTER2} ${POOL} ${dp_image} 100
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'master_position'
+compare_images ${POOL} ${dp_image}@snap1
+compare_images ${POOL} ${dp_image}@snap2
+compare_images ${POOL} ${dp_image}
+
+testlog "TEST: disable mirroring / delete non-primary image"
+image2=test2
+image3=test3
+image4=test4
+image5=test5
+for i in ${image2} ${image3} ${image4} ${image5}; do
+  create_image ${CLUSTER2} ${POOL} ${i}
+  write_image ${CLUSTER2} ${POOL} ${i} 100
+  create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
+  create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
+  if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
+    protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
+    protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
+  fi
+  write_image ${CLUSTER2} ${POOL} ${i} 100
+  wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
+  wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
+done
+
+set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
+for i in ${image2} ${image4}; do
+  disable_mirror ${CLUSTER2} ${POOL} ${i}
+done
+
+unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
+unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
+for i in ${image3} ${image5}; do
+  remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
+  remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
+  remove_image_retry ${CLUSTER2} ${POOL} ${i}
+done
+
+for i in ${image2} ${image3} ${image4} ${image5}; do
+  wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
+done
+
+set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
+for i in ${image2} ${image4}; do
+  wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
+  wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
+  compare_images ${POOL} ${i}
+done
+
+testlog "TEST: remove mirroring pool"
+pool=pool_to_remove
+for cluster in ${CLUSTER1} ${CLUSTER2}; do
+    CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${pool} 16 16
+    CEPH_ARGS='' rbd --cluster ${cluster} pool init ${pool}
+    rbd --cluster ${cluster} mirror pool enable ${pool} pool
+done
+rbd --cluster ${CLUSTER1} mirror pool peer add ${pool} ${CLUSTER2}
+rbd --cluster ${CLUSTER2} mirror pool peer add ${pool} ${CLUSTER1}
+rdp_image=test_remove_data_pool
+create_image ${CLUSTER2} ${pool} ${image} 128
+create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool}
+write_image ${CLUSTER2} ${pool} ${image} 100
+write_image ${CLUSTER2} ${POOL} ${rdp_image} 100
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${pool} ${image} 'up+replaying' 'master_position'
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'master_position'
+for cluster in ${CLUSTER1} ${CLUSTER2}; do
+    CEPH_ARGS='' ceph --cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it
+done
+remove_image_retry ${CLUSTER2} ${POOL} ${rdp_image}
+wait_for_image_present ${CLUSTER1} ${POOL} ${rdp_image} 'deleted'
+for i in 0 1 2 4 8 8 8 8 16 16; do
+    sleep $i
+    admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} || break
+done
+admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false
+
+testlog "TEST: snapshot rename"
+snap_name='snap_rename'
+create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
+for i in `seq 1 20`; do
+  rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
+done
+wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
+
+testlog "TEST: disable mirror while daemon is stopped"
+stop_mirrors ${CLUSTER1}
+stop_mirrors ${CLUSTER2}
+set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
+disable_mirror ${CLUSTER2} ${POOL} ${image}
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+fi
+start_mirrors ${CLUSTER1}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
+set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+
+testlog "TEST: simple image resync"
+request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+compare_images ${POOL} ${image}
+
+testlog "TEST: image resync while replayer is stopped"
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
+  admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+  admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+  compare_images ${POOL} ${image}
+fi
+
+testlog "TEST: request image resync while daemon is offline"
+stop_mirrors ${CLUSTER1}
+request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
+start_mirrors ${CLUSTER1}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+compare_images ${POOL} ${image}
+
+testlog "TEST: client disconnect"
+image=laggy
+create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
+write_image ${CLUSTER2} ${POOL} ${image} 10
+
+testlog " - replay stopped after disconnect"
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+disconnect_image ${CLUSTER2} ${POOL} ${image}
+test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
+
+testlog " - replay started after resync requested"
+request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+compare_images ${POOL} ${image}
+
+testlog " - disconnected after max_concurrent_object_sets reached"
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+  set_image_meta ${CLUSTER2} ${POOL} ${image} \
+	         conf_rbd_journal_max_concurrent_object_sets 1
+  write_image ${CLUSTER2} ${POOL} ${image} 20 16384
+  write_image ${CLUSTER2} ${POOL} ${image} 20 16384
+  test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+  set_image_meta ${CLUSTER2} ${POOL} ${image} \
+	         conf_rbd_journal_max_concurrent_object_sets 0
+
+  testlog " - replay is still stopped (disconnected) after restart"
+  admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
+fi
+
+testlog " - replay started after resync requested"
+request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+compare_images ${POOL} ${image}
+
+testlog " - rbd_mirroring_resync_after_disconnect config option"
+set_image_meta ${CLUSTER2} ${POOL} ${image} \
+	       conf_rbd_mirroring_resync_after_disconnect true
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image})
+disconnect_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+compare_images ${POOL} ${image}
+set_image_meta ${CLUSTER2} ${POOL} ${image} \
+	       conf_rbd_mirroring_resync_after_disconnect false
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+disconnect_image ${CLUSTER2} ${POOL} ${image}
+test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
+
+testlog "TEST: split-brain"
+image=split-brain
+create_image ${CLUSTER2} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+promote_image ${CLUSTER1} ${POOL} ${image} --force
+wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+write_image ${CLUSTER1} ${POOL} ${image} 10
+demote_image ${CLUSTER1} ${POOL} ${image}
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
+request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
+wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+
+if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+  # teuthology will trash the daemon
+  testlog "TEST: no blacklists"
+  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
+  CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
+fi
diff --git a/qa/workunits/rbd/rbd_mirror_bootstrap.sh b/qa/workunits/rbd/rbd_mirror_bootstrap.sh
new file mode 100755
index 00000000..e0a096ee
--- /dev/null
+++ b/qa/workunits/rbd/rbd_mirror_bootstrap.sh
@@ -0,0 +1,49 @@
+#!/bin/sh -ex
+#
+# rbd_mirror_bootstrap.sh - test peer bootstrap create/import
+#
+
+RBD_MIRROR_MANUAL_PEERS=1
+RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-1}
+. $(dirname $0)/rbd_mirror_helpers.sh
+
+setup
+
+testlog "TEST: bootstrap cluster2 from cluster1"
+# create token on cluster1 and import to cluster2
+TOKEN=${TEMPDIR}/peer-token
+TOKEN_2=${TEMPDIR}/peer-token-2
+CEPH_ARGS='' rbd --cluster ${CLUSTER1} mirror pool peer bootstrap create ${POOL} > ${TOKEN}
+CEPH_ARGS='' rbd --cluster ${CLUSTER1} mirror pool peer bootstrap create ${PARENT_POOL} > ${TOKEN_2}
+cmp ${TOKEN} ${TOKEN_2}
+
+CEPH_ARGS='' rbd --cluster ${CLUSTER2} --pool ${POOL} mirror pool peer bootstrap import ${TOKEN} --direction rx-only
+CEPH_ARGS='' rbd --cluster ${CLUSTER2} --pool ${PARENT_POOL} mirror pool peer bootstrap import ${TOKEN} --direction rx-tx
+
+start_mirrors ${CLUSTER1}
+start_mirrors ${CLUSTER2}
+
+testlog "TEST: verify rx-only direction"
+[ "$(rbd --cluster ${CLUSTER1} --pool ${POOL} mirror pool info --format xml |
+	${XMLSTARLET} sel -t -v  '//mirror/peers/peer[1]/uuid')" = "" ]
+
+create_image ${CLUSTER1} ${POOL} image1
+
+wait_for_image_replay_started ${CLUSTER2} ${POOL} image1
+write_image ${CLUSTER1} ${POOL} image1 100
+wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} image1
+
+testlog "TEST: verify rx-tx direction"
+create_image ${CLUSTER1} ${PARENT_POOL} image1
+create_image ${CLUSTER2} ${PARENT_POOL} image2
+
+enable_mirror ${CLUSTER1} ${PARENT_POOL} image1
+enable_mirror ${CLUSTER2} ${PARENT_POOL} image2
+
+wait_for_image_replay_started ${CLUSTER2} ${PARENT_POOL} image1
+write_image ${CLUSTER1} ${PARENT_POOL} image1 100
+wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${PARENT_POOL} image1
+
+wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} image2
+write_image ${CLUSTER2} ${PARENT_POOL} image2 100
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} image2
diff --git a/qa/workunits/rbd/rbd_mirror_fsx_compare.sh b/qa/workunits/rbd/rbd_mirror_fsx_compare.sh
new file mode 100755
index 00000000..0ba3c97d
--- /dev/null
+++ b/qa/workunits/rbd/rbd_mirror_fsx_compare.sh
@@ -0,0 +1,38 @@
+#!/bin/sh -ex
+#
+# rbd_mirror_fsx_compare.sh - test rbd-mirror daemon under FSX workload
+#
+# The script is used to compare FSX-generated images between two clusters.
+#
+
+. $(dirname $0)/rbd_mirror_helpers.sh
+
+trap 'cleanup $?' INT TERM EXIT
+
+setup_tempdir
+
+testlog "TEST: wait for all images"
+image_count=$(rbd --cluster ${CLUSTER1} --pool ${POOL} ls | wc -l)
+retrying_seconds=0
+sleep_seconds=10
+while [ ${retrying_seconds} -le 7200 ]; do
+    [ $(rbd --cluster ${CLUSTER2} --pool ${POOL} ls | wc -l) -ge ${image_count} ] && break
+    sleep ${sleep_seconds}
+    retrying_seconds=$(($retrying_seconds+${sleep_seconds}))
+done
+
+testlog "TEST: snapshot all pool images"
+snap_id=`uuidgen`
+for image in $(rbd --cluster ${CLUSTER1} --pool ${POOL} ls); do
+    create_snapshot ${CLUSTER1} ${POOL} ${image} ${snap_id}
+done
+
+testlog "TEST: wait for snapshots"
+for image in $(rbd --cluster ${CLUSTER1} --pool ${POOL} ls); do
+    wait_for_snap_present ${CLUSTER2} ${POOL} ${image} ${snap_id}
+done
+
+testlog "TEST: compare image snapshots"
+for image in $(rbd --cluster ${CLUSTER1} --pool ${POOL} ls); do
+    compare_image_snapshots ${POOL} ${image}
+done
diff --git a/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh b/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh
new file mode 100755
index 00000000..d988987b
--- /dev/null
+++ b/qa/workunits/rbd/rbd_mirror_fsx_prepare.sh
@@ -0,0 +1,10 @@
+#!/bin/sh -ex
+#
+# rbd_mirror_fsx_prepare.sh - test rbd-mirror daemon under FSX workload
+#
+# The script is used to compare FSX-generated images between two clusters.
+#
+
+. $(dirname $0)/rbd_mirror_helpers.sh
+
+setup
diff --git a/qa/workunits/rbd/rbd_mirror_ha.sh b/qa/workunits/rbd/rbd_mirror_ha.sh
new file mode 100755
index 00000000..171ab569
--- /dev/null
+++ b/qa/workunits/rbd/rbd_mirror_ha.sh
@@ -0,0 +1,210 @@
+#!/bin/sh -ex
+#
+# rbd_mirror_ha.sh - test rbd-mirror daemons in HA mode
+#
+
+RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-7}
+
+. $(dirname $0)/rbd_mirror_helpers.sh
+
+setup
+
+is_leader()
+{
+    local instance=$1
+    local pool=$2
+
+    test -n "${pool}" || pool=${POOL}
+
+    admin_daemon "${CLUSTER1}:${instance}" \
+		 rbd mirror status ${pool} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} |
+	grep '"leader": true'
+}
+
+wait_for_leader()
+{
+    local s instance
+
+    for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64; do
+	sleep $s
+	for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+	    is_leader ${instance} || continue
+	    LEADER=${instance}
+	    return 0
+	done
+    done
+
+    LEADER=
+    return 1
+}
+
+release_leader()
+{
+    local pool=$1
+    local cmd="rbd mirror leader release"
+
+    test -n "${pool}" && cmd="${cmd} ${pool} ${CLUSTER2}"
+
+    admin_daemon "${CLUSTER1}:${LEADER}" ${cmd}
+}
+
+wait_for_leader_released()
+{
+    local i
+
+    test -n "${LEADER}"
+    for i in `seq 10`; do
+	is_leader ${LEADER} || return 0
+	sleep 1
+    done
+
+    return 1
+}
+
+test_replay()
+{
+    local image
+
+    for image; do
+	wait_for_image_replay_started ${CLUSTER1}:${LEADER} ${POOL} ${image}
+	write_image ${CLUSTER2} ${POOL} ${image} 100
+	wait_for_replay_complete ${CLUSTER1}:${LEADER} ${CLUSTER2} ${POOL} \
+				 ${image}
+	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' \
+                                    'master_position' \
+                                    "${MIRROR_USER_ID_PREFIX}${LEADER} on $(hostname -s)"
+	if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
+	    wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} \
+					'down+unknown'
+	fi
+	compare_images ${POOL} ${image}
+    done
+}
+
+testlog "TEST: start first daemon instance and test replay"
+start_mirror ${CLUSTER1}:0
+image1=test1
+create_image ${CLUSTER2} ${POOL} ${image1}
+LEADER=0
+test_replay ${image1}
+
+testlog "TEST: release leader and wait it is reacquired"
+is_leader 0 ${POOL}
+is_leader 0 ${PARENT_POOL}
+release_leader ${POOL}
+wait_for_leader_released
+is_leader 0 ${PARENT_POOL}
+wait_for_leader
+release_leader
+wait_for_leader_released
+expect_failure "" is_leader 0 ${PARENT_POOL}
+wait_for_leader
+
+testlog "TEST: start second daemon instance and test replay"
+start_mirror ${CLUSTER1}:1
+image2=test2
+create_image ${CLUSTER2} ${POOL} ${image2}
+test_replay ${image1} ${image2}
+
+testlog "TEST: release leader and test it is acquired by secondary"
+is_leader 0 ${POOL}
+is_leader 0 ${PARENT_POOL}
+release_leader ${POOL}
+wait_for_leader_released
+wait_for_leader
+test_replay ${image1} ${image2}
+release_leader
+wait_for_leader_released
+wait_for_leader
+test "${LEADER}" = 0
+
+testlog "TEST: stop first daemon instance and test replay"
+stop_mirror ${CLUSTER1}:0
+image3=test3
+create_image ${CLUSTER2} ${POOL} ${image3}
+LEADER=1
+test_replay ${image1} ${image2} ${image3}
+
+testlog "TEST: start first daemon instance and test replay"
+start_mirror ${CLUSTER1}:0
+image4=test4
+create_image ${CLUSTER2} ${POOL} ${image4}
+test_replay ${image3} ${image4}
+
+testlog "TEST: crash leader and test replay"
+stop_mirror ${CLUSTER1}:1 -KILL
+image5=test5
+create_image ${CLUSTER2} ${POOL} ${image5}
+LEADER=0
+test_replay ${image1} ${image4} ${image5}
+
+testlog "TEST: start crashed leader and test replay"
+start_mirror ${CLUSTER1}:1
+image6=test6
+create_image ${CLUSTER2} ${POOL} ${image6}
+test_replay ${image1} ${image6}
+
+testlog "TEST: start yet another daemon instance and test replay"
+start_mirror ${CLUSTER1}:2
+image7=test7
+create_image ${CLUSTER2} ${POOL} ${image7}
+test_replay ${image1} ${image7}
+
+testlog "TEST: release leader and test it is acquired by secondary"
+is_leader 0
+release_leader
+wait_for_leader_released
+wait_for_leader
+test_replay ${image1} ${image2}
+
+testlog "TEST: stop leader and test replay"
+stop_mirror ${CLUSTER1}:${LEADER}
+image8=test8
+create_image ${CLUSTER2} ${POOL} ${image8}
+prev_leader=${LEADER}
+wait_for_leader
+test_replay ${image1} ${image8}
+
+testlog "TEST: start previous leader and test replay"
+start_mirror ${CLUSTER1}:${prev_leader}
+image9=test9
+create_image ${CLUSTER2} ${POOL} ${image9}
+test_replay ${image1} ${image9}
+
+testlog "TEST: crash leader and test replay"
+stop_mirror ${CLUSTER1}:${LEADER} -KILL
+image10=test10
+create_image ${CLUSTER2} ${POOL} ${image10}
+prev_leader=${LEADER}
+wait_for_leader
+test_replay ${image1} ${image10}
+
+testlog "TEST: start previous leader and test replay"
+start_mirror ${CLUSTER1}:${prev_leader}
+image11=test11
+create_image ${CLUSTER2} ${POOL} ${image11}
+test_replay ${image1} ${image11}
+
+testlog "TEST: start some more daemon instances and test replay"
+start_mirror ${CLUSTER1}:3
+start_mirror ${CLUSTER1}:4
+start_mirror ${CLUSTER1}:5
+start_mirror ${CLUSTER1}:6
+image13=test13
+create_image ${CLUSTER2} ${POOL} ${image13}
+test_replay ${leader} ${image1} ${image13}
+
+testlog "TEST: release leader and test it is acquired by secondary"
+release_leader
+wait_for_leader_released
+wait_for_leader
+test_replay ${image1} ${image2}
+
+testlog "TEST: in loop: stop leader and test replay"
+for i in 0 1 2 3 4 5; do
+    stop_mirror ${CLUSTER1}:${LEADER}
+    wait_for_leader
+    test_replay ${image1}
+done
+
+stop_mirror ${CLUSTER1}:${LEADER}
diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh
new file mode 100755
index 00000000..563e256c
--- /dev/null
+++ b/qa/workunits/rbd/rbd_mirror_helpers.sh
@@ -0,0 +1,1211 @@
+#!/bin/sh
+#
+# rbd_mirror_helpers.sh - shared rbd-mirror daemon helper functions
+#
+# The scripts starts two ("local" and "remote") clusters using mstart.sh script,
+# creates a temporary directory, used for cluster configs, daemon logs, admin
+# socket, temporary files, and launches rbd-mirror daemon.
+#
+# There are several env variables useful when troubleshooting a test failure:
+#
+#  RBD_MIRROR_NOCLEANUP  - if not empty, don't run the cleanup (stop processes,
+#                          destroy the clusters and remove the temp directory)
+#                          on exit, so it is possible to check the test state
+#                          after failure.
+#  RBD_MIRROR_TEMDIR     - use this path when creating the temporary directory
+#                          (should not exist) instead of running mktemp(1).
+#  RBD_MIRROR_ARGS       - use this to pass additional arguments to started
+#                          rbd-mirror daemons.
+#  RBD_MIRROR_VARGS      - use this to pass additional arguments to vstart.sh
+#                          when starting clusters.
+#  RBD_MIRROR_INSTANCES  - number of daemons to start per cluster
+#  RBD_MIRROR_CONFIG_KEY - if not empty, use config-key for remote cluster
+#                          secrets
+# The cleanup can be done as a separate step, running the script with
+# `cleanup ${RBD_MIRROR_TEMDIR}' arguments.
+#
+# Note, as other workunits tests, rbd_mirror.sh expects to find ceph binaries
+# in PATH.
+#
+# Thus a typical troubleshooting session:
+#
+# From Ceph src dir (CEPH_SRC_PATH), start the test in NOCLEANUP mode and with
+# TEMPDIR pointing to a known location:
+#
+#   cd $CEPH_SRC_PATH
+#   PATH=$CEPH_SRC_PATH:$PATH
+#   RBD_MIRROR_NOCLEANUP=1 RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
+#     ../qa/workunits/rbd/rbd_mirror.sh
+#
+# After the test failure cd to TEMPDIR and check the current state:
+#
+#   cd /tmp/tmp.rbd_mirror
+#   ls
+#   less rbd-mirror.cluster1_daemon.$pid.log
+#   ceph --cluster cluster1 -s
+#   ceph --cluster cluster1 -s
+#   rbd --cluster cluster2 -p mirror ls
+#   rbd --cluster cluster2 -p mirror journal status --image test
+#   ceph --admin-daemon rbd-mirror.cluster1_daemon.cluster1.$pid.asok help
+#   ...
+#
+# Also you can execute commands (functions) from the script:
+#
+#   cd $CEPH_SRC_PATH
+#   export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror
+#   ../qa/workunits/rbd/rbd_mirror.sh status
+#   ../qa/workunits/rbd/rbd_mirror.sh stop_mirror cluster1
+#   ../qa/workunits/rbd/rbd_mirror.sh start_mirror cluster2
+#   ../qa/workunits/rbd/rbd_mirror.sh flush cluster2
+#   ...
+#
+# Eventually, run the cleanup:
+#
+#   cd $CEPH_SRC_PATH
+#   RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
+#     ../qa/workunits/rbd/rbd_mirror.sh cleanup
+#
+
+if type xmlstarlet > /dev/null 2>&1; then
+    XMLSTARLET=xmlstarlet
+elif type xml > /dev/null 2>&1; then
+    XMLSTARLET=xml
+else
+    echo "Missing xmlstarlet binary!"
+    exit 1
+fi
+
+RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-2}
+
+CLUSTER1=cluster1
+CLUSTER2=cluster2
+PEER_CLUSTER_SUFFIX=
+POOL=mirror
+PARENT_POOL=mirror_parent
+TEMPDIR=
+CEPH_ID=${CEPH_ID:-mirror}
+MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.}
+export CEPH_ARGS="--id ${CEPH_ID}"
+
+LAST_MIRROR_INSTANCE=$((${RBD_MIRROR_INSTANCES} - 1))
+
+CEPH_ROOT=$(readlink -f $(dirname $0)/../../../src)
+CEPH_BIN=.
+CEPH_SRC=.
+if [ -e CMakeCache.txt ]; then
+    CEPH_SRC=${CEPH_ROOT}
+    CEPH_ROOT=${PWD}
+    CEPH_BIN=./bin
+
+    # needed for ceph CLI under cmake
+    export LD_LIBRARY_PATH=${CEPH_ROOT}/lib:${LD_LIBRARY_PATH}
+    export PYTHONPATH=${PYTHONPATH}:${CEPH_SRC}/pybind
+    for x in ${CEPH_ROOT}/lib/cython_modules/lib* ; do
+        export PYTHONPATH="${PYTHONPATH}:${x}"
+    done
+fi
+
+# These vars facilitate running this script in an environment with
+# ceph installed from packages, like teuthology. These are not defined
+# by default.
+#
+# RBD_MIRROR_USE_EXISTING_CLUSTER - if set, do not start and stop ceph clusters
+# RBD_MIRROR_USE_RBD_MIRROR - if set, use an existing instance of rbd-mirror
+#                             running as ceph client $CEPH_ID. If empty,
+#                             this script will start and stop rbd-mirror
+
+#
+# Functions
+#
+
+# Parse a value in format cluster[:instance] and set cluster and instance vars.
+set_cluster_instance()
+{
+    local val=$1
+    local cluster_var_name=$2
+    local instance_var_name=$3
+
+    cluster=${val%:*}
+    instance=${val##*:}
+
+    if [ "${instance}" =  "${val}" ]; then
+	# instance was not specified, use default
+	instance=0
+    fi
+
+    eval ${cluster_var_name}=${cluster}
+    eval ${instance_var_name}=${instance}
+}
+
+daemon_asok_file()
+{
+    local local_cluster=$1
+    local cluster=$2
+    local instance
+
+    set_cluster_instance "${local_cluster}" local_cluster instance
+
+    echo $(ceph-conf --cluster $local_cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'admin socket')
+}
+
+daemon_pid_file()
+{
+    local cluster=$1
+    local instance
+
+    set_cluster_instance "${cluster}" cluster instance
+
+    echo $(ceph-conf --cluster $cluster --name "client.${MIRROR_USER_ID_PREFIX}${instance}" 'pid file')
+}
+
+testlog()
+{
+    echo $(date '+%F %T') $@ | tee -a "${TEMPDIR}/rbd-mirror.test.log" >&2
+}
+
+expect_failure()
+{
+    local expected="$1" ; shift
+    local out=${TEMPDIR}/expect_failure.out
+
+    if "$@" > ${out} 2>&1 ; then
+        cat ${out} >&2
+        return 1
+    fi
+
+    if [ -z "${expected}" ]; then
+	return 0
+    fi
+
+    if ! grep -q "${expected}" ${out} ; then
+        cat ${out} >&2
+        return 1
+    fi
+
+    return 0
+}
+
+create_users()
+{
+    local cluster=$1
+
+    CEPH_ARGS='' ceph --cluster "${cluster}" \
+        auth get-or-create client.${CEPH_ID} \
+        mon 'profile rbd' osd 'profile rbd' mgr 'profile rbd' >> \
+        ${CEPH_ROOT}/run/${cluster}/keyring
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        CEPH_ARGS='' ceph --cluster "${cluster}" \
+            auth get-or-create client.${MIRROR_USER_ID_PREFIX}${instance} \
+            mon 'profile rbd-mirror' osd 'profile rbd' mgr 'profile rbd' >> \
+            ${CEPH_ROOT}/run/${cluster}/keyring
+    done
+}
+
+update_users()
+{
+    local cluster=$1
+
+    CEPH_ARGS='' ceph --cluster "${cluster}" \
+        auth caps client.${CEPH_ID} \
+        mon 'profile rbd' osd 'profile rbd'
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        CEPH_ARGS='' ceph --cluster "${cluster}" \
+            auth caps client.${MIRROR_USER_ID_PREFIX}${instance} \
+            mon 'profile rbd-mirror' osd 'profile rbd'
+    done
+}
+
+setup_cluster()
+{
+    local cluster=$1
+
+    CEPH_ARGS='' ${CEPH_SRC}/mstart.sh ${cluster} -n ${RBD_MIRROR_VARGS}
+
+    cd ${CEPH_ROOT}
+    rm -f ${TEMPDIR}/${cluster}.conf
+    ln -s $(readlink -f run/${cluster}/ceph.conf) \
+       ${TEMPDIR}/${cluster}.conf
+
+    cd ${TEMPDIR}
+    create_users "${cluster}"
+
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        cat<<EOF >> ${TEMPDIR}/${cluster}.conf
+[client.${MIRROR_USER_ID_PREFIX}${instance}]
+    admin socket = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.asok
+    pid file = ${TEMPDIR}/rbd-mirror.\$cluster-\$name.pid
+    log file = ${TEMPDIR}/rbd-mirror.${cluster}_daemon.${instance}.log
+EOF
+    done
+}
+
+setup_pools()
+{
+    local cluster=$1
+    local remote_cluster=$2
+    local mon_map_file
+    local mon_addr
+    local admin_key_file
+    local uuid
+
+    CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${POOL} 64 64
+    CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${PARENT_POOL} 64 64
+
+    CEPH_ARGS='' rbd --cluster ${cluster} pool init ${POOL}
+    CEPH_ARGS='' rbd --cluster ${cluster} pool init ${PARENT_POOL}
+
+    CEPH_ARGS='' rbd --cluster ${cluster} mirror pool enable ${POOL} pool
+    rbd --cluster ${cluster} mirror pool enable ${PARENT_POOL} image
+
+    if [ -z ${RBD_MIRROR_MANUAL_PEERS} ]; then
+      if [ -z ${RBD_MIRROR_CONFIG_KEY} ]; then
+        rbd --cluster ${cluster} mirror pool peer add ${POOL} ${remote_cluster}
+        rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} ${remote_cluster}
+      else
+        mon_map_file=${TEMPDIR}/${remote_cluster}.monmap
+        CEPH_ARGS='' ceph --cluster ${remote_cluster} mon getmap > ${mon_map_file}
+        mon_addr=$(monmaptool --print ${mon_map_file} | grep -E 'mon\.' |
+          head -n 1 | sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' | sed -E 's/\/[0-9]+//g')
+
+        admin_key_file=${TEMPDIR}/${remote_cluster}.client.${CEPH_ID}.key
+        CEPH_ARGS='' ceph --cluster ${remote_cluster} auth get-key client.${CEPH_ID} > ${admin_key_file}
+
+        CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer add ${POOL} \
+            client.${CEPH_ID}@${remote_cluster}-DNE \
+            --remote-mon-host "${mon_addr}" --remote-key-file ${admin_key_file}
+
+        uuid=$(rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} client.${CEPH_ID}@${remote_cluster}-DNE)
+        CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr}
+        CEPH_ARGS='' rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file}
+
+        PEER_CLUSTER_SUFFIX=-DNE
+      fi
+    fi
+}
+
+setup_tempdir()
+{
+    if [ -n "${RBD_MIRROR_TEMDIR}" ]; then
+	test -d "${RBD_MIRROR_TEMDIR}" ||
+	mkdir "${RBD_MIRROR_TEMDIR}"
+	TEMPDIR="${RBD_MIRROR_TEMDIR}"
+	cd ${TEMPDIR}
+    else
+	TEMPDIR=`mktemp -d`
+    fi
+}
+
+setup()
+{
+    local c
+    trap 'cleanup $?' INT TERM EXIT
+
+    setup_tempdir
+    if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then
+	setup_cluster "${CLUSTER1}"
+	setup_cluster "${CLUSTER2}"
+    else
+	update_users "${CLUSTER1}"
+	update_users "${CLUSTER2}"
+    fi
+
+    setup_pools "${CLUSTER1}" "${CLUSTER2}"
+    setup_pools "${CLUSTER2}" "${CLUSTER1}"
+}
+
+cleanup()
+{
+    local error_code=$1
+
+    set +e
+
+    if [ "${error_code}" -ne 0 ]; then
+        status
+    fi
+
+    if [ -z "${RBD_MIRROR_NOCLEANUP}" ]; then
+        local cluster instance
+
+        CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it
+        CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it
+        CEPH_ARGS='' ceph --cluster ${CLUSTER1} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it
+        CEPH_ARGS='' ceph --cluster ${CLUSTER2} osd pool rm ${PARENT_POOL} ${PARENT_POOL} --yes-i-really-really-mean-it
+
+        for cluster in "${CLUSTER1}" "${CLUSTER2}"; do
+	    stop_mirrors "${cluster}"
+        done
+
+        if [ -z "${RBD_MIRROR_USE_EXISTING_CLUSTER}" ]; then
+            cd ${CEPH_ROOT}
+            CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER1}
+            CEPH_ARGS='' ${CEPH_SRC}/mstop.sh ${CLUSTER2}
+        fi
+        test "${RBD_MIRROR_TEMDIR}" = "${TEMPDIR}" || rm -Rf ${TEMPDIR}
+    fi
+
+    if [ "${error_code}" -eq 0 ]; then
+        echo "OK"
+    else
+        echo "FAIL"
+    fi
+
+    exit ${error_code}
+}
+
+start_mirror()
+{
+    local cluster=$1
+    local instance
+
+    set_cluster_instance "${cluster}" cluster instance
+
+    test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return
+
+    rbd-mirror \
+	--cluster ${cluster} \
+        --id ${MIRROR_USER_ID_PREFIX}${instance} \
+	--rbd-mirror-delete-retry-interval=5 \
+	--rbd-mirror-image-state-check-interval=5 \
+	--rbd-mirror-journal-poll-age=1 \
+	--rbd-mirror-pool-replayers-refresh-interval=5 \
+	--debug-rbd=30 --debug-journaler=30 \
+	--debug-rbd_mirror=30 \
+	--daemonize=true \
+	${RBD_MIRROR_ARGS}
+}
+
+start_mirrors()
+{
+    local cluster=$1
+
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        start_mirror "${cluster}:${instance}"
+    done
+}
+
+stop_mirror()
+{
+    local cluster=$1
+    local sig=$2
+
+    test -n "${RBD_MIRROR_USE_RBD_MIRROR}" && return
+
+    local pid
+    pid=$(cat $(daemon_pid_file "${cluster}") 2>/dev/null) || :
+    if [ -n "${pid}" ]
+    then
+	kill ${sig} ${pid}
+	for s in 1 2 4 8 16 32; do
+	    sleep $s
+	    ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' && break
+	done
+	ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}'
+    fi
+    rm -f $(daemon_asok_file "${cluster}" "${CLUSTER1}")
+    rm -f $(daemon_asok_file "${cluster}" "${CLUSTER2}")
+    rm -f $(daemon_pid_file "${cluster}")
+}
+
+stop_mirrors()
+{
+    local cluster=$1
+    local sig=$2
+
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+        stop_mirror "${cluster}:${instance}" "${sig}"
+    done
+}
+
+admin_daemon()
+{
+    local cluster=$1 ; shift
+    local instance
+
+    set_cluster_instance "${cluster}" cluster instance
+
+    local asok_file=$(daemon_asok_file "${cluster}:${instance}" "${cluster}")
+    test -S "${asok_file}"
+
+    ceph --admin-daemon ${asok_file} $@
+}
+
+admin_daemons()
+{
+    local cluster_instance=$1 ; shift
+    local cluster="${cluster_instance%:*}"
+    local instance="${cluster_instance##*:}"
+    local loop_instance
+
+    for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16; do
+	sleep ${s}
+	if [ "${instance}" != "${cluster_instance}" ]; then
+	    admin_daemon "${cluster}:${instance}" $@ && return 0
+	else
+	    for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+		admin_daemon "${cluster}:${loop_instance}" $@ && return 0
+	    done
+	fi
+    done
+    return 1
+}
+
+all_admin_daemons()
+{
+    local cluster=$1 ; shift
+
+    for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+	admin_daemon "${cluster}:${instance}" $@
+    done
+}
+
+status()
+{
+    local cluster daemon image_pool image
+
+    for cluster in ${CLUSTER1} ${CLUSTER2}
+    do
+	echo "${cluster} status"
+	CEPH_ARGS='' ceph --cluster ${cluster} -s
+	CEPH_ARGS='' ceph --cluster ${cluster} service dump
+	CEPH_ARGS='' ceph --cluster ${cluster} service status
+	echo
+
+	for image_pool in ${POOL} ${PARENT_POOL}
+	do
+	    echo "${cluster} ${image_pool} images"
+	    rbd --cluster ${cluster} -p ${image_pool} ls -l
+	    echo
+
+	    echo "${cluster} ${image_pool} mirror pool status"
+	    CEPH_ARGS='' rbd --cluster ${cluster} -p ${image_pool} mirror pool status --verbose
+	    echo
+
+	    for image in `rbd --cluster ${cluster} -p ${image_pool} ls 2>/dev/null`
+	    do
+	        echo "image ${image} info"
+	        rbd --cluster ${cluster} -p ${image_pool} info ${image}
+	        echo
+	        echo "image ${image} journal status"
+	        rbd --cluster ${cluster} -p ${image_pool} journal status --image ${image}
+	        echo
+	    done
+	done
+    done
+
+    local ret
+
+    for cluster in "${CLUSTER1}" "${CLUSTER2}"
+    do
+	for instance in `seq 0 ${LAST_MIRROR_INSTANCE}`; do
+	    local pid_file=$(daemon_pid_file ${cluster}:${instance})
+	    if [ ! -e ${pid_file} ]
+	    then
+		echo "${cluster} rbd-mirror not running or unknown" \
+		     "(${pid_file} not exist)"
+		continue
+	    fi
+
+	    local pid
+	    pid=$(cat ${pid_file} 2>/dev/null) || :
+	    if [ -z "${pid}" ]
+	    then
+		echo "${cluster} rbd-mirror not running or unknown" \
+		     "(can't find pid using ${pid_file})"
+		ret=1
+		continue
+	    fi
+
+	    echo "${daemon} rbd-mirror process in ps output:"
+	    if ps auxww |
+		awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}'
+	    then
+		echo
+		echo "${cluster} rbd-mirror not running" \
+		     "(can't find pid $pid in ps output)"
+		ret=1
+		continue
+	    fi
+	    echo
+
+	    local asok_file=$(daemon_asok_file ${cluster}:${instance} ${cluster})
+	    if [ ! -S "${asok_file}" ]
+	    then
+		echo "${cluster} rbd-mirror asok is unknown (${asok_file} not exits)"
+		ret=1
+		continue
+	    fi
+
+	    echo "${cluster} rbd-mirror status"
+	    ceph --admin-daemon ${asok_file} rbd mirror status
+	    echo
+	done
+    done
+
+    return ${ret}
+}
+
+flush()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local cmd="rbd mirror flush"
+
+    if [ -n "${image}" ]
+    then
+       cmd="${cmd} ${pool}/${image}"
+    fi
+
+    admin_daemons "${cluster}" ${cmd}
+}
+
+test_image_replay_state()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local test_state=$4
+    local status_result
+    local current_state=stopped
+
+    status_result=$(admin_daemons "${cluster}" rbd mirror status ${pool}/${image} | grep -i 'state') || return 1
+    echo "${status_result}" | grep -i 'Replaying' && current_state=started
+    test "${test_state}" = "${current_state}"
+}
+
+wait_for_image_replay_state()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local state=$4
+    local s
+
+    # TODO: add a way to force rbd-mirror to update replayers
+    for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
+	sleep ${s}
+	test_image_replay_state "${cluster}" "${pool}" "${image}" "${state}" && return 0
+    done
+    return 1
+}
+
+wait_for_image_replay_started()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    wait_for_image_replay_state "${cluster}" "${pool}" "${image}" started
+}
+
+wait_for_image_replay_stopped()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    wait_for_image_replay_state "${cluster}" "${pool}" "${image}" stopped
+}
+
+get_position()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local id_regexp=$4
+
+    # Parse line like below, looking for the first position
+    # [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=9], [object_number=0, tag_tid=3, entry_tid=8], [object_number=3, tag_tid=3, entry_tid=7], [object_number=2, tag_tid=3, entry_tid=6]]]]
+
+    local status_log=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.status
+    rbd --cluster ${cluster} -p ${pool} journal status --image ${image} |
+	tee ${status_log} >&2
+    sed -nEe 's/^.*\[id='"${id_regexp}"',.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' \
+	${status_log}
+}
+
+get_master_position()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    get_position "${cluster}" "${pool}" "${image}" ''
+}
+
+get_mirror_position()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    get_position "${cluster}" "${pool}" "${image}" '..*'
+}
+
+wait_for_replay_complete()
+{
+    local local_cluster=$1
+    local cluster=$2
+    local pool=$3
+    local image=$4
+    local s master_pos mirror_pos last_mirror_pos
+    local master_tag master_entry mirror_tag mirror_entry
+
+    while true; do
+        for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32; do
+	    sleep ${s}
+	    flush "${local_cluster}" "${pool}" "${image}"
+	    master_pos=$(get_master_position "${cluster}" "${pool}" "${image}")
+	    mirror_pos=$(get_mirror_position "${cluster}" "${pool}" "${image}")
+	    test -n "${master_pos}" -a "${master_pos}" = "${mirror_pos}" && return 0
+            test "${mirror_pos}" != "${last_mirror_pos}" && break
+        done
+
+        test "${mirror_pos}" = "${last_mirror_pos}" && return 1
+        last_mirror_pos="${mirror_pos}"
+
+        # handle the case where the mirror is ahead of the master
+        master_tag=$(echo "${master_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2)
+        mirror_tag=$(echo "${mirror_pos}" | grep -Eo "tag_tid=[0-9]*" | cut -d'=' -f 2)
+        master_entry=$(echo "${master_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2)
+        mirror_entry=$(echo "${mirror_pos}" | grep -Eo "entry_tid=[0-9]*" | cut -d'=' -f 2)
+        test "${master_tag}" = "${mirror_tag}" -a ${master_entry} -le ${mirror_entry} && return 0
+    done
+    return 1
+}
+
+test_status_in_pool_dir()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local state_pattern="$4"
+    local description_pattern="$5"
+    local service_pattern="$6"
+
+    local status_log=${TEMPDIR}/${cluster}-${pool}-${image}.mirror_status
+    CEPH_ARGS='' rbd --cluster ${cluster} -p ${pool} mirror image status ${image} |
+	tee ${status_log} >&2
+    grep "state: .*${state_pattern}" ${status_log} || return 1
+    grep "description: .*${description_pattern}" ${status_log} || return 1
+
+    if [ -n "${service_pattern}" ]; then
+        grep "service: *${service_pattern}" ${status_log} || return 1
+    elif echo ${state_pattern} | grep '^up+'; then
+        grep "service: *${MIRROR_USER_ID_PREFIX}.* on " ${status_log} || return 1
+    else
+        grep "service: " ${status_log} && return 1
+    fi
+
+    # recheck using `mirror pool status` command to stress test it.
+
+    local last_update="$(sed -nEe 's/^ *last_update: *(.*) *$/\1/p' ${status_log})"
+    test_mirror_pool_status_verbose \
+        ${cluster} ${pool} ${image} "${state_pattern}" "${last_update}" &&
+    return 0
+
+    echo "'mirror pool status' test failed" >&2
+    exit 1
+}
+
+test_mirror_pool_status_verbose()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local state_pattern="$4"
+    local prev_last_update="$5"
+
+    local status_log=${TEMPDIR}/${cluster}-${pool}.mirror_status
+
+    rbd --cluster ${cluster} mirror pool status ${pool} --verbose --format xml \
+        > ${status_log}
+
+    local last_update state
+    last_update=$($XMLSTARLET sel -t -v \
+        "//images/image[name='${image}']/last_update" < ${status_log})
+    state=$($XMLSTARLET sel -t -v \
+        "//images/image[name='${image}']/state" < ${status_log})
+
+    echo "${state}" | grep "${state_pattern}" ||
+    test "${last_update}" '>' "${prev_last_update}"
+}
+
+wait_for_status_in_pool_dir()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local state_pattern="$4"
+    local description_pattern="$5"
+    local service_pattern="$6"
+
+    for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
+	sleep ${s}
+	test_status_in_pool_dir ${cluster} ${pool} ${image} "${state_pattern}" \
+                                "${description_pattern}" "${service_pattern}" &&
+            return 0
+    done
+    return 1
+}
+
+create_image()
+{
+    local cluster=$1 ; shift
+    local pool=$1 ; shift
+    local image=$1 ; shift
+    local size=128
+
+    if [ -n "$1" ]; then
+	size=$1
+	shift
+    fi
+
+    rbd --cluster ${cluster} -p ${pool} create --size ${size} \
+	--image-feature layering,exclusive-lock,journaling $@ ${image}
+}
+
+set_image_meta()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local key=$4
+    local val=$5
+
+    rbd --cluster ${cluster} -p ${pool} image-meta set ${image} $key $val
+}
+
+compare_image_meta()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local key=$4
+    local value=$5
+
+    test `rbd --cluster ${cluster} -p ${pool} image-meta get ${image} ${key}` = "${value}"
+}
+
+rename_image()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local new_name=$4
+
+    rbd --cluster=${cluster} -p ${pool} rename ${image} ${new_name}
+}
+
+remove_image()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster=${cluster} -p ${pool} snap purge ${image}
+    rbd --cluster=${cluster} -p ${pool} rm ${image}
+}
+
+remove_image_retry()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    for s in 0 1 2 4 8 16 32; do
+        sleep ${s}
+        remove_image ${cluster} ${pool} ${image} && return 0
+    done
+    return 1
+}
+
+trash_move() {
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster=${cluster} -p ${pool} trash move ${image}
+}
+
+trash_restore() {
+    local cluster=$1
+    local pool=$2
+    local image_id=$3
+
+    rbd --cluster=${cluster} -p ${pool} trash restore ${image_id}
+}
+
+clone_image()
+{
+    local cluster=$1
+    local parent_pool=$2
+    local parent_image=$3
+    local parent_snap=$4
+    local clone_pool=$5
+    local clone_image=$6
+
+    shift 6
+
+    rbd --cluster ${cluster} clone \
+        ${parent_pool}/${parent_image}@${parent_snap} \
+        ${clone_pool}/${clone_image} \
+        --image-feature layering,exclusive-lock,journaling $@
+}
+
+disconnect_image()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster ${cluster} -p ${pool} journal client disconnect \
+	--image ${image}
+}
+
+create_snapshot()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local snap=$4
+
+    rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap}
+}
+
+remove_snapshot()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local snap=$4
+
+    rbd --cluster ${cluster} -p ${pool} snap rm ${image}@${snap}
+}
+
+rename_snapshot()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local snap=$4
+    local new_snap=$5
+
+    rbd --cluster ${cluster} -p ${pool} snap rename ${image}@${snap} ${image}@${new_snap}
+}
+
+purge_snapshots()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster ${cluster} -p ${pool} snap purge ${image}
+}
+
+protect_snapshot()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local snap=$4
+
+    rbd --cluster ${cluster} -p ${pool} snap protect ${image}@${snap}
+}
+
+unprotect_snapshot()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local snap=$4
+
+    rbd --cluster ${cluster} -p ${pool} snap unprotect ${image}@${snap}
+}
+
+wait_for_snap_present()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local snap_name=$4
+    local s
+
+    for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do
+	sleep ${s}
+        rbd --cluster ${cluster} -p ${pool} info ${image}@${snap_name} || continue
+        return 0
+    done
+    return 1
+}
+
+test_snap_moved_to_trash()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local snap_name=$4
+
+    rbd --cluster ${cluster} snap ls ${pool}/${image} --all |
+        grep -F " trash (${snap_name})"
+}
+
+wait_for_snap_moved_to_trash()
+{
+    local s
+
+    for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do
+	sleep ${s}
+        test_snap_moved_to_trash $@ || continue
+        return 0
+    done
+    return 1
+}
+
+test_snap_removed_from_trash()
+{
+    test_snap_moved_to_trash $@ && return 1
+    return 0
+}
+
+wait_for_snap_removed_from_trash()
+{
+    local s
+
+    for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32; do
+	sleep ${s}
+        test_snap_removed_from_trash $@ || continue
+        return 0
+    done
+    return 1
+}
+
+write_image()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local count=$4
+    local size=$5
+
+    test -n "${size}" || size=4096
+
+    rbd --cluster ${cluster} -p ${pool} bench ${image} --io-type write \
+	--io-size ${size} --io-threads 1 --io-total $((size * count)) \
+	--io-pattern rand
+}
+
+stress_write_image()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local duration=$(awk 'BEGIN {srand(); print int(10 * rand()) + 5}')
+
+    set +e
+    timeout ${duration}s ceph_test_rbd_mirror_random_write \
+        --cluster ${cluster} ${pool} ${image} \
+        --debug-rbd=20 --debug-journaler=20 \
+        2> ${TEMPDIR}/rbd-mirror-random-write.log
+    error_code=$?
+    set -e
+
+    if [ $error_code -eq 124 ]; then
+        return 0
+    fi
+    return 1
+}
+
+compare_images()
+{
+    local pool=$1
+    local image=$2
+
+    local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
+    local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
+
+    rm -f ${rmt_export} ${loc_export}
+    rbd --cluster ${CLUSTER2} -p ${pool} export ${image} - | xxd > ${rmt_export}
+    rbd --cluster ${CLUSTER1} -p ${pool} export ${image} - | xxd > ${loc_export}
+    sdiff -s ${rmt_export} ${loc_export} | head -n 64
+    rm -f ${rmt_export} ${loc_export}
+}
+
+compare_image_snapshots()
+{
+    local pool=$1
+    local image=$2
+
+    local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
+    local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
+
+    for snap_name in $(rbd --cluster ${CLUSTER1} -p ${pool} --format xml \
+                           snap list ${image} | \
+                           $XMLSTARLET sel -t -v "//snapshot/name" | \
+                           grep -E -v "^\.rbd-mirror\."); do
+        rm -f ${rmt_export} ${loc_export}
+        rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} - | xxd > ${rmt_export}
+        rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} - | xxd > ${loc_export}
+        sdiff -s ${rmt_export} ${loc_export} | head -n 64
+    done
+    rm -f ${rmt_export} ${loc_export}
+}
+
+demote_image()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster=${cluster} mirror image demote ${pool}/${image}
+}
+
+promote_image()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local force=$4
+
+    rbd --cluster=${cluster} mirror image promote ${pool}/${image} ${force}
+}
+
+set_pool_mirror_mode()
+{
+    local cluster=$1
+    local pool=$2
+    local mode=$3
+
+    rbd --cluster=${cluster} -p ${pool} mirror pool enable ${mode}
+}
+
+disable_mirror()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster=${cluster} mirror image disable ${pool}/${image}
+}
+
+enable_mirror()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster=${cluster} mirror image enable ${pool}/${image}
+}
+
+test_image_present()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local test_state=$4
+    local image_id=$5
+    local current_state=deleted
+    local current_image_id
+
+    current_image_id=$(get_image_id ${cluster} ${pool} ${image})
+    test -n "${current_image_id}" &&
+    test -z "${image_id}" -o "${image_id}" = "${current_image_id}" &&
+    current_state=present
+
+    test "${test_state}" = "${current_state}"
+}
+
+wait_for_image_present()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local state=$4
+    local image_id=$5
+    local s
+
+    test -n "${image_id}" ||
+    image_id=$(get_image_id ${cluster} ${pool} ${image})
+
+    # TODO: add a way to force rbd-mirror to update replayers
+    for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32; do
+	sleep ${s}
+	test_image_present \
+            "${cluster}" "${pool}" "${image}" "${state}" "${image_id}" &&
+        return 0
+    done
+    return 1
+}
+
+get_image_id()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster=${cluster} -p ${pool} info ${image} |
+	sed -ne 's/^.*block_name_prefix: rbd_data\.//p'
+}
+
+request_resync_image()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local image_id_var_name=$4
+
+    eval "${image_id_var_name}='$(get_image_id ${cluster} ${pool} ${image})'"
+    eval 'test -n "$'${image_id_var_name}'"'
+
+    rbd --cluster=${cluster} -p ${pool} mirror image resync ${image}
+}
+
+get_image_data_pool()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster ${cluster} -p ${pool} info ${image} |
+        awk '$1 == "data_pool:" {print $2}'
+}
+
+get_clone_format()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster ${cluster} info ${pool}/${image} |
+        awk 'BEGIN {
+               format = 1
+             }
+             $1 == "parent:" {
+               parent = $2
+             }
+             /op_features: .*clone-child/ {
+               format = 2
+             }
+             END {
+               if (!parent) exit 1
+               print format
+             }'
+}
+
+#
+# Main
+#
+
+if [ "$#" -gt 0 ]
+then
+    if [ -z "${RBD_MIRROR_TEMDIR}" ]
+    then
+       echo "RBD_MIRROR_TEMDIR is not set" >&2
+       exit 1
+    fi
+
+    TEMPDIR="${RBD_MIRROR_TEMDIR}"
+    cd ${TEMPDIR}
+    $@
+    exit $?
+fi
diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh
new file mode 100755
index 00000000..a1d7d034
--- /dev/null
+++ b/qa/workunits/rbd/rbd_mirror_stress.sh
@@ -0,0 +1,186 @@
+#!/bin/sh -ex
+#
+# rbd_mirror_stress.sh - stress test rbd-mirror daemon
+#
+# The following additional environment variables affect the test:
+#
+#  RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write
+#                             tool during the many image test
+#
+
+IMAGE_COUNT=50
+export LOCKDEP=0
+
+. $(dirname $0)/rbd_mirror_helpers.sh
+
+setup
+
+create_snap()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+    local snap_name=$4
+
+    rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap_name} \
+	--debug-rbd=20 --debug-journaler=20 2> ${TEMPDIR}/rbd-snap-create.log
+}
+
+compare_image_snaps()
+{
+    local pool=$1
+    local image=$2
+    local snap_name=$3
+
+    local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
+    local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
+
+    rm -f ${rmt_export} ${loc_export}
+    rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} ${rmt_export}
+    rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} ${loc_export}
+    cmp ${rmt_export} ${loc_export}
+    rm -f ${rmt_export} ${loc_export}
+}
+
+wait_for_pool_images()
+{
+    local cluster=$1
+    local pool=$2
+    local image_count=$3
+    local s
+    local count
+    local last_count=0
+
+    while true; do
+        for s in `seq 1 40`; do
+            test $s -ne 1 && sleep 30
+            count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2)
+            test "${count}" = "${image_count}" && return 0
+
+            # reset timeout if making forward progress
+            test $count -ne $last_count && break
+        done
+
+        test $count -eq $last_count && break
+        last_count=$count
+    done
+    rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
+    return 1
+}
+
+wait_for_pool_healthy()
+{
+    local cluster=$1
+    local pool=$2
+    local s
+    local state
+
+    for s in `seq 1 40`; do
+        test $s -ne 1 && sleep 30
+        state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:' | cut -d' ' -f 2)
+        test "${state}" = "ERROR" && break
+        test "${state}" = "OK" && return 0
+    done
+    rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
+    return 1
+}
+
+start_mirrors ${CLUSTER1}
+start_mirrors ${CLUSTER2}
+
+testlog "TEST: add image and test replay after client crashes"
+image=test
+create_image ${CLUSTER2} ${POOL} ${image} '512M'
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+
+for i in `seq 1 10`
+do
+  stress_write_image ${CLUSTER2} ${POOL} ${image}
+
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
+
+  snap_name="snap${i}"
+  create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
+  compare_image_snaps ${POOL} ${image} ${snap_name}
+done
+
+for i in `seq 1 10`
+do
+  snap_name="snap${i}"
+  remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
+done
+
+remove_image_retry ${CLUSTER2} ${POOL} ${image}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
+
+testlog "TEST: create many images"
+snap_name="snap"
+for i in `seq 1 ${IMAGE_COUNT}`
+do
+  image="image_${i}"
+  create_image ${CLUSTER2} ${POOL} ${image} '128M'
+  if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then
+    write_image ${CLUSTER2} ${POOL} ${image} 100
+  else
+    stress_write_image ${CLUSTER2} ${POOL} ${image}
+  fi
+done
+
+wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
+wait_for_pool_healthy ${CLUSTER2} ${POOL}
+
+wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
+wait_for_pool_healthy ${CLUSTER1} ${POOL}
+
+testlog "TEST: compare many images"
+for i in `seq 1 ${IMAGE_COUNT}`
+do
+  image="image_${i}"
+  create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
+  compare_image_snaps ${POOL} ${image} ${snap_name}
+done
+
+testlog "TEST: delete many images"
+for i in `seq 1 ${IMAGE_COUNT}`
+do
+  image="image_${i}"
+  remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
+  remove_image_retry ${CLUSTER2} ${POOL} ${image}
+done
+
+testlog "TEST: image deletions should propagate"
+wait_for_pool_images ${CLUSTER1} ${POOL} 0
+wait_for_pool_healthy ${CLUSTER1} ${POOL} 0
+for i in `seq 1 ${IMAGE_COUNT}`
+do
+  image="image_${i}"
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
+done
+
+testlog "TEST: delete images during bootstrap"
+set_pool_mirror_mode ${CLUSTER1} ${POOL} 'image'
+set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
+
+start_mirror ${CLUSTER1}
+image=test
+
+for i in `seq 1 10`
+do
+  image="image_${i}"
+  create_image ${CLUSTER2} ${POOL} ${image} '512M'
+  enable_mirror ${CLUSTER2} ${POOL} ${image}
+
+  stress_write_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
+
+  disable_mirror ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
+  purge_snapshots ${CLUSTER2} ${POOL} ${image}
+  remove_image_retry ${CLUSTER2} ${POOL} ${image}
+done
diff --git a/qa/workunits/rbd/read-flags.sh b/qa/workunits/rbd/read-flags.sh
new file mode 100755
index 00000000..7d787ce6
--- /dev/null
+++ b/qa/workunits/rbd/read-flags.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -ex
+
+# create a snapshot, then export it and check that setting read flags works
+# by looking at --debug-ms output
+
+function clean_up {
+    rm -f test.log || true
+    rbd snap remove test@snap || true
+    rbd rm test || true
+}
+
+function test_read_flags {
+    local IMAGE=$1
+    local SET_BALANCED=$2
+    local SET_LOCALIZED=$3
+    local EXPECT_BALANCED=$4
+    local EXPECT_LOCALIZED=$5
+
+    local EXTRA_ARGS="--log-file test.log --debug-ms 1 --no-log-to-stderr"
+    if [ "$SET_BALANCED" = 'y' ]; then
+	EXTRA_ARGS="$EXTRA_ARGS --rbd-balance-snap-reads"
+    elif [ "$SET_LOCALIZED" = 'y' ]; then
+	EXTRA_ARGS="$EXTRA_ARGS --rbd-localize-snap-reads"
+    fi
+
+    rbd export $IMAGE - $EXTRA_ARGS > /dev/null
+    if [ "$EXPECT_BALANCED" = 'y' ]; then
+	grep -q balance_reads test.log
+    else
+	grep -L balance_reads test.log | grep -q test.log
+    fi
+    if [ "$EXPECT_LOCALIZED" = 'y' ]; then
+	grep -q localize_reads test.log
+    else
+	grep -L localize_reads test.log | grep -q test.log
+    fi
+    rm -f test.log
+
+}
+
+clean_up
+
+trap clean_up INT TERM EXIT
+
+rbd create --image-feature layering -s 10 test
+rbd snap create test@snap
+
+# export from non snapshot with or without settings should not have flags
+test_read_flags test n n n n
+test_read_flags test y y n n
+
+# export from snapshot should have read flags in log if they are set
+test_read_flags test@snap n n n n
+test_read_flags test@snap y n y n
+test_read_flags test@snap n y n y
+
+# balanced_reads happens to take priority over localize_reads
+test_read_flags test@snap y y y n
+
+echo OK
diff --git a/qa/workunits/rbd/simple_big.sh b/qa/workunits/rbd/simple_big.sh
new file mode 100755
index 00000000..70aafda4
--- /dev/null
+++ b/qa/workunits/rbd/simple_big.sh
@@ -0,0 +1,12 @@
+#!/bin/sh -ex
+
+mb=100000
+
+rbd create foo --size $mb
+DEV=$(sudo rbd map foo)
+dd if=/dev/zero of=$DEV bs=1M count=$mb
+dd if=$DEV of=/dev/null bs=1M count=$mb
+sudo rbd unmap $DEV
+rbd rm foo
+
+echo OK
diff --git a/qa/workunits/rbd/test_admin_socket.sh b/qa/workunits/rbd/test_admin_socket.sh
new file mode 100755
index 00000000..6b960787
--- /dev/null
+++ b/qa/workunits/rbd/test_admin_socket.sh
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+set -ex
+
+TMPDIR=/tmp/rbd_test_admin_socket$$
+mkdir $TMPDIR
+trap "rm -fr $TMPDIR" 0
+
+. $(dirname $0)/../../standalone/ceph-helpers.sh
+
+function expect_false()
+{
+    set -x
+    if "$@"; then return 1; else return 0; fi
+}
+
+function rbd_watch_out_file()
+{
+    echo ${TMPDIR}/rbd_watch_$1.out
+}
+
+function rbd_watch_pid_file()
+{
+    echo ${TMPDIR}/rbd_watch_$1.pid
+}
+
+function rbd_watch_fifo()
+{
+    echo ${TMPDIR}/rbd_watch_$1.fifo
+}
+
+function rbd_watch_asok()
+{
+    echo ${TMPDIR}/rbd_watch_$1.asok
+}
+
+function rbd_get_perfcounter()
+{
+    local image=$1
+    local counter=$2
+    local name
+
+    name=$(ceph --format xml --admin-daemon $(rbd_watch_asok ${image}) \
+		perf schema | $XMLSTARLET el -d3 |
+		  grep "/librbd-.*-${image}/${counter}\$")
+    test -n "${name}" || return 1
+
+    ceph --format xml --admin-daemon $(rbd_watch_asok ${image}) perf dump |
+	$XMLSTARLET sel -t -m "${name}" -v .
+}
+
+function rbd_check_perfcounter()
+{
+    local image=$1
+    local counter=$2
+    local expected_val=$3
+    local val=
+
+    val=$(rbd_get_perfcounter ${image} ${counter})
+
+    test "${val}" -eq "${expected_val}"
+}
+
+function rbd_watch_start()
+{
+    local image=$1
+    local asok=$(rbd_watch_asok ${image})
+
+    mkfifo $(rbd_watch_fifo ${image})
+    (cat $(rbd_watch_fifo ${image}) |
+	    rbd --admin-socket ${asok} watch ${image} \
+                > $(rbd_watch_out_file ${image}) 2>&1)&
+
+    # find pid of the started rbd watch process
+    local pid
+    for i in `seq 10`; do
+	pid=$(ps auxww | awk "/[r]bd --admin.* watch ${image}/ {print \$2}")
+	test -n "${pid}" && break
+	sleep 0.1
+    done
+    test -n "${pid}"
+    echo ${pid} > $(rbd_watch_pid_file ${image})
+
+    # find watcher admin socket
+    test -n "${asok}"
+    for i in `seq 10`; do
+	test -S "${asok}" && break
+	sleep 0.1
+    done
+    test -S "${asok}"
+
+    # configure debug level
+    ceph --admin-daemon "${asok}" config set debug_rbd 20
+
+    # check that watcher is registered
+    rbd status ${image} | expect_false grep "Watchers: none"
+}
+
+function rbd_watch_end()
+{
+    local image=$1
+    local regexp=$2
+
+    # send 'enter' to watch to exit
+    echo > $(rbd_watch_fifo ${image})
+    # just in case it is not terminated
+    kill $(cat $(rbd_watch_pid_file ${image})) || :
+
+    # output rbd watch out file for easier troubleshooting
+    cat $(rbd_watch_out_file ${image})
+
+    # cleanup
+    rm -f $(rbd_watch_fifo ${image}) $(rbd_watch_pid_file ${image}) \
+       $(rbd_watch_out_file ${image}) $(rbd_watch_asok ${image})
+}
+
+pool="rbd"
+image=testimg$$
+ceph_admin="ceph --admin-daemon $(rbd_watch_asok ${image})"
+
+rbd create --size 128 ${pool}/${image}
+
+# check rbd cache commands are present in help output
+rbd_cache_flush="rbd cache flush ${pool}/${image}"
+rbd_cache_invalidate="rbd cache invalidate ${pool}/${image}"
+
+rbd_watch_start ${image}
+${ceph_admin} help | fgrep "${rbd_cache_flush}"
+${ceph_admin} help | fgrep "${rbd_cache_invalidate}"
+rbd_watch_end ${image}
+
+# test rbd cache commands with disabled and enabled cache
+for conf_rbd_cache in false true; do
+
+    rbd image-meta set ${image} conf_rbd_cache ${conf_rbd_cache}
+
+    rbd_watch_start ${image}
+
+    rbd_check_perfcounter ${image} flush 0
+    ${ceph_admin} ${rbd_cache_flush}
+    # 'flush' counter should increase regardless if cache is enabled
+    rbd_check_perfcounter ${image} flush 1
+
+    rbd_check_perfcounter ${image} invalidate_cache 0
+    ${ceph_admin} ${rbd_cache_invalidate}
+    # 'invalidate_cache' counter should increase regardless if cache is enabled
+    rbd_check_perfcounter ${image} invalidate_cache 1
+
+    rbd_watch_end ${image}
+done
+
+rbd rm ${image}
diff --git a/qa/workunits/rbd/test_librbd.sh b/qa/workunits/rbd/test_librbd.sh
new file mode 100755
index 00000000..447306bb
--- /dev/null
+++ b/qa/workunits/rbd/test_librbd.sh
@@ -0,0 +1,9 @@
+#!/bin/sh -e
+
+if [ -n "${VALGRIND}" ]; then
+  valgrind ${VALGRIND} --suppressions=${TESTDIR}/valgrind.supp \
+    --error-exitcode=1 ceph_test_librbd
+else
+  ceph_test_librbd
+fi
+exit 0
diff --git a/qa/workunits/rbd/test_librbd_python.sh b/qa/workunits/rbd/test_librbd_python.sh
new file mode 100755
index 00000000..656a5bd1
--- /dev/null
+++ b/qa/workunits/rbd/test_librbd_python.sh
@@ -0,0 +1,12 @@
+#!/bin/sh -ex
+
+relpath=$(dirname $0)/../../../src/test/pybind
+
+if [ -n "${VALGRIND}" ]; then
+  valgrind ${VALGRIND} --suppressions=${TESTDIR}/valgrind.supp \
+    --errors-for-leak-kinds=definite --error-exitcode=1 \
+    nosetests -v $relpath/test_rbd.py
+else
+  nosetests -v $relpath/test_rbd.py
+fi
+exit 0
diff --git a/qa/workunits/rbd/test_lock_fence.sh b/qa/workunits/rbd/test_lock_fence.sh
new file mode 100755
index 00000000..e425e161
--- /dev/null
+++ b/qa/workunits/rbd/test_lock_fence.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# can't use -e because of background process
+set -x
+
+IMAGE=rbdrw-image
+LOCKID=rbdrw
+RELPATH=$(dirname $0)/../../../src/test/librbd
+RBDRW=$RELPATH/rbdrw.py
+
+rbd create $IMAGE --size 10 --image-format 2 --image-shared || exit 1
+
+# rbdrw loops doing I/O to $IMAGE after locking with lockid $LOCKID
+python $RBDRW $IMAGE $LOCKID &
+iochild=$!
+
+# give client time to lock and start reading/writing
+LOCKS='[]'
+while [ "$LOCKS" == '[]' ]
+do
+    LOCKS=$(rbd lock list $IMAGE --format json)
+    sleep 1
+done
+
+clientaddr=$(rbd lock list $IMAGE | tail -1 | awk '{print $NF;}')
+clientid=$(rbd lock list $IMAGE | tail -1 | awk '{print $1;}')
+echo "clientaddr: $clientaddr"
+echo "clientid: $clientid"
+
+ceph osd blacklist add $clientaddr || exit 1
+
+wait $iochild
+rbdrw_exitcode=$?
+if [ $rbdrw_exitcode != 108 ]
+then
+	echo "wrong exitcode from rbdrw: $rbdrw_exitcode"
+	exit 1
+else
+	echo "rbdrw stopped with ESHUTDOWN"
+fi
+
+set -e
+ceph osd blacklist rm $clientaddr
+rbd lock remove $IMAGE $LOCKID "$clientid"
+# rbdrw will have exited with an existing watch, so, until #3527 is fixed,
+# hang out until the watch expires
+sleep 30
+rbd rm $IMAGE
+echo OK
diff --git a/qa/workunits/rbd/test_rbd_mirror.sh b/qa/workunits/rbd/test_rbd_mirror.sh
new file mode 100755
index 00000000..e139dd7e
--- /dev/null
+++ b/qa/workunits/rbd/test_rbd_mirror.sh
@@ -0,0 +1,9 @@
+#!/bin/sh -e
+
+if [ -n "${VALGRIND}" ]; then
+  valgrind ${VALGRIND} --suppressions=${TESTDIR}/valgrind.supp \
+    --error-exitcode=1 ceph_test_rbd_mirror
+else
+  ceph_test_rbd_mirror
+fi
+exit 0
diff --git a/qa/workunits/rbd/test_rbd_tasks.sh b/qa/workunits/rbd/test_rbd_tasks.sh
new file mode 100755
index 00000000..b9663e60
--- /dev/null
+++ b/qa/workunits/rbd/test_rbd_tasks.sh
@@ -0,0 +1,276 @@
+#!/usr/bin/env bash
+set -ex
+
+POOL=rbd_tasks
+POOL_NS=ns1
+
+setup() {
+  trap 'cleanup' INT TERM EXIT
+
+  ceph osd pool create ${POOL} 128
+  rbd pool init ${POOL}
+  rbd namespace create ${POOL}/${POOL_NS}
+
+  TEMPDIR=`mktemp -d`
+}
+
+cleanup() {
+  ceph osd pool rm ${POOL} ${POOL} --yes-i-really-really-mean-it
+
+  rm -rf ${TEMPDIR}
+}
+
+wait_for() {
+  local TEST_FN=$1
+  shift 1
+  local TEST_FN_ARGS=("$@")
+
+  for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
+    sleep ${s}
+
+    ${TEST_FN} "${TEST_FN_ARGS[@]}" || continue
+    return 0
+  done
+  return 1
+}
+
+task_exists() {
+  local TASK_ID=$1
+  [[ -z "${TASK_ID}" ]] && exit 1
+
+  ceph rbd task list ${TASK_ID} || return 1
+  return 0
+}
+
+task_dne() {
+  local TASK_ID=$1
+  [[ -z "${TASK_ID}" ]] && exit 1
+
+  ceph rbd task list ${TASK_ID} || return 0
+  return 1
+}
+
+task_in_progress() {
+  local TASK_ID=$1
+  [[ -z "${TASK_ID}" ]] && exit 1
+
+  [[ $(ceph rbd task list ${TASK_ID} | jq '.in_progress') == 'true' ]]
+}
+
+test_remove() {
+  echo "test_remove"
+
+  local IMAGE=`uuidgen`
+  rbd create --size 1 --image-shared ${POOL}/${IMAGE}
+
+  # MGR might require some time to discover the OSD map w/ new pool
+  wait_for ceph rbd task add remove ${POOL}/${IMAGE}
+}
+
+test_flatten() {
+  echo "test_flatten"
+
+  local PARENT_IMAGE=`uuidgen`
+  local CHILD_IMAGE=`uuidgen`
+
+  rbd create --size 1 --image-shared ${POOL}/${PARENT_IMAGE}
+  rbd snap create ${POOL}/${PARENT_IMAGE}@snap
+  rbd clone ${POOL}/${PARENT_IMAGE}@snap ${POOL}/${POOL_NS}/${CHILD_IMAGE} --rbd-default-clone-format=2
+  [[ "$(rbd info --format json ${POOL}/${POOL_NS}/${CHILD_IMAGE} | jq 'has("parent")')" == "true" ]]
+
+  local TASK_ID=`ceph rbd task add flatten ${POOL}/${POOL_NS}/${CHILD_IMAGE} | jq --raw-output ".id"`
+  wait_for task_dne ${TASK_ID}
+
+  [[ "$(rbd info --format json ${POOL}/${POOL_NS}/${CHILD_IMAGE} | jq 'has("parent")')" == "false" ]]
+}
+
+test_trash_remove() {
+  echo "test_trash_remove"
+
+  local IMAGE=`uuidgen`
+  rbd create --size 1 --image-shared ${POOL}/${IMAGE}
+  local IMAGE_ID=`rbd info --format json ${POOL}/${IMAGE} | jq --raw-output ".id"`
+  rbd trash mv ${POOL}/${IMAGE}
+  [[ -n "$(rbd trash list ${POOL})" ]] || exit 1
+
+  local TASK_ID=`ceph rbd task add trash remove ${POOL}/${IMAGE_ID} | jq --raw-output ".id"`
+  wait_for task_dne ${TASK_ID}
+
+  [[ -z "$(rbd trash list ${POOL})" ]] || exit 1
+}
+
+test_migration_execute() {
+  echo "test_migration_execute"
+
+  local SOURCE_IMAGE=`uuidgen`
+  local TARGET_IMAGE=`uuidgen`
+  rbd create --size 1 --image-shared ${POOL}/${SOURCE_IMAGE}
+  rbd migration prepare ${POOL}/${SOURCE_IMAGE} ${POOL}/${TARGET_IMAGE}
+  [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq --raw-output '.migration.state')" == "prepared" ]]
+
+  local TASK_ID=`ceph rbd task add migration execute ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"`
+  wait_for task_dne ${TASK_ID}
+
+  [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq --raw-output '.migration.state')" == "executed" ]]
+}
+
+test_migration_commit() {
+  echo "test_migration_commit"
+
+  local SOURCE_IMAGE=`uuidgen`
+  local TARGET_IMAGE=`uuidgen`
+  rbd create --size 1 --image-shared ${POOL}/${SOURCE_IMAGE}
+  rbd migration prepare ${POOL}/${SOURCE_IMAGE} ${POOL}/${TARGET_IMAGE}
+  [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq --raw-output '.migration.state')" == "prepared" ]]
+
+  local TASK_ID=`ceph rbd task add migration execute ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"`
+  wait_for task_dne ${TASK_ID}
+
+  TASK_ID=`ceph rbd task add migration commit ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"`
+  wait_for task_dne ${TASK_ID}
+
+  [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq 'has("migration")')" == "false" ]]
+  (rbd info ${POOL}/${SOURCE_IMAGE} && return 1) || true
+  rbd info ${POOL}/${TARGET_IMAGE}
+}
+
+test_migration_abort() {
+  echo "test_migration_abort"
+
+  local SOURCE_IMAGE=`uuidgen`
+  local TARGET_IMAGE=`uuidgen`
+  rbd create --size 1 --image-shared ${POOL}/${SOURCE_IMAGE}
+  rbd migration prepare ${POOL}/${SOURCE_IMAGE} ${POOL}/${TARGET_IMAGE}
+  [[ "$(rbd status --format json ${POOL}/${TARGET_IMAGE} | jq --raw-output '.migration.state')" == "prepared" ]]
+
+  local TASK_ID=`ceph rbd task add migration execute ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"`
+  wait_for task_dne ${TASK_ID}
+
+  TASK_ID=`ceph rbd task add migration abort ${POOL}/${TARGET_IMAGE} | jq --raw-output ".id"`
+  wait_for task_dne ${TASK_ID}
+
+  [[ "$(rbd status --format json ${POOL}/${SOURCE_IMAGE} | jq 'has("migration")')" == "false" ]]
+  rbd info ${POOL}/${SOURCE_IMAGE}
+  (rbd info ${POOL}/${TARGET_IMAGE} && return 1) || true
+}
+
+test_list() {
+  echo "test_list"
+
+  local IMAGE_1=`uuidgen`
+  local IMAGE_2=`uuidgen`
+
+  rbd create --size 1T --image-shared ${POOL}/${IMAGE_1}
+  rbd create --size 1T --image-shared ${POOL}/${IMAGE_2}
+
+  local TASK_ID_1=`ceph rbd task add remove ${POOL}/${IMAGE_1} | jq --raw-output ".id"`
+  local TASK_ID_2=`ceph rbd task add remove ${POOL}/${IMAGE_2} | jq --raw-output ".id"`
+
+  local LIST_FILE="${TEMPDIR}/list_file"
+  ceph rbd task list > ${LIST_FILE}
+  cat ${LIST_FILE}
+
+  [[ $(jq "[.[] | .id] | contains([\"${TASK_ID_1}\", \"${TASK_ID_2}\"])" ${LIST_FILE}) == "true" ]]
+
+  ceph rbd task cancel ${TASK_ID_1}
+  ceph rbd task cancel ${TASK_ID_2}
+}
+
+test_cancel() {
+  echo "test_cancel"
+
+  local IMAGE=`uuidgen`
+  rbd create --size 1T --image-shared ${POOL}/${IMAGE}
+  local TASK_ID=`ceph rbd task add remove ${POOL}/${IMAGE} | jq --raw-output ".id"`
+
+  wait_for task_exists ${TASK_ID}
+
+  ceph rbd task cancel ${TASK_ID}
+  wait_for task_dne ${TASK_ID}
+}
+
+test_duplicate_task() {
+  echo "test_duplicate_task"
+
+  local IMAGE=`uuidgen`
+  rbd create --size 1T --image-shared ${POOL}/${IMAGE}
+  local IMAGE_ID=`rbd info --format json ${POOL}/${IMAGE} | jq --raw-output ".id"`
+  rbd trash mv ${POOL}/${IMAGE}
+
+  local TASK_ID_1=`ceph rbd task add trash remove ${POOL}/${IMAGE_ID} | jq --raw-output ".id"`
+  local TASK_ID_2=`ceph rbd task add trash remove ${POOL}/${IMAGE_ID} | jq --raw-output ".id"`
+
+  [[ "${TASK_ID_1}" == "${TASK_ID_2}" ]]
+
+  ceph rbd task cancel ${TASK_ID_1}
+}
+
+test_duplicate_name() {
+  echo "test_duplicate_name"
+
+  local IMAGE=`uuidgen`
+  rbd create --size 1G --image-shared ${POOL}/${IMAGE}
+  local TASK_ID_1=`ceph rbd task add remove ${POOL}/${IMAGE} | jq --raw-output ".id"`
+
+  wait_for task_dne ${TASK_ID_1}
+
+  rbd create --size 1G --image-shared ${POOL}/${IMAGE}
+  local TASK_ID_2=`ceph rbd task add remove ${POOL}/${IMAGE} | jq --raw-output ".id"`
+
+  [[ "${TASK_ID_1}" != "${TASK_ID_2}" ]]
+  wait_for task_dne ${TASK_ID_2}
+
+  local TASK_ID_3=`ceph rbd task add remove ${POOL}/${IMAGE} | jq --raw-output ".id"`
+
+  [[ "${TASK_ID_2}" == "${TASK_ID_3}" ]]
+}
+
+test_progress() {
+  echo "test_progress"
+
+  local IMAGE_1=`uuidgen`
+  local IMAGE_2=`uuidgen`
+
+  rbd create --size 1 --image-shared ${POOL}/${IMAGE_1}
+  local TASK_ID_1=`ceph rbd task add remove ${POOL}/${IMAGE_1} | jq --raw-output ".id"`
+
+  wait_for task_dne ${TASK_ID_1}
+
+  local PROGRESS_FILE="${TEMPDIR}/progress_file"
+  ceph progress json > ${PROGRESS_FILE}
+  cat ${PROGRESS_FILE}
+
+  [[ $(jq "[.completed | .[].id] | contains([\"${TASK_ID_1}\"])" ${PROGRESS_FILE}) == "true" ]]
+
+  rbd create --size 1T --image-shared ${POOL}/${IMAGE_2}
+  local TASK_ID_2=`ceph rbd task add remove ${POOL}/${IMAGE_2} | jq --raw-output ".id"`
+
+  wait_for task_in_progress ${TASK_ID_2}
+  ceph progress json > ${PROGRESS_FILE}
+  cat ${PROGRESS_FILE}
+
+  [[ $(jq "[.events | .[].id] | contains([\"${TASK_ID_2}\"])" ${PROGRESS_FILE}) == "true" ]]
+
+  ceph rbd task cancel ${TASK_ID_2}
+  wait_for task_dne ${TASK_ID_2}
+
+  ceph progress json > ${PROGRESS_FILE}
+  cat ${PROGRESS_FILE}
+
+  [[ $(jq "[.completed | map(select(.failed)) | .[].id] | contains([\"${TASK_ID_2}\"])" ${PROGRESS_FILE}) == "true" ]]
+}
+
+setup
+test_remove
+test_flatten
+test_trash_remove
+test_migration_execute
+test_migration_commit
+test_migration_abort
+test_list
+test_cancel
+test_duplicate_task
+test_duplicate_name
+test_progress
+
+echo OK
diff --git a/qa/workunits/rbd/test_rbdmap_RBDMAPFILE.sh b/qa/workunits/rbd/test_rbdmap_RBDMAPFILE.sh
new file mode 100755
index 00000000..501c69cd
--- /dev/null
+++ b/qa/workunits/rbd/test_rbdmap_RBDMAPFILE.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# Regression test for http://tracker.ceph.com/issues/14984
+#
+# When the bug is present, starting the rbdmap service causes
+# a bogus log message to be emitted to the log because the RBDMAPFILE
+# environment variable is not set.
+#
+# When the bug is not present, starting the rbdmap service will emit
+# no log messages, because /etc/ceph/rbdmap does not contain any lines
+# that require processing.
+#
+set -ex
+
+echo "TEST: save timestamp for use later with journalctl --since"
+TIMESTAMP=$(date +%Y-%m-%d\ %H:%M:%S)
+
+echo "TEST: assert that rbdmap has not logged anything since boot"
+journalctl -b 0 -t rbdmap | grep 'rbdmap\[[[:digit:]]' && exit 1
+journalctl -b 0 -t init-rbdmap | grep 'rbdmap\[[[:digit:]]' && exit 1
+
+echo "TEST: restart the rbdmap.service"
+sudo systemctl restart rbdmap.service
+
+echo "TEST: ensure that /usr/bin/rbdmap runs to completion"
+until sudo systemctl status rbdmap.service | grep 'active (exited)' ; do
+    sleep 0.5
+done
+
+echo "TEST: assert that rbdmap has not logged anything since TIMESTAMP"
+journalctl --since "$TIMESTAMP" -t rbdmap  | grep 'rbdmap\[[[:digit:]]' && exit 1
+journalctl --since "$TIMESTAMP" -t init-rbdmap | grep 'rbdmap\[[[:digit:]]' && exit 1
+
+exit 0
diff --git a/qa/workunits/rbd/verify_pool.sh b/qa/workunits/rbd/verify_pool.sh
new file mode 100755
index 00000000..08bcca50
--- /dev/null
+++ b/qa/workunits/rbd/verify_pool.sh
@@ -0,0 +1,27 @@
+#!/bin/sh -ex
+
+POOL_NAME=rbd_test_validate_pool
+PG_NUM=32
+
+tear_down () {
+  ceph osd pool delete $POOL_NAME $POOL_NAME --yes-i-really-really-mean-it || true
+}
+
+set_up () {
+  tear_down
+  ceph osd pool create $POOL_NAME $PG_NUM
+  ceph osd pool mksnap $POOL_NAME snap
+  rbd pool init $POOL_NAME
+}
+
+trap tear_down EXIT HUP INT
+set_up
+
+# creating an image in a pool-managed snapshot pool should fail
+rbd create --pool $POOL_NAME --size 1 foo && exit 1 || true
+
+# should succeed if the pool already marked as validated
+printf "overwrite validated" | rados --pool $POOL_NAME put rbd_info -
+rbd create --pool $POOL_NAME --size 1 foo
+
+echo OK
diff --git a/qa/workunits/rename/all.sh b/qa/workunits/rename/all.sh
new file mode 100755
index 00000000..f97ff773
--- /dev/null
+++ b/qa/workunits/rename/all.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -ex
+
+dir=`dirname $0`
+
+CEPH_TOOL='./ceph'
+$CEPH_TOOL || CEPH_TOOL='ceph'
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/prepare.sh
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/pri_nul.sh
+rm ./?/* || true
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/rem_nul.sh
+rm ./?/* || true
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/pri_pri.sh
+rm ./?/* || true
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/rem_pri.sh
+rm ./?/* || true
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/rem_rem.sh
+rm ./?/* || true
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/pri_nul.sh
+rm -r ./?/* || true
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/pri_pri.sh
+rm -r ./?/* || true
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/dir_pri_pri.sh
+rm -r ./?/* || true
+
+CEPH_ARGS=$CEPH_ARGS CEPH_TOOL=$CEPH_TOOL $dir/dir_pri_nul.sh
+rm -r ./?/* || true
+
diff --git a/qa/workunits/rename/dir_pri_nul.sh b/qa/workunits/rename/dir_pri_nul.sh
new file mode 100755
index 00000000..dd8106b6
--- /dev/null
+++ b/qa/workunits/rename/dir_pri_nul.sh
@@ -0,0 +1,28 @@
+#!/bin/sh -ex
+
+# dir: srcdn=destdn
+mkdir ./a/dir1
+mv ./a/dir1 ./a/dir1.renamed
+
+# dir: diff
+mkdir ./a/dir2
+mv ./a/dir2 ./b/dir2
+
+# dir: diff, child subtree on target
+mkdir -p ./a/dir3/child/foo
+$CEPH_TOOL mds tell 0 export_dir /a/dir3/child 1
+sleep 5
+mv ./a/dir3 ./b/dir3
+
+# dir: diff, child subtree on other
+mkdir -p ./a/dir4/child/foo
+$CEPH_TOOL mds tell 0 export_dir /a/dir4/child 2
+sleep 5
+mv ./a/dir4 ./b/dir4
+
+# dir: witness subtree adjustment
+mkdir -p ./a/dir5/1/2/3/4
+$CEPH_TOOL mds tell 0 export_dir /a/dir5/1/2/3 2
+sleep 5
+mv ./a/dir5 ./b
+
diff --git a/qa/workunits/rename/dir_pri_pri.sh b/qa/workunits/rename/dir_pri_pri.sh
new file mode 100755
index 00000000..de235fcd
--- /dev/null
+++ b/qa/workunits/rename/dir_pri_pri.sh
@@ -0,0 +1,11 @@
+#!/bin/sh -ex
+
+# dir, srcdn=destdn
+mkdir ./a/dir1
+mkdir ./a/dir2
+mv -T ./a/dir1 ./a/dir2
+
+# dir, different
+mkdir ./a/dir3
+mkdir ./b/dir4
+mv -T ./a/dir3 ./b/dir4
diff --git a/qa/workunits/rename/plan.txt b/qa/workunits/rename/plan.txt
new file mode 100644
index 00000000..b423b414
--- /dev/null
+++ b/qa/workunits/rename/plan.txt
@@ -0,0 +1,111 @@
+#!/bin/sh
+
+# srcdn         destdn        targeti
+
+## pri auth      null auth     -
+## pri rep       null auth     -
+## rem auth      null auth     -
+## rem rep       null auth     -
+
+#/ pri auth      null rep      - dup of pr_na
+#/ pri rep       null rep      -
+#/ rem auth      null rep      - dup of rr_na
+#/ rem rep       null rep      -
+
+
+## pri auth      pri auth      -
+# pri rep       pri auth      -
+## rem auth      pri auth      -
+# rem rep       pri auth      -
+
+# pri auth      pri rep       -
+# pri rep       pri rep       -
+# rem auth      pri rep       -
+# rem rep       pri rep       -
+
+## pri auth      rem auth      auth
+# pri rep       rem auth      auth
+## rem auth      rem auth      auth
+# rem rep       rem auth      auth
+
+# pri auth      rem rep       auth
+# pri rep       rem rep       auth
+# rem auth      rem rep       auth
+# rem rep       rem rep       auth
+
+# pri auth      rem auth      rep
+# pri rep       rem auth      rep
+# rem auth      rem auth      rep
+# rem rep       rem auth      rep
+
+# pri auth      rem rep       rep
+# pri rep       rem rep       rep
+# rem auth      rem rep       rep
+# rem rep       rem rep       rep
+
+
+types of operations
+
+pri  nul
+  srcdn=destdn
+  diff
+
+rem  nul
+  srci=srcdn=destdn
+  srci=srcdn
+  srcdn=destdn
+  srci=destdn
+  all different
+
+pri  pri
+  srcdn=destdn
+  different
+
+rem  pri
+  srci=srcdn=destdn
+  srci=srcdn
+  srcdn=destdn
+  srci=destdn
+  all different
+
+pri  rem
+  srcdn=destdn=desti
+  srcdn=destdn
+  destdn=desti
+  srcdn=desti
+  all different
+
+rem  rem
+  srci=srcdn=destdn=desti
+  srci=srcdn=destdn
+  srci=srcdn=desti
+  srci=destdn=desti
+  srcdni=destdn=desti
+  srci=srcdn destdn=desti
+  srci=destdn srcdn=desti
+  srci=desti srcdn=destdn
+  srci=srcdn
+  srci=destdn
+  srci=desti
+  srcdn=destdn
+  srcdn=desti
+  destdn=desti
+  all different
+
+
+
+
+
+
+
+
+
+p   n   same
+r   n   same
+p   n   diff
+r   n   diff
+
+p   p   same
+r   p   same
+
+p   r   
diff --git a/qa/workunits/rename/prepare.sh b/qa/workunits/rename/prepare.sh
new file mode 100755
index 00000000..b5ba4ae5
--- /dev/null
+++ b/qa/workunits/rename/prepare.sh
@@ -0,0 +1,21 @@
+#!/bin/sh -ex
+
+$CEPH_TOOL mds tell 0 injectargs '--mds-bal-interval 0'
+$CEPH_TOOL mds tell 1 injectargs '--mds-bal-interval 0'
+$CEPH_TOOL mds tell 2 injectargs '--mds-bal-interval 0'
+$CEPH_TOOL mds tell 3 injectargs '--mds-bal-interval 0'
+#$CEPH_TOOL mds tell 4 injectargs '--mds-bal-interval 0'
+
+mkdir -p ./a/a
+mkdir -p ./b/b
+mkdir -p ./c/c
+mkdir -p ./d/d
+
+mount_dir=`df . | grep -o " /.*" | grep -o "/.*"`
+cur_dir=`pwd`
+ceph_dir=${cur_dir##$mount_dir}
+$CEPH_TOOL mds tell 0 export_dir $ceph_dir/b 1
+$CEPH_TOOL mds tell 0 export_dir $ceph_dir/c 2
+$CEPH_TOOL mds tell 0 export_dir $ceph_dir/d 3
+sleep 5
+
diff --git a/qa/workunits/rename/pri_nul.sh b/qa/workunits/rename/pri_nul.sh
new file mode 100755
index 00000000..c40ec1d2
--- /dev/null
+++ b/qa/workunits/rename/pri_nul.sh
@@ -0,0 +1,11 @@
+#!/bin/sh -ex
+
+# srcdn=destdn
+touch ./a/file1
+mv ./a/file1 ./a/file1.renamed
+
+# different
+touch ./a/file2
+mv ./a/file2 ./b
+
+
diff --git a/qa/workunits/rename/pri_pri.sh b/qa/workunits/rename/pri_pri.sh
new file mode 100755
index 00000000..b74985fe
--- /dev/null
+++ b/qa/workunits/rename/pri_pri.sh
@@ -0,0 +1,12 @@
+#!/bin/sh -ex
+
+# srcdn=destdn
+touch ./a/file1
+touch ./a/file2
+mv ./a/file1 ./a/file2
+
+# different (srcdn != destdn)
+touch ./a/file3
+touch ./b/file4
+mv ./a/file3 ./b/file4
+
diff --git a/qa/workunits/rename/pri_rem.sh b/qa/workunits/rename/pri_rem.sh
new file mode 100755
index 00000000..a1cd03d1
--- /dev/null
+++ b/qa/workunits/rename/pri_rem.sh
@@ -0,0 +1,31 @@
+#!/bin/sh -ex
+
+dotest() {
+    src=$1
+    desti=$2
+    destdn=$3
+    n=$4
+
+    touch ./$src/src$n
+    touch ./$desti/desti$n
+    ln ./$desti/desti$n ./$destdn/destdn$n
+
+    mv ./$src/src$n ./$destdn/destdn$n
+}
+
+
+# srcdn=destdn=desti
+dotest 'a' 'a' 'a' 1
+
+# destdn=desti
+dotest 'b' 'a' 'a' 2
+
+# srcdn=destdn
+dotest 'a' 'b' 'a' 3
+
+# srcdn=desti
+dotest 'a' 'a' 'b' 4
+
+# all different
+dotest 'a' 'b' 'c' 5
+
diff --git a/qa/workunits/rename/rem_nul.sh b/qa/workunits/rename/rem_nul.sh
new file mode 100755
index 00000000..a7103310
--- /dev/null
+++ b/qa/workunits/rename/rem_nul.sh
@@ -0,0 +1,29 @@
+#!/bin/sh -ex
+
+dotest() {
+    srci=$1
+    srcdn=$2
+    dest=$3
+    n=$4
+
+    touch ./$srci/srci$n
+    ln ./$srci/srci$n ./$srcdn/srcdn$n
+
+    mv ./$srcdn/srcdn$n ./$dest/dest$n
+}
+
+# srci=srcdn=destdn
+dotest 'a' 'a' 'a' 1
+
+# srcdn=destdn
+dotest 'b' 'a' 'a' 2
+
+# srci=destdn
+dotest 'a' 'b' 'a' 3
+
+# srci=srcdn
+dotest 'a' 'a' 'b' 4
+
+# all different
+dotest 'a' 'b' 'c' 5
+
diff --git a/qa/workunits/rename/rem_pri.sh b/qa/workunits/rename/rem_pri.sh
new file mode 100755
index 00000000..501ac5e1
--- /dev/null
+++ b/qa/workunits/rename/rem_pri.sh
@@ -0,0 +1,29 @@
+#!/bin/sh -ex
+
+dotest() {
+    srci=$1
+    srcdn=$2
+    dest=$3
+    n=$4
+
+    touch ./$srci/srci$n
+    ln ./$srci/srci$n ./$srcdn/srcdn$n
+    touch ./$dest/dest$n
+
+    mv ./$srcdn/srcdn$n ./$dest/dest$n
+}
+
+# srci=srcdn=destdn
+dotest 'a' 'a' 'a' 1
+
+# srcdn=destdn
+dotest 'b' 'a' 'a' 2
+
+# srci=destdn
+dotest 'a' 'b' 'a' 3
+
+# srci=srcdn
+dotest 'a' 'a' 'b' 4
+
+# all different
+dotest 'a' 'b' 'c' 5
diff --git a/qa/workunits/rename/rem_rem.sh b/qa/workunits/rename/rem_rem.sh
new file mode 100755
index 00000000..80028c51
--- /dev/null
+++ b/qa/workunits/rename/rem_rem.sh
@@ -0,0 +1,61 @@
+#!/bin/sh -ex
+
+dotest() {
+    srci=$1
+    srcdn=$2
+    desti=$3
+    destdn=$4
+    n=$5
+
+    touch ./$srci/srci$n
+    ln ./$srci/srci$n ./$srcdn/srcdn$n
+    touch ./$desti/desti$n
+    ln ./$desti/desti$n ./$destdn/destdn$n
+
+    mv ./$srcdn/srcdn$n ./$destdn/destdn$n
+}
+
+#  srci=srcdn=destdn=desti
+dotest 'a' 'a' 'a' 'a' 1
+
+#  srcdn=destdn=desti
+dotest 'b' 'a' 'a' 'a' 2
+
+#  srci=destdn=desti
+dotest 'a' 'b' 'a' 'a' 3
+
+#  srci=srcdn=destdn
+dotest 'a' 'a' 'b' 'a' 4
+
+#  srci=srcdn=desti
+dotest 'a' 'a' 'a' 'b' 5
+
+#  srci=srcdn destdn=desti
+dotest 'a' 'a' 'b' 'b' 6
+
+#  srci=destdn srcdn=desti
+dotest 'a' 'b' 'b' 'a' 7
+
+#  srci=desti srcdn=destdn
+dotest 'a' 'b' 'a' 'b' 8
+
+#  srci=srcdn
+dotest 'a' 'a' 'b' 'c' 9
+
+#  srci=desti
+dotest 'a' 'b' 'a' 'c' 10
+
+#  srci=destdn
+dotest 'a' 'b' 'c' 'a' 11
+
+#  srcdn=desti
+dotest 'a' 'b' 'b' 'c' 12
+
+#  srcdn=destdn
+dotest 'a' 'b' 'c' 'b' 13
+
+#  destdn=desti
+dotest 'a' 'b' 'c' 'c' 14
+
+#  all different
+dotest 'a' 'b' 'c' 'd' 15
diff --git a/qa/workunits/rest/test-restful.sh b/qa/workunits/rest/test-restful.sh
new file mode 100755
index 00000000..34fb1890
--- /dev/null
+++ b/qa/workunits/rest/test-restful.sh
@@ -0,0 +1,16 @@
+#!/bin/sh -ex
+
+mydir=`dirname $0`
+
+secret=`ceph config-key get mgr/restful/keys/admin`
+active=`ceph mgr dump | jq -r .active_name`
+echo "active $active  admin secret $secret"
+
+prefix="mgr/restful/$active"
+addr=`ceph config-key get $prefix/server_addr || echo 127.0.0.1`
+port=`ceph config-key get $prefix/server_port || echo 8003`
+url="https://$addr:$port"
+echo "prefix $prefix url $url"
+$mydir/test_mgr_rest_api.py $url $secret
+
+echo $0 OK
diff --git a/qa/workunits/rest/test_mgr_rest_api.py b/qa/workunits/rest/test_mgr_rest_api.py
new file mode 100755
index 00000000..fc240934
--- /dev/null
+++ b/qa/workunits/rest/test_mgr_rest_api.py
@@ -0,0 +1,97 @@
+#! /usr/bin/env python
+
+import requests
+import time
+import sys
+import json
+
+# Do not show the stupid message about verify=False.  ignore exceptions bc
+# this doesn't work on some distros.
+try:
+    from requests.packages.urllib3.exceptions import InsecureRequestWarning
+    requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+except:
+    pass
+
+if len(sys.argv) < 3:
+    print("Usage: %s <url> <admin_key>" % sys.argv[0])
+    sys.exit(1)
+
+addr = sys.argv[1]
+auth = ('admin', sys.argv[2])
+headers = {'Content-type': 'application/json'}
+
+request = None
+
+# Create a pool and get its id
+request = requests.post(
+    addr + '/pool?wait=yes',
+    data=json.dumps({'name': 'supertestfriends', 'pg_num': 128}),
+    headers=headers,
+    verify=False,
+    auth=auth)
+print(request.text)
+request = requests.get(addr + '/pool', verify=False, auth=auth)
+assert(request.json()[-1]['pool_name'] == 'supertestfriends')
+pool_id = request.json()[-1]['pool']
+
+# get a mon name
+request = requests.get(addr + '/mon', verify=False, auth=auth)
+firstmon = request.json()[0]['name']
+print('first mon is %s' % firstmon)
+
+# get a server name
+request = requests.get(addr + '/osd', verify=False, auth=auth)
+aserver = request.json()[0]['server']
+print('a server is %s' % aserver)
+
+
+screenplay = [
+    ('get',    '/', {}),
+    ('get',    '/config/cluster', {}),
+    ('get',    '/crush/rule', {}),
+    ('get',    '/doc', {}),
+    ('get',    '/mon', {}),
+    ('get',    '/mon/' + firstmon, {}),
+    ('get',    '/osd', {}),
+    ('get',    '/osd/0', {}),
+    ('get',    '/osd/0/command', {}),
+    ('get',    '/pool/1', {}),
+    ('get',    '/server', {}),
+    ('get',    '/server/' + aserver, {}),
+    ('post',   '/osd/0/command', {'command': 'scrub'}),
+    ('post',   '/pool?wait=1', {'name': 'supertestfriends', 'pg_num': 128}),
+    ('patch',  '/osd/0', {'in': False}),
+    ('patch',  '/config/osd', {'pause': True}),
+    ('get',    '/config/osd', {}),
+    ('patch',  '/pool/' + str(pool_id), {'size': 2}),
+    ('patch',  '/config/osd', {'pause': False}),
+    ('patch',  '/osd/0', {'in': True}),
+    ('get',    '/pool', {}),
+    ('delete', '/pool/' + str(pool_id) + '?wait=1', {}),
+    ('get',    '/request?page=0', {}),
+    ('delete', '/request', {}),
+    ('get',    '/request', {}),
+    ('patch',  '/pool/1', {'pg_num': 128}),
+    ('patch',  '/pool/1', {'pgp_num': 128}),
+    ('get',    '/perf?daemon=.*', {}),
+]
+
+for method, endpoint, args in screenplay:
+    if method == 'sleep':
+        time.sleep(endpoint)
+        continue
+    url = addr + endpoint
+    print("URL = " + url)
+    request = getattr(requests, method)(
+        url,
+        data=json.dumps(args) if args else None,
+        headers=headers,
+        verify=False,
+        auth=auth)
+    print(request.text)
+    if request.status_code != 200 or 'error' in request.json():
+        print('ERROR: %s request for URL "%s" failed' % (method, url))
+        sys.exit(1)
+
+print('OK')
diff --git a/qa/workunits/restart/test-backtraces.py b/qa/workunits/restart/test-backtraces.py
new file mode 100755
index 00000000..86048e6d
--- /dev/null
+++ b/qa/workunits/restart/test-backtraces.py
@@ -0,0 +1,256 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import subprocess
+import json
+import os
+import time
+import sys
+
+if sys.version_info[0] == 2:
+    range = xrange # noqa
+
+elif sys.version_info[0] == 3:
+    range = range
+
+import rados as rados
+import cephfs as cephfs
+
+prefix='testbt'
+
+def get_name(b, i, j):
+    c = '{pre}.{pid}.{i}.{j}'.format(pre=prefix, pid=os.getpid(), i=i, j=j)
+    return c, b + '/' + c
+
+def mkdir(ceph, d):
+    print("mkdir {d}".format(d=d), file=sys.stderr)
+    ceph.mkdir(d, 0o755)
+    return ceph.stat(d)['st_ino']
+
+def create(ceph, f):
+    print("creating {f}".format(f=f), file=sys.stderr)
+    fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644)
+    ceph.close(fd)
+    return ceph.stat(f)['st_ino']
+
+def set_mds_config_param(ceph, param):
+    with open('/dev/null', 'rb') as devnull:
+        confarg = ''
+        if conf != '':
+            confarg = '-c {c}'.format(c=conf)
+        r = subprocess.call("ceph {ca} mds tell a injectargs '{p}'".format(ca=confarg, p=param), shell=True, stdout=devnull)
+        if r != 0:
+            raise Exception
+
+
+class _TrimIndentFile(object):
+    def __init__(self, fp):
+        self.fp = fp
+
+    def readline(self):
+        line = self.fp.readline()
+        return line.lstrip(' \t')
+
+def _optionxform(s):
+    s = s.replace('_', ' ')
+    s = '_'.join(s.split())
+    return s
+
+def conf_set_kill_mds(location, killnum):
+    print('setting mds kill config option for {l}.{k}'.format(l=location, k=killnum), file=sys.stderr)
+    print("restart mds a mds_kill_{l}_at {k}".format(l=location, k=killnum))
+    sys.stdout.flush()
+    for l in sys.stdin.readline():
+        if l == 'restarted':
+            break
+
+def flush(ceph, testnum):
+    print('flushing {t}'.format(t=testnum), file=sys.stderr)
+    set_mds_config_param(ceph, '--mds_log_max_segments 1')
+
+    for i in range(1, 500):
+        f = '{p}.{pid}.{t}.{i}'.format(p=prefix, pid=os.getpid(), t=testnum, i=i)
+        print('flushing with create {f}'.format(f=f), file=sys.stderr)
+        fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644)
+        ceph.close(fd)
+        ceph.unlink(f)
+
+    print('flush doing shutdown', file=sys.stderr)
+    ceph.shutdown()
+    print('flush reinitializing ceph', file=sys.stderr)
+    ceph = cephfs.LibCephFS(conffile=conf)
+    print('flush doing mount', file=sys.stderr)
+    ceph.mount()
+    return ceph
+
+def kill_mds(ceph, location, killnum):
+    print('killing mds: {l}.{k}'.format(l=location, k=killnum), file=sys.stderr)
+    set_mds_config_param(ceph, '--mds_kill_{l}_at {k}'.format(l=location, k=killnum))
+
+def wait_for_mds(ceph):
+    # wait for restart
+    while True:
+        confarg = ''
+        if conf != '':
+            confarg = '-c {c}'.format(c=conf)
+        r = subprocess.check_output("ceph {ca} mds stat".format(ca=confarg), shell=True).decode()
+        if r.find('a=up:active'):
+            break
+        time.sleep(1)
+
+def decode(value):
+
+    tmpfile = '/tmp/{p}.{pid}'.format(p=prefix, pid=os.getpid())
+    with open(tmpfile, 'w+') as f:
+      f.write(value)
+
+    p = subprocess.Popen(
+        [
+            'ceph-dencoder',
+            'import',
+            tmpfile,
+            'type',
+            'inode_backtrace_t',
+            'decode',
+            'dump_json',
+        ],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+      )
+    (stdout, _) = p.communicate(input=value)
+    p.stdin.close()
+    if p.returncode != 0:
+        raise Exception
+    os.remove(tmpfile)
+    return json.loads(stdout)
+
+class VerifyFailure(Exception):
+    pass
+
+def verify(rados_ioctx, ino, values, pool):
+    print('getting parent attr for ino: %lx.00000000' % ino, file=sys.stderr)
+    savede = None
+    for i in range(1, 20):
+        try:
+            savede = None
+            binbt = rados_ioctx.get_xattr('%lx.00000000' % ino, 'parent')
+        except rados.ObjectNotFound as e:
+            # wait for a bit to let segments get flushed out
+            savede = e
+            time.sleep(10)
+    if savede:
+        raise savede
+
+    bt = decode(binbt)
+
+    ind = 0
+    if bt['ino'] != ino:
+        raise VerifyFailure('inode mismatch: {bi} != {ino}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
+                    bi=bt['ancestors'][ind]['dname'], ino=ino, bt=bt, i=ino, v=values))
+    for (n, i) in values:
+        if bt['ancestors'][ind]['dirino'] != i:
+            raise VerifyFailure('ancestor dirino mismatch: {b} != {ind}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
+                    b=bt['ancestors'][ind]['dirino'], ind=i, bt=bt, i=ino, v=values))
+        if bt['ancestors'][ind]['dname'] != n:
+            raise VerifyFailure('ancestor dname mismatch: {b} != {n}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
+                    b=bt['ancestors'][ind]['dname'], n=n, bt=bt, i=ino, v=values))
+        ind += 1
+
+    if bt['pool'] != pool:
+        raise VerifyFailure('pool mismatch: {btp} != {p}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
+                    btp=bt['pool'], p=pool, bt=bt, i=ino, v=values))
+
+def make_abc(ceph, rooti, i):
+    expected_bt = []
+    c, d = get_name("/", i, 0)
+    expected_bt = [(c, rooti)] + expected_bt
+    di = mkdir(ceph, d)
+    c, d = get_name(d, i, 1)
+    expected_bt = [(c, di)] + expected_bt
+    di = mkdir(ceph, d)
+    c, f = get_name(d, i, 2)
+    fi = create(ceph, f)
+    expected_bt = [(c, di)] + expected_bt
+    return fi, expected_bt
+
+test = -1
+if len(sys.argv) > 1:
+    test = int(sys.argv[1])
+
+conf = ''
+if len(sys.argv) > 2:
+    conf = sys.argv[2]
+
+radosobj = rados.Rados(conffile=conf)
+radosobj.connect()
+ioctx = radosobj.open_ioctx('data')
+
+ceph = cephfs.LibCephFS(conffile=conf)
+ceph.mount()
+
+rooti = ceph.stat('/')['st_ino']
+
+test = -1
+if len(sys.argv) > 1:
+    test = int(sys.argv[1])
+
+conf = '/etc/ceph/ceph.conf'
+if len(sys.argv) > 2:
+    conf = sys.argv[2]
+
+# create /a/b/c
+# flush
+# verify
+
+i = 0
+if test < 0 or test == i:
+  print('Running test %d: basic verify' % i, file=sys.stderr)
+  ino, expected_bt = make_abc(ceph, rooti, i)
+  ceph = flush(ceph, i)
+  verify(ioctx, ino, expected_bt, 0)
+
+i += 1
+
+# kill-mds-at-openc-1
+# create /a/b/c
+# restart-mds
+# flush
+# verify
+
+if test < 0 or test == i:
+  print('Running test %d: kill openc' % i, file=sys.stderr)
+  print("restart mds a")
+  sys.stdout.flush()
+  kill_mds(ceph, 'openc', 1)
+  ino, expected_bt = make_abc(ceph, rooti, i)
+  ceph = flush(ceph, i)
+  verify(ioctx, ino, expected_bt, 0)
+
+i += 1
+
+# kill-mds-at-openc-1
+# create /a/b/c
+# restart-mds with kill-mds-at-replay-1
+# restart-mds
+# flush
+# verify
+if test < 0 or test == i:
+  print('Running test %d: kill openc/replay' % i, file=sys.stderr)
+  # these are reversed because we want to prepare the config
+  conf_set_kill_mds('journal_replay', 1)
+  kill_mds(ceph, 'openc', 1)
+  print("restart mds a")
+  sys.stdout.flush()
+  ino, expected_bt = make_abc(ceph, rooti, i)
+  ceph = flush(ceph, i)
+  verify(ioctx, ino, expected_bt, 0)
+
+i += 1
+
+ioctx.close()
+radosobj.shutdown()
+ceph.shutdown()
+
+print("done")
+sys.stdout.flush()
diff --git a/qa/workunits/rgw/run-s3tests.sh b/qa/workunits/rgw/run-s3tests.sh
new file mode 100755
index 00000000..913dd6a0
--- /dev/null
+++ b/qa/workunits/rgw/run-s3tests.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -ex
+
+# run s3-tests from current directory. assume working
+# ceph environment (radosgw-admin in path) and rgw on localhost:8000
+# (the vstart default).
+
+branch=$1
+[ -z "$1" ] && branch=master
+port=$2
+[ -z "$2" ] && port=8000   # this is vstart's default
+
+##
+
+if [ -e CMakeCache.txt ]; then
+    BIN_PATH=$PWD/bin
+elif [ -e $root_path/../build/CMakeCache.txt ]; then
+    cd $root_path/../build
+    BIN_PATH=$PWD/bin
+fi
+PATH=$PATH:$BIN_PATH
+
+dir=tmp.s3-tests.$$
+
+# clone and bootstrap
+mkdir $dir
+cd $dir
+git clone https://github.com/ceph/s3-tests
+cd s3-tests
+git checkout ceph-$branch
+VIRTUALENV_PYTHON=/usr/bin/python2 ./bootstrap
+cd ../..
+
+# users
+akey1=access1
+skey1=secret1
+radosgw-admin user create --uid=s3test1 --display-name='tester1' \
+	      --access-key=$akey1 --secret=$skey1 --email=tester1@ceph.com
+
+akey2=access2
+skey2=secret2
+radosgw-admin user create --uid=s3test2 --display-name='tester2' \
+        --access-key=$akey2 --secret=$skey2 --email=tester2@ceph.com
+
+cat <<EOF > s3.conf
+[DEFAULT]
+## replace with e.g. "localhost" to run against local software
+host = 127.0.0.1
+## uncomment the port to use something other than 80
+port = $port
+## say "no" to disable TLS
+is_secure = no
+[fixtures]
+## all the buckets created will start with this prefix;
+## {random} will be filled with random characters to pad
+## the prefix to 30 characters long, and avoid collisions
+bucket prefix = s3testbucket-{random}-
+[s3 main]
+## the tests assume two accounts are defined, "main" and "alt".
+## user_id is a 64-character hexstring
+user_id = s3test1
+## display name typically looks more like a unix login, "jdoe" etc
+display_name = tester1
+## replace these with your access keys
+access_key = $akey1
+secret_key = $skey1
+email = tester1@ceph.com
+[s3 alt]
+## another user account, used for ACL-related tests
+user_id = s3test2
+display_name = tester2
+## the "alt" user needs to have email set, too
+email = tester2@ceph.com
+access_key = $akey2
+secret_key = $skey2
+EOF
+
+S3TEST_CONF=`pwd`/s3.conf $dir/s3-tests/virtualenv/bin/nosetests -a '!fails_on_rgw' -v 
+
+rm -rf $dir
+
+echo OK.
+
diff --git a/qa/workunits/rgw/s3_bucket_quota.pl b/qa/workunits/rgw/s3_bucket_quota.pl
new file mode 100755
index 00000000..7f5476ef
--- /dev/null
+++ b/qa/workunits/rgw/s3_bucket_quota.pl
@@ -0,0 +1,393 @@
+#! /usr/bin/perl
+
+=head1 NAME
+
+s3_bucket_quota.pl - Script to test the rgw bucket quota functionality using s3 interface. 
+
+=head1 SYNOPSIS
+
+Use:
+        perl s3_bucket_quota.pl [--help]
+
+Examples:
+        perl s3_bucket_quota.pl 
+        or
+        perl s3_bucket_quota.pl  --help
+
+=head1 DESCRIPTION
+
+This script intends to test the rgw bucket quota funcionality using s3 interface 
+and reports the test results
+
+=head1 ARGUMENTS
+
+s3_bucket_quota.pl takes the following arguments:
+   --help
+   (optional) Displays the usage message.
+
+=cut
+
+use Amazon::S3;
+use Data::Dumper;
+#use strict;
+use IO::File;
+use Getopt::Long;
+use Digest::MD5;
+use Pod::Usage();
+use FindBin;
+use lib $FindBin::Bin;
+use s3_utilities;
+use Net::Domain qw(hostfqdn);
+
+my $help;
+
+Getopt::Long::GetOptions(
+    'help' => \$help
+);
+Pod::Usage::pod2usage(-verbose => 1) && exit if ($help);
+
+#== local variables ===
+our $mytestfilename;
+my $mytestfilename1;
+my $logmsg;
+my $kruft;
+my $s3;
+my $hostdom  = $ENV{RGW_FQDN}||hostfqdn();
+my $port     = $ENV{RGW_PORT}||80;
+our $hostname = "$hostdom:$port";
+our $testfileloc;
+my $rgw_user = "qa_user";
+
+# Function that deletes the user $rgw_user and write to logfile. 
+sub delete_user
+{
+    my $cmd = "$radosgw_admin user rm --uid=$rgw_user";
+    my $cmd_op = get_command_output($cmd);
+    if ($cmd_op !~ /aborting/){
+        print "user $rgw_user deleted\n";
+    } else {
+        print "user $rgw_user NOT deleted\n";
+        return 1;
+    }
+    return 0;
+}
+
+sub quota_set_max_size {
+    my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-size=1048576000`; 
+    if ($set_quota !~ /./){
+      print "quota set for the bucket: $bucketname \n";
+    } else {
+      print "quota set failed for the bucket: $bucketname \n";
+      exit 1;
+    }
+    return 0;
+}
+
+sub quota_set_max_size_zero {
+    run_s3($rgw_user);
+    my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-size=0`; 
+    if ($set_quota !~ /./){
+      pass ("quota set for the bucket: $bucketname with max size as zero\n");
+    } else {
+      fail ("quota set with max size 0 failed for the bucket: $bucketname \n");
+    }
+    delete_bucket();
+}
+
+sub quota_set_max_objs_zero {
+    run_s3($rgw_user);
+    my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-objects=0`; 
+    if ($set_quota !~ /./){
+      pass ("quota set for the bucket: $bucketname with max objects as zero\n");
+    } else {
+      fail ("quota set with max objects 0 failed for the bucket: $bucketname \n");
+    }
+    delete_bucket();
+}
+
+sub quota_set_neg_size {
+    run_s3($rgw_user);
+    my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-size=-1`; 
+    if ($set_quota !~ /./){
+      pass ("quota set for the bucket: $bucketname with max size -1\n");
+    } else {
+      fail ("quota set failed for the bucket: $bucketname with max size -1 \n");
+    }
+    delete_bucket();
+}
+
+sub quota_set_neg_objs {
+    run_s3($rgw_user);
+    my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-objects=-1`; 
+    if ($set_quota !~ /./){
+      pass ("quota set for the bucket: $bucketname max objects -1 \n");
+    } else {
+      fail ("quota set failed for the bucket: $bucketname \n with max objects -1");
+    }
+    delete_bucket();
+}
+
+sub quota_set_user_objs {
+    my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=bucket`; 
+    my $set_quota1 = `$radosgw_admin quota set --bucket=$bucketname --max-objects=1`; 
+    if ($set_quota1 !~ /./){
+      print "bucket quota max_objs set for the given user: $bucketname \n";
+    } else {
+      print "bucket quota max_objs set failed for the given user: $bucketname \n";
+      exit 1;
+    }
+    return 0;
+}
+
+sub quota_set_user_size {
+    my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=bucket`; 
+    my $set_quota1 = `$radosgw_admin quota set --bucket=$bucketname --max-size=1048576000`; 
+    if ($set_quota1 !~ /./){
+      print "bucket quota max size set for the given user: $bucketname \n";
+    } else {
+      print "bucket quota max size set failed for the user: $bucketname \n";
+      exit 1;
+    }
+    return 0;
+}
+
+sub quota_set_max_obj {
+    # set max objects 
+    my $set_quota = `$radosgw_admin quota set --bucket=$bucketname --max-objects=1`; 
+    if ($set_quota !~ /./){ 
+      print "quota set for the bucket: $bucketname \n"; 
+    } else {
+      print "quota set failed for the bucket: $bucketname \n"; 
+     exit 1;
+    }
+    return 0;
+}
+
+sub quota_enable {
+    my $en_quota = `$radosgw_admin quota enable --bucket=$bucketname`; 
+    if ($en_quota !~ /./){ 
+      print "quota enabled for the bucket: $bucketname \n"; 
+    } else {
+      print "quota enable failed for the bucket: $bucketname \n"; 
+      exit 1;
+    }
+    return 0;
+}
+
+sub quota_disable {
+    my $dis_quota = `$radosgw_admin quota disable --bucket=$bucketname`; 
+    if ($dis_quota !~ /./){ 
+      print "quota disabled for the bucket: $bucketname \n"; 
+    } else {
+      print "quota disable failed for the bucket: $bucketname \n"; 
+      exit 1;
+    }
+    return 0;
+}
+
+# upload a file to the bucket
+sub upload_file {
+    print "adding file to bucket: $mytestfilename\n";
+    ($bucket->add_key_filename( $mytestfilename, $testfileloc,
+        { content_type => 'text/plain', },
+    ) and (print "upload file successful\n" ) and return 0 ) or (return 1);
+}
+
+# delete the bucket
+sub delete_bucket {
+   #($bucket->delete_key($mytestfilename1) and print "delete keys on bucket succeeded second time\n" ) or die $s3->err . "delete keys on bucket failed second time\n" . $s3->errstr;
+   ($bucket->delete_bucket) and (print "bucket delete succeeded \n") or die $s3->err . "delete bucket failed\n" . $s3->errstr;
+}
+
+# set bucket quota with max_objects and verify 
+sub test_max_objects {
+    my $size = '10Mb';
+    create_file($size);
+    run_s3($rgw_user);
+    quota_set_max_obj();
+    quota_enable();
+    my $ret_value = upload_file();
+    if ($ret_value == 0){
+        pass ( "Test max objects passed" );
+    } else {
+        fail ( "Test max objects failed" );
+    }
+    delete_user();
+    delete_keys($mytestfilename);
+    delete_bucket();
+}
+
+# Set bucket quota for specific user and ensure max objects set for the user is validated
+sub test_max_objects_per_user{
+    my $size = '10Mb';
+    create_file($size);
+    run_s3($rgw_user);
+    quota_set_user_objs();
+    quota_enable();
+    my $ret_value = upload_file();
+    if ($ret_value == 0){
+        pass ( "Test max objects for the given user passed" );
+    } else {
+        fail ( "Test max objects for the given user failed" );
+    }
+    delete_user();
+    delete_keys($mytestfilename);
+    delete_bucket();
+}
+
+# set bucket quota with max_objects and try to exceed the max_objects and verify 
+sub test_beyond_max_objs {
+    my $size = "10Mb";
+    create_file($size);
+    run_s3($rgw_user);
+    quota_set_max_obj();
+    quota_enable();
+    upload_file();
+    my $ret_value = readd_file();
+    if ($ret_value == 1){
+        pass ( "set max objects and test beyond max objects passed" );
+    } else {
+        fail ( "set max objects and test beyond max objects failed" );
+    }
+    delete_user();
+    delete_keys($mytestfilename);
+    delete_bucket();
+}
+
+# set bucket quota for a user with max_objects and try to exceed the max_objects and verify 
+sub test_beyond_max_objs_user {
+    my $size = "10Mb";
+    create_file($size);
+    run_s3($rgw_user);
+    quota_set_user_objs();
+    quota_enable();
+    upload_file();
+    my $ret_value = readd_file();
+    if ($ret_value == 1){
+        pass ( "set max objects for a given user and test beyond max objects passed" );
+    } else {
+        fail ( "set max objects for a given user and test beyond max objects failed" );
+    }
+    delete_user();
+    delete_keys($mytestfilename);
+    delete_bucket();
+}
+
+# set bucket quota for max size and ensure it is validated
+sub test_quota_size {
+    my $ret_value;
+    my $size = "2Gb";
+    create_file($size);
+    run_s3($rgw_user);
+    quota_set_max_size();    
+    quota_enable();
+    my $ret_value = upload_file();
+    if ($ret_value == 1) {
+        pass ( "set max size and ensure that objects upload beyond max size is not entertained" );
+        my $retdel = delete_keys($mytestfilename);
+        if ($retdel == 0) {
+            print "delete objects successful \n";
+            my $size1 = "1Gb";
+            create_file($size1);
+            my $ret_val1 = upload_file(); 
+            if ($ret_val1 == 0) {
+                pass ( "set max size and ensure that the max size is in effect" );
+            } else {
+                fail ( "set max size and ensure the max size takes effect" );
+            }
+        }
+    } else {
+        fail ( "set max size and ensure that objects beyond max size is not allowed" );
+    }
+    delete_user();
+    delete_keys($mytestfilename);
+    delete_bucket();
+}
+
+# set bucket quota for max size for a given user and ensure it is validated
+sub test_quota_size_user {
+    my $ret_value;
+    my $size = "2Gb";
+    create_file($size);
+    run_s3($rgw_user);
+    quota_set_user_size();
+    quota_enable();
+    my $ret_value = upload_file();
+    if ($ret_value == 1) {
+        pass ( "set max size for a given user and ensure that objects upload beyond max size is not entertained" );
+        my $retdel = delete_keys($mytestfilename);
+        if ($retdel == 0) {
+            print "delete objects successful \n";
+            my $size1 = "1Gb";
+            create_file($size1);
+            my $ret_val1 = upload_file();
+            if ($ret_val1 == 0) {
+                pass ( "set max size for a given user and ensure that the max size is in effect" );
+            } else {
+                fail ( "set max size for a given user and ensure the max size takes effect" );
+            }
+        }
+    } else {
+        fail ( "set max size for a given user and ensure that objects beyond max size is not allowed" );
+    }
+    delete_user();
+    delete_keys($mytestfilename);
+    delete_bucket();
+}
+
+# set bucket quota size but disable quota and verify
+sub test_quota_size_disabled {
+    my $ret_value;
+    my $size = "2Gb";
+    create_file($size);
+    run_s3($rgw_user);
+    quota_set_max_size();
+    quota_disable();
+    my $ret_value = upload_file();
+    if ($ret_value == 0) {
+        pass ( "bucket quota size doesnt take effect when quota is disabled" );
+    } else {
+        fail ( "bucket quota size doesnt take effect when quota is disabled" );
+    }
+    delete_user();
+    delete_keys($mytestfilename);
+    delete_bucket();
+}
+
+# set bucket quota size for a given user but disable quota and verify
+sub test_quota_size_disabled_user {
+    my $ret_value;
+    my $size = "2Gb";
+    create_file($size);
+    run_s3($rgw_user);
+    quota_set_user_size();
+    quota_disable();
+    my $ret_value = upload_file();
+    if ($ret_value == 0) {
+        pass ( "bucket quota size for a given user doesnt take effect when quota is disabled" );
+    } else {
+        fail ( "bucket quota size for a given user doesnt take effect when quota is disabled" );
+    }
+    delete_user();
+    delete_keys($mytestfilename);
+    delete_bucket();
+}
+
+# set bucket quota for specified user and verify
+
+#== Main starts here===
+ceph_os_info();
+test_max_objects();
+test_max_objects_per_user();
+test_beyond_max_objs();
+test_beyond_max_objs_user();
+quota_set_max_size_zero();
+quota_set_max_objs_zero();
+quota_set_neg_objs();
+quota_set_neg_size();
+test_quota_size(); 
+test_quota_size_user();
+test_quota_size_disabled();
+test_quota_size_disabled_user();
+
+print "OK";
diff --git a/qa/workunits/rgw/s3_multipart_upload.pl b/qa/workunits/rgw/s3_multipart_upload.pl
new file mode 100755
index 00000000..ab29e6b0
--- /dev/null
+++ b/qa/workunits/rgw/s3_multipart_upload.pl
@@ -0,0 +1,151 @@
+#! /usr/bin/perl
+
+=head1 NAME
+
+s3_multipart_upload.pl - Script to test rgw multipart upload using s3 interface.
+
+=head1 SYNOPSIS
+
+Use:
+        perl s3_multipart_upload.pl [--help]
+
+Examples:
+        perl s3_multipart_upload.pl
+        or
+        perl s3_multipart_upload.pl  --help
+
+=head1 DESCRIPTION
+
+This script intends to test the rgw multipart upload followed by a download
+and verify checksum using s3 interface and reports test results
+
+=head1 ARGUMENTS
+
+s3_multipart_upload.pl takes the following arguments:
+   --help
+   (optional) Displays the usage message.
+
+=cut
+
+use Amazon::S3;
+use Data::Dumper;
+use IO::File;
+use Getopt::Long;
+use Digest::MD5;
+use Pod::Usage();
+use FindBin;
+use lib $FindBin::Bin;
+use s3_utilities;
+use Net::Domain qw(hostfqdn);
+
+my $help;
+
+Getopt::Long::GetOptions(
+    'help' => \$help
+);
+Pod::Usage::pod2usage(-verbose => 1) && exit if ($help);
+
+#== local variables ===
+my $s3;
+my $hostdom  = $ENV{RGW_FQDN}||hostfqdn();
+my $port     = $ENV{RGW_PORT}||80;
+our $hostname = "$hostdom:$port";
+our $testfileloc;
+our $mytestfilename;
+
+# upload a file to the bucket
+sub upload_file {
+    my ($fsize, $i) = @_;
+    create_file($fsize, $i);
+    print "adding file to bucket $bucketname: $mytestfilename\n";
+    ($bucket->add_key_filename( $mytestfilename, $testfileloc,
+        { content_type => 'text/plain', },
+    ) and (print "upload file successful\n" ) and return 0 ) or (print "upload failed\n" and return 1);
+}
+
+# delete the bucket
+sub delete_bucket {
+   ($bucket->delete_bucket) and (print "bucket delete succeeded \n") or die $s3->err . "delete bucket failed\n" . $s3->errstr;
+}
+
+# Function to perform multipart upload of given file size to the user bucket via s3 interface
+sub multipart_upload
+{
+    my ($size, $parts) = @_;
+    # generate random user every time
+    my $user = rand();
+    # Divide the file size in to equal parts and upload to bucket in multiple parts
+    my $fsize = ($size/$parts);
+    my $fsize1;
+    run_s3($user);
+    if ($parts == 10){
+        $fsize1 = '100Mb';
+    } elsif ($parts == 100){
+        $fsize1 = '10Mb';
+    }
+    foreach my $i(1..$parts){
+       print "uploading file - part $i \n";
+       upload_file($fsize1, $i);
+    }
+    fetch_file_from_bucket($fsize1, $parts);
+    compare_cksum($fsize1, $parts);
+    purge_data($user);
+}
+
+# Function to download the files from bucket to verify there is no data corruption
+sub fetch_file_from_bucket
+{
+    # fetch file from the bucket
+    my ($fsize, $parts) = @_;
+    foreach my $i(1..$parts){
+    my $src_file = "$fsize.$i";
+    my $dest_file = "/tmp/downloadfile.$i";
+    print
+      "Downloading $src_file from bucket to $dest_file \n";
+    $response =
+      $bucket->get_key_filename( $src_file, GET,
+        $dest_file )
+      or die $s3->err . ": " . $s3->errstr;
+    }
+}
+
+# Compare the source file with destination file and verify checksum to ensure
+# the files are not corrupted
+sub compare_cksum
+{
+    my ($fsize, $parts)=@_;
+    my $md5    = Digest::MD5->new;
+    my $flag = 0;
+    foreach my $i (1..$parts){
+        my $src_file = "/tmp/"."$fsize".".$i";
+        my $dest_file = "/tmp/downloadfile".".$i";
+        open( FILE, $src_file )
+         or die "Error: Could not open $src_file for MD5 checksum...";
+        open( DLFILE, $dest_file )
+         or die "Error: Could not open $dest_file for MD5 checksum.";
+        binmode(FILE);
+        binmode(DLFILE);
+        my $md5sum   = $md5->addfile(*FILE)->hexdigest;
+        my $md5sumdl = $md5->addfile(*DLFILE)->hexdigest;
+        close FILE;
+        close DLFILE;
+        # compare the checksums
+        if ( $md5sum eq $md5sumdl ) {
+            $flag++;
+        }
+    }
+    if ($flag == $parts){
+       pass("checksum verification for multipart upload passed" );
+    }else{
+       fail("checksum verification for multipart upload failed" );
+    }
+}
+
+#== Main starts here===
+ceph_os_info();
+check();
+# The following test runs multi part upload of file size 1Gb in 10 parts
+multipart_upload('1048576000', 10);
+# The following test runs multipart upload of 1 Gb file in 100 parts
+multipart_upload('1048576000', 100);
+print "OK";
diff --git a/qa/workunits/rgw/s3_user_quota.pl b/qa/workunits/rgw/s3_user_quota.pl
new file mode 100755
index 00000000..6d5c02a9
--- /dev/null
+++ b/qa/workunits/rgw/s3_user_quota.pl
@@ -0,0 +1,191 @@
+#! /usr/bin/perl
+
+=head1 NAME
+
+s3_user_quota.pl - Script to test the rgw user quota functionality using s3 interface.
+
+=head1 SYNOPSIS
+
+Use:
+        perl s3_user_quota.pl [--help]
+
+Examples:
+        perl s3_user_quota.pl
+        or
+        perl s3_user_quota.pl  --help
+
+=head1 DESCRIPTION
+
+This script intends to test the rgw user quota funcionality using s3 interface
+and reports the test results
+
+=head1 ARGUMENTS
+
+s3_user_quota.pl takes the following arguments:
+   --help
+   (optional) Displays the usage message.
+
+=cut
+
+use Amazon::S3;
+use Data::Dumper;
+use IO::File;
+use Getopt::Long;
+use Digest::MD5;
+use Pod::Usage();
+use FindBin;
+use lib $FindBin::Bin;
+use s3_utilities;
+use Net::Domain qw(hostfqdn);
+
+my $help;
+
+Getopt::Long::GetOptions(
+    'help' => \$help
+);
+Pod::Usage::pod2usage(-verbose => 1) && exit if ($help);
+
+#== local variables ===
+our $mytestfilename;
+my $mytestfilename1;
+my $logmsg;
+my $kruft;
+my $s3;
+my $hostdom  = $ENV{RGW_FQDN}||hostfqdn();
+my $port     = $ENV{RGW_PORT}||80;
+our $hostname = "$hostdom:$port";
+our $testfileloc;
+our $cnt;
+
+sub quota_set_max_size_per_user {
+    my ($maxsize, $size1,$rgw_user) = @_;
+    run_s3($rgw_user);
+    my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=user --max-size=$maxsize`;
+    if (($set_quota !~ /./)&&($maxsize == 0)){
+      my $ret = test_max_objs($size1, $rgw_user);
+      if ($ret == 1){
+         pass("quota set for user: $rgw_user with max_size=$maxsize passed" );
+      }else {
+         fail("quota set for user: $rgw_user with max_size=$maxsize failed" );
+      }
+    } elsif (($set_quota !~ /./) && ($maxsize != 0)) {
+      my $ret = test_max_objs($size1, $rgw_user);
+      if ($ret == 0){
+         pass("quota set for user: $rgw_user with max_size=$maxsize passed" );
+      }else {
+         fail("quota set for user: $rgw_user with max_size=$maxsize failed" );
+      }
+    }
+    delete_keys($mytestfilename);
+    purge_data($rgw_user);
+    return 0;
+}
+
+sub max_size_per_user {
+    my ($maxsize, $size1,$rgw_user) = @_;
+    run_s3($rgw_user);
+    my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=user --max-size=$maxsize`;
+    if (($set_quota !~ /./) && ($maxsize != 0)) {
+      my $ret = test_max_objs($size1, $rgw_user);
+      if ($ret == 0){
+         $cnt++;
+      }
+    }
+    return $cnt;
+}
+
+sub quota_set_max_obj_per_user {
+    # set max objects
+    my ($maxobjs, $size1, $rgw_user) = @_;
+    run_s3($rgw_user);
+    my $set_quota = `$radosgw_admin quota set --uid=$rgw_user --quota-scope=user --max-objects=$maxobjs`;
+    if (($set_quota !~ /./) && ($maxobjs == 0)){
+      my $ret = test_max_objs($size1, $rgw_user);
+      if ($ret == 1){
+         pass("quota set for user: $rgw_user with max_objects=$maxobjs passed" );
+      }else {
+         fail("quota set for user: $rgw_user with max_objects=$maxobjs failed" );
+      }
+    } elsif (($set_quota !~ /./) && ($maxobjs == 1)) {
+      my $ret = test_max_objs($size1, $rgw_user);
+      if ($ret == 0){
+         pass("quota set for user: $rgw_user with max_objects=$maxobjs passed" );
+      }else {
+         fail("quota set for user: $rgw_user with max_objects=$maxobjs failed" );
+      }
+    }
+    delete_keys($mytestfilename);
+    purge_data($rgw_user);
+}
+ 
+sub quota_enable_user {
+    my ($rgw_user) = @_;
+    my $en_quota = `$radosgw_admin quota enable --uid=$rgw_user --quota-scope=user`;
+    if ($en_quota !~ /./){
+      print "quota enabled for the user $rgw_user \n";
+    } else {
+      print "quota enable failed for the user $rgw_user \n";
+      exit 1;
+    }
+    return 0;
+}
+
+sub quota_disable_user {
+    my $dis_quota = `$radosgw_admin quota disable --uid=$rgw_user --quota-scope=user`;
+    if ($dis_quota !~ /./){
+      print "quota disabled for the user $rgw_user \n";
+    } else {
+      print "quota disable failed for the user $rgw_user \n";
+      exit 1;
+    }
+    return 0;
+}
+
+# upload a file to the bucket
+sub upload_file {
+    print "adding file to bucket $bucketname: $mytestfilename\n";
+    ($bucket->add_key_filename( $mytestfilename, $testfileloc,
+        { content_type => 'text/plain', },
+    ) and (print "upload file successful\n" ) and return 0 ) or (return 1);
+}
+
+# delete the bucket
+sub delete_bucket {
+   ($bucket->delete_bucket) and (print "bucket delete succeeded \n") or die $s3->err . "delete bucket failed\n" . $s3->errstr;
+}
+
+#Function to upload the given file size to bucket and verify
+sub test_max_objs {
+    my ($size, $rgw_user) = @_;
+    create_file($size);
+    quota_enable_user($rgw_user);
+    my $ret_value = upload_file();
+    return $ret_value;
+}
+
+# set user quota and ensure it is validated
+sub test_user_quota_max_size{
+    my ($max_buckets,$size, $fsize) = @_;
+    my $usr = rand();
+    foreach my $i (1..$max_buckets){
+       my $ret_value = max_size_per_user($size, $fsize, $usr );
+    }
+    if ($ret_value == $max_buckets){
+       fail( "user quota max size for $usr failed on $max_buckets buckets" );
+    } else {
+       pass( "user quota max size for $usr passed on $max_buckets buckets" );
+    }
+    delete_keys($mytestfilename);
+    purge_data($usr);
+}
+
+#== Main starts here===
+ceph_os_info();
+check();
+quota_set_max_obj_per_user('0', '10Mb', 'usr1');
+quota_set_max_obj_per_user('1', '10Mb', 'usr2');
+quota_set_max_size_per_user(0, '10Mb', 'usr1');
+quota_set_max_size_per_user(1048576000, '1Gb', 'usr2');
+test_user_quota_max_size(3,1048576000,'100Mb');
+test_user_quota_max_size(2,1048576000, '1Gb');
+print "OK";
diff --git a/qa/workunits/rgw/s3_utilities.pm b/qa/workunits/rgw/s3_utilities.pm
new file mode 100644
index 00000000..12e6af0a
--- /dev/null
+++ b/qa/workunits/rgw/s3_utilities.pm
@@ -0,0 +1,220 @@
+# Common subroutines shared by the s3 testing code
+my $sec;
+my $min;
+my $hour;
+my $mon;
+my $year;
+my $mday;
+my $wday;
+my $yday;
+my $isdst;
+my $PASS_CNT = 0;
+my $FAIL_CNT = 0;
+
+our $radosgw_admin = $ENV{RGW_ADMIN}||"sudo radosgw-admin";
+
+# function to get the current time stamp from the test set up
+sub get_timestamp {
+   ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
+   if ($mon < 10) { $mon = "0$mon"; }
+   if ($hour < 10) { $hour = "0$hour"; }
+   if ($min < 10) { $min = "0$min"; }
+   if ($sec < 10) { $sec = "0$sec"; }
+   $year=$year+1900;
+   return $year . '_' . $mon . '_' . $mday . '_' . $hour . '_' . $min . '_' . $sec;
+}
+
+# Function to check if radosgw is already running
+sub get_status {
+    my $service = "radosgw";
+    my $cmd = "pgrep $service";
+    my $status = get_cmd_op($cmd);
+    if ($status =~ /\d+/ ){
+        return 0;
+    }
+    return 1;
+}
+
+# function to execute the command and return output
+sub get_cmd_op
+{
+    my $cmd = shift;
+    my $excmd = `$cmd`;
+    return $excmd;
+}
+
+#Function that executes the CLI commands and returns the output of the command
+sub get_command_output {
+    my $cmd_output = shift;
+    open( FH, ">>$test_log" );
+    print FH "\"$cmd_output\"\n";
+    my $exec_cmd = `$cmd_output 2>&1`;
+    print FH "$exec_cmd\n";
+    close(FH);
+    return $exec_cmd;
+}
+
+# Function to get the hostname
+sub get_hostname
+{
+    my $cmd = "hostname";
+    my $get_host = get_command_output($cmd);
+    chomp($get_host);
+    return($get_host);
+}
+
+sub pass {
+    my ($comment) = @_;
+    print "Comment required." unless length $comment;
+    chomp $comment;
+    print_border2();
+    print "Test case: $TC_CNT PASSED - $comment \n";
+    print_border2();
+    $PASS_CNT++;
+}
+
+sub fail {
+    my ($comment) = @_;
+    print "Comment required." unless length $comment;
+    chomp $comment;
+    print_border2();
+    print "Test case: $TC_CNT FAILED - $comment \n";
+    print_border2();
+    $FAIL_CNT++;
+}
+
+sub print_border2 {
+    print "~" x 90 . "\n";
+}
+
+# Function to create the user "qa_user" and extract the user access_key and secret_key of the user
+sub get_user_info
+{
+    my ($rgw_user) = @_;
+    my $cmd = "$radosgw_admin user create --uid=$rgw_user --display-name=$rgw_user";
+    my $cmd_op = get_command_output($cmd);
+    if ($cmd_op !~ /keys/){
+        return (0,0);
+    }
+    my @get_user = (split/\n/,$cmd_op);
+    foreach (@get_user) {
+        if ($_ =~ /access_key/ ){
+            $get_acc_key = $_;
+        } elsif ($_ =~ /secret_key/ ){
+            $get_sec_key = $_;
+        }
+    }
+    my $access_key = $get_acc_key;
+    my $acc_key = (split /:/, $access_key)[1];
+    $acc_key =~ s/\\//g;
+    $acc_key =~ s/ //g;
+    $acc_key =~ s/"//g;
+    $acc_key =~ s/,//g;
+    my $secret_key = $get_sec_key;
+    my $sec_key = (split /:/, $secret_key)[1];
+    $sec_key =~ s/\\//g;
+    $sec_key =~ s/ //g;
+    $sec_key =~ s/"//g;
+    $sec_key =~ s/,//g;
+    return ($acc_key, $sec_key);
+}
+
+# Function that deletes the given user and all associated user data 
+sub purge_data
+{
+    my ($rgw_user) = @_;
+    my $cmd = "$radosgw_admin user rm --uid=$rgw_user --purge-data";
+    my $cmd_op = get_command_output($cmd);
+    if ($cmd_op !~ /./){
+        print "user $rgw_user deleted\n";
+    } else {
+        print "user $rgw_user NOT deleted\n";
+        return 1;
+    }
+    return 0;
+}
+
+# Function to get the Ceph and distro info
+sub ceph_os_info
+{
+        my $ceph_v = get_command_output ( "ceph -v" );
+        my @ceph_arr = split(" ",$ceph_v);
+        $ceph_v = "Ceph Version:   $ceph_arr[2]";
+        my $os_distro = get_command_output ( "lsb_release -d" );
+        my @os_arr = split(":",$os_distro);
+        $os_distro = "Linux Flavor:$os_arr[1]";
+        return ($ceph_v, $os_distro);
+}
+
+# Execute the test case based on the input to the script
+sub create_file {
+    my ($file_size, $part) = @_;
+    my $cnt;
+    $mytestfilename = "$file_size.$part"; 
+    $testfileloc = "/tmp/".$mytestfilename;
+    if ($file_size == '10Mb'){
+        $cnt = 1;
+    } elsif ($file_size == '100Mb'){
+        $cnt = 10;
+    } elsif ($file_size == '500Mb'){
+        $cnt = 50;
+    } elsif ($file_size == '1Gb'){
+        $cnt = 100;
+    } elsif ($file_size == '2Gb'){
+        $cnt = 200;
+    } 
+    my $ret = system("dd if=/dev/zero of=$testfileloc bs=10485760 count=$cnt");
+    if ($ret) { exit 1 };
+    return 0;
+}
+
+sub run_s3
+{
+# Run tests for the S3 functionality
+    # Modify access key and secret key to suit the user account
+    my ($user) = @_;
+    our ( $access_key, $secret_key ) = get_user_info($user);
+    if ( ($access_key) && ($secret_key) ) {
+       $s3 = Amazon::S3->new(
+            {
+                aws_access_key_id     => $access_key,
+                aws_secret_access_key => $secret_key,
+                host                  => $hostname,
+                secure                => 0,
+                retry                 => 1,
+            }
+      );
+    }
+
+our $bucketname = 'buck_'.get_timestamp();
+# create a new bucket (the test bucket)
+our $bucket = $s3->add_bucket( { bucket => $bucketname } )
+      or die $s3->err. "bucket $bucketname create failed\n". $s3->errstr;
+    print "Bucket Created: $bucketname \n";
+    return 0;
+}
+
+# delete keys
+sub delete_keys {
+   (($bucket->delete_key($_[0])) and return 0) or return 1;
+}
+
+# Read the file back to bucket 
+sub readd_file {
+    system("dd if=/dev/zero of=/tmp/10MBfile1 bs=10485760 count=1");
+    $mytestfilename1 = '10MBfile1';
+    print "readding file to bucket: $mytestfilename1\n";
+    ((($bucket->add_key_filename( $mytestfilename1, $testfileloc,
+        { content_type => 'text/plain', },
+    )) and (print "readding file success\n") and return 0) or (return 1));
+}
+
+# check if rgw service is already running
+sub check
+{
+    my $state = get_status();
+    if ($state) {
+        exit 1;
+    }
+}
+1
diff --git a/qa/workunits/rgw/test_rgw_orphan_list.sh b/qa/workunits/rgw/test_rgw_orphan_list.sh
new file mode 100755
index 00000000..4299078a
--- /dev/null
+++ b/qa/workunits/rgw/test_rgw_orphan_list.sh
@@ -0,0 +1,511 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# if defined, debug messages will be displayed and prepended with the string
+# debug="DEBUG"
+
+huge_size=2222 # in megabytes
+big_size=6 # in megabytes
+
+huge_obj=/tmp/huge_obj.temp.$$
+big_obj=/tmp/big_obj.temp.$$
+empty_obj=/tmp/empty_obj.temp.$$
+
+fifo=/tmp/orphan-fifo.$$
+awscli_dir=${HOME}/awscli_temp
+export PATH=${PATH}:${awscli_dir}
+
+rgw_host=$(hostname --fqdn)
+if echo "$rgw_host" | grep -q '\.' ; then
+    :
+else
+    host_domain=".front.sepia.ceph.com"
+    echo "WARNING: rgw hostname -- $rgw_host -- does not appear to be fully qualified; PUNTING and appending $host_domain"
+    rgw_host="${rgw_host}${host_domain}"
+fi
+rgw_port=80
+
+echo "Fully Qualified Domain Name: $rgw_host"
+
+success() {
+    echo OK.
+    exit 0
+}
+
+########################################################################
+# INSTALL AND CONFIGURE TOOLING
+
+install_awscli() {
+    # NB: this does verify authenticity and integrity of downloaded
+    # file; see
+    # https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html
+    here="$(pwd)"
+    cd "$HOME"
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+    unzip awscliv2.zip
+    mkdir -p $awscli_dir
+    ./aws/install -i $awscli_dir
+    cd "$here"
+}
+
+uninstall_awscli() {
+    here="$(pwd)"
+    cd "$HOME"
+    rm -rf $awscli_dir ./aws awscliv2.zip
+    cd "$here"
+}
+
+sudo yum -y install s3cmd
+sudo yum -y install python3-setuptools
+sudo yum -y install python3-pip
+sudo pip3 install --upgrade setuptools
+sudo pip3 install python-swiftclient
+
+# get ready for transition from s3cmd to awscli
+if false ;then
+    install_awscli
+    aws --version
+    uninstall_awscli
+fi
+
+s3config=/tmp/s3config.$$
+
+# do not include the port when it is 80; the host base is used in the
+# v4 signature and it needs to follow this convention for signatures
+# to match
+if [ "$rgw_port" -ne 80 ] ;then
+    s3_host_base="${rgw_host}:${rgw_port}"
+else
+    s3_host_base="$rgw_host"
+fi
+
+cat >${s3config} <<EOF
+[default]
+host_base = $s3_host_base
+access_key = 0555b35654ad1656d804
+secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
+bucket_location = us-east-1
+check_ssl_certificate = True
+check_ssl_hostname = True
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+enable_multipart = True
+encoding = UTF-8
+encrypt = False
+follow_symlinks = False
+force = False
+guess_mime_type = True
+host_bucket = anything.with.three.dots
+multipart_chunk_size_mb = 15
+multipart_max_chunks = 10000
+recursive = False
+recv_chunk = 65536
+send_chunk = 65536
+signature_v2 = False
+socket_timeout = 300
+use_https = False
+use_mime_magic = True
+verbosity = WARNING
+EOF
+
+
+# set up swift authentication
+export ST_AUTH=http://${rgw_host}:${rgw_port}/auth/v1.0
+export ST_USER=test:tester
+export ST_KEY=testing
+
+create_users() {
+    # Create S3 user
+    local akey='0555b35654ad1656d804'
+    local skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
+    radosgw-admin user create --uid testid \
+		  --access-key $akey --secret $skey \
+		  --display-name 'M. Tester' --email tester@ceph.com
+
+    # Create Swift user
+    radosgw-admin user create --subuser=test:tester \
+		  --display-name=Tester-Subuser --key-type=swift \
+		  --secret=testing --access=full
+}
+
+myswift() {
+    if [ -n "$debug" ] ;then
+	echo "${debug}: swift --verbose --debug $@"
+    fi
+    swift --verbose --debug "$@"
+    local code=$?
+    if [ $code -ne 0 ] ;then
+	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
+	exit $code
+    fi
+}
+
+mys3cmd() {
+    if [ -n "$debug" ] ;then
+	echo "${debug}: s3cmd --config=${s3config} --verbose --debug $@"
+    fi
+    s3cmd --config=${s3config} --verbose --debug "$@"
+    local code=$?
+    if [ $code -ne 0 ] ;then
+	echo "ERROR: code = $code ; command = s3cmd --config=${s3config} --verbose --debug "$@""
+	exit $code
+    fi
+}
+
+mys3uploadkill() {
+    if [ $# -ne 5 ] ;then
+	echo "$0: error expecting 5 arguments"
+	exit 1
+    fi
+
+    set -v
+    local_file="$1"
+    remote_bkt="$2"
+    remote_obj="$3"
+    fifo="$4"
+    stop_part="$5"
+    
+    mkfifo $fifo
+
+    s3cmd --config=${s3config} put $local_file \
+	  s3://${remote_bkt}/${remote_obj} \
+	  --progress \
+	  --multipart-chunk-size-mb=5 >$fifo &
+    set +e # don't allow errors to stop script
+    while read line ;do
+        echo "$line" | grep --quiet "part $stop_part "
+        if [ ${PIPESTATUS[1]} -eq 0 ] ;then
+	    kill -9 $(jobs -p)
+            break
+        fi
+    done <$fifo
+    set -e
+
+    rm -f $fifo
+}
+
+mys3upload() {
+    obj=$1
+    bucket=$2
+    dest_obj=$3
+
+    mys3cmd put -q $obj s3://${bucket}/$dest_obj
+}
+
+########################################################################
+# PREP
+
+create_users
+dd if=/dev/urandom of=$big_obj bs=1M count=${big_size}
+dd if=/dev/urandom of=$huge_obj bs=1M count=${huge_size}
+touch $empty_obj
+
+quick_tests() {
+    echo TRY A SWIFT COMMAND
+    myswift upload swift-plain-ctr $big_obj --object-name swift-obj-test
+    myswift list
+    myswift list swift-plain-ctr
+
+    echo TRY A RADOSGW-ADMIN COMMAND
+    radosgw-admin bucket list # make sure rgw is up and running
+}
+
+########################################################################
+# S3 TESTS
+
+####################################
+# regular multipart test
+
+mys3cmd mb s3://multipart-bkt
+mys3upload $huge_obj multipart-bkt multipart-obj
+mys3cmd ls
+mys3cmd ls s3://multipart-bkt
+
+####################################
+# multipart test with incomplete uploads
+
+bkt="incomplete-mp-bkt-1"
+
+mys3cmd mb s3://$bkt
+mys3uploadkill $huge_obj $bkt incomplete-mp-obj-1 $fifo 20
+mys3uploadkill $huge_obj $bkt incomplete-mp-obj-2 $fifo 100
+
+####################################
+# resharded bucket
+
+bkt=resharded-bkt-1
+
+mys3cmd mb s3://$bkt
+
+for f in $(seq 8) ; do
+    dest_obj="reshard-obj-${f}"
+    mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
+done
+
+radosgw-admin bucket reshard --num-shards 3 --bucket=$bkt --yes-i-really-mean-it
+radosgw-admin bucket reshard --num-shards 5 --bucket=$bkt --yes-i-really-mean-it
+
+####################################
+# versioned bucket
+
+if true ;then
+    echo "WARNING: versioned bucket test currently turned off"
+else
+    bkt=versioned-bkt-1
+
+    mys3cmd mb s3://$bkt
+
+    # bucket-enable-versioning $bkt
+
+    for f in $(seq 3) ;do
+	for g in $(seq 10) ;do
+	    dest_obj="versioned-obj-${g}"
+	    mys3cmd put -q $big_obj s3://${bkt}/$dest_obj
+	done
+    done
+
+    for g in $(seq 1 2 10) ;do
+	dest_obj="versioned-obj-${g}"
+	mys3cmd rm s3://${bkt}/$dest_obj
+    done
+fi
+
+############################################################
+# copy small objects
+
+o_bkt="orig-bkt-1"
+d_bkt="copy-bkt-1"
+mys3cmd mb s3://$o_bkt
+
+for f in $(seq 4) ;do
+    dest_obj="orig-obj-$f"
+    mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
+done
+
+mys3cmd mb s3://$d_bkt
+
+mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
+mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
+
+for f in $(seq 5 6) ;do
+    dest_obj="orig-obj-$f"
+    mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
+done
+
+############################################################
+# copy small objects and delete original
+
+o_bkt="orig-bkt-2"
+d_bkt="copy-bkt-2"
+
+mys3cmd mb s3://$o_bkt
+
+for f in $(seq 4) ;do
+    dest_obj="orig-obj-$f"
+    mys3cmd put -q $big_obj s3://${o_bkt}/$dest_obj
+done
+
+mys3cmd mb s3://$d_bkt
+
+mys3cmd cp s3://${o_bkt}/orig-obj-1 s3://${d_bkt}/copied-obj-1
+mys3cmd cp s3://${o_bkt}/orig-obj-3 s3://${d_bkt}/copied-obj-3
+
+for f in $(seq 5 6) ;do
+    dest_obj="orig-obj-$f"
+    mys3cmd put -q $big_obj s3://${d_bkt}/$dest_obj
+done
+
+mys3cmd rb --recursive s3://${o_bkt}
+
+############################################################
+# copy multipart objects
+
+o_bkt="orig-mp-bkt-3"
+d_bkt="copy-mp-bkt-3"
+
+mys3cmd mb s3://$o_bkt
+
+for f in $(seq 2) ;do
+    dest_obj="orig-multipart-obj-$f"
+    mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
+done
+
+mys3cmd mb s3://$d_bkt
+
+mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
+	s3://${d_bkt}/copied-multipart-obj-1
+
+for f in $(seq 5 5) ;do
+    dest_obj="orig-multipart-obj-$f"
+    mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
+done
+
+
+############################################################
+# copy multipart objects and delete original
+
+o_bkt="orig-mp-bkt-4"
+d_bkt="copy-mp-bkt-4"
+
+mys3cmd mb s3://$o_bkt
+
+for f in $(seq 2) ;do
+    dest_obj="orig-multipart-obj-$f"
+    mys3cmd put -q $huge_obj s3://${o_bkt}/$dest_obj
+done
+
+mys3cmd mb s3://$d_bkt
+
+mys3cmd cp s3://${o_bkt}/orig-multipart-obj-1 \
+	s3://${d_bkt}/copied-multipart-obj-1
+
+for f in $(seq 5 5) ;do
+    dest_obj="orig-multipart-obj-$f"
+    mys3cmd put -q $huge_obj s3://${d_bkt}/$dest_obj
+done
+
+mys3cmd rb --recursive s3://$o_bkt
+
+########################################################################
+# SWIFT TESTS
+
+# 600MB
+segment_size=629145600
+
+############################################################
+# plain test
+
+for f in $(seq 4) ;do
+    myswift upload swift-plain-ctr $big_obj --object-name swift-obj-$f
+done
+
+############################################################
+# zero-len test
+
+myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/
+myswift upload swift-zerolen-ctr $big_obj --object-name subdir/abc1
+myswift upload swift-zerolen-ctr $empty_obj --object-name subdir/empty1
+myswift upload swift-zerolen-ctr $big_obj --object-name subdir/xyz1
+
+############################################################
+# dlo test
+
+# upload in 300MB segments
+myswift upload swift-dlo-ctr $huge_obj --object-name dlo-obj-1 \
+      -S $segment_size
+
+############################################################
+# slo test
+
+# upload in 300MB segments
+myswift upload swift-slo-ctr $huge_obj --object-name slo-obj-1 \
+      -S $segment_size --use-slo
+
+############################################################
+# large object copy test
+
+# upload in 300MB segments
+o_ctr=swift-orig-ctr
+o_obj=slo-orig-obj-1
+d_ctr=swift-copy-ctr
+d_obj=slo-copy-obj-1
+myswift upload $o_ctr $big_obj --object-name $o_obj
+
+myswift copy --destination /${d_ctr}/${d_obj} \
+      $o_ctr $o_obj
+
+myswift delete $o_ctr $o_obj
+
+############################################################
+# huge dlo object copy test
+
+o_ctr=swift-orig-dlo-ctr-1
+o_obj=dlo-orig-dlo-obj-1
+d_ctr=swift-copy-dlo-ctr-1
+d_obj=dlo-copy-dlo-obj-1
+
+myswift upload $o_ctr $huge_obj --object-name $o_obj \
+      -S $segment_size
+
+myswift copy --destination /${d_ctr}/${d_obj} \
+      $o_ctr $o_obj
+
+############################################################
+# huge dlo object copy and orig delete
+
+o_ctr=swift-orig-dlo-ctr-2
+o_obj=dlo-orig-dlo-obj-2
+d_ctr=swift-copy-dlo-ctr-2
+d_obj=dlo-copy-dlo-obj-2
+
+myswift upload $o_ctr $huge_obj --object-name $o_obj \
+      -S $segment_size
+
+myswift copy --destination /${d_ctr}/${d_obj} \
+      $o_ctr $o_obj
+
+myswift delete $o_ctr $o_obj
+
+############################################################
+# huge slo object copy test
+
+o_ctr=swift-orig-slo-ctr-1
+o_obj=slo-orig-slo-obj-1
+d_ctr=swift-copy-slo-ctr-1
+d_obj=slo-copy-slo-obj-1
+myswift upload $o_ctr $huge_obj --object-name $o_obj \
+      -S $segment_size --use-slo
+
+myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
+
+############################################################
+# huge slo object copy test and orig delete
+
+o_ctr=swift-orig-slo-ctr-2
+o_obj=slo-orig-slo-obj-2
+d_ctr=swift-copy-slo-ctr-2
+d_obj=slo-copy-slo-obj-2
+myswift upload $o_ctr $huge_obj --object-name $o_obj \
+      -S $segment_size --use-slo
+
+myswift copy --destination /${d_ctr}/${d_obj} $o_ctr $o_obj
+
+myswift delete $o_ctr $o_obj
+
+########################################################################
+# FORCE GARBAGE COLLECTION
+
+sleep 6 # since for testing age at which gc can happen is 5 secs
+radosgw-admin gc process --include-all
+
+
+########################################
+# DO ORPHAN LIST
+
+pool="default.rgw.buckets.data"
+
+rgw-orphan-list $pool
+
+# we only expect there to be one output file, but loop just in case
+ol_error=""
+for f in orphan-list-*.out ; do
+    if [ -s "$f"  ] ;then # if file non-empty
+	ol_error="${ol_error}:$f"
+	echo "One ore more orphans found in $f:"
+	cat "$f"
+    fi
+done
+
+if [ -n "$ol_error" ] ;then
+    echo "ERROR: orphans found when none expected"
+    exit 1
+fi
+
+########################################################################
+# CLEAN UP
+
+rm -f $empty_obj $big_obj $huge_obj $s3config
+
+success
diff --git a/qa/workunits/suites/blogbench.sh b/qa/workunits/suites/blogbench.sh
new file mode 100755
index 00000000..a05d8d21
--- /dev/null
+++ b/qa/workunits/suites/blogbench.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -e
+
+echo "getting blogbench"
+wget http://download.ceph.com/qa/blogbench-1.0.tar.bz2
+#cp /home/gregf/src/blogbench-1.0.tar.bz2 .
+tar -xvf blogbench-1.0.tar.bz2
+cd blogbench-1.0/
+echo "making blogbench"
+./configure
+make
+cd src
+mkdir blogtest_in
+echo "running blogbench"
+./blogbench -d blogtest_in
diff --git a/qa/workunits/suites/bonnie.sh b/qa/workunits/suites/bonnie.sh
new file mode 100755
index 00000000..b60cc6a5
--- /dev/null
+++ b/qa/workunits/suites/bonnie.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+bonnie_bin=`which bonnie++`
+[ $? -eq 1 ] && bonnie_bin=/usr/sbin/bonnie++
+
+uid_flags=""
+[ "`id -u`" == "0" ] && uid_flags="-u root"
+
+$bonnie_bin $uid_flags -n 100
diff --git a/qa/workunits/suites/cephfs_journal_tool_smoke.sh b/qa/workunits/suites/cephfs_journal_tool_smoke.sh
new file mode 100755
index 00000000..3fe01ed6
--- /dev/null
+++ b/qa/workunits/suites/cephfs_journal_tool_smoke.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+export BIN="${BIN:-cephfs-journal-tool --rank=cephfs:0}"
+export JOURNAL_FILE=/tmp/journal.bin
+export JSON_OUTPUT=/tmp/json.tmp
+export BINARY_OUTPUT=/tmp/binary.tmp
+
+if [ -d $BINARY_OUTPUT ] ; then
+    rm -rf $BINARY_OUTPUT
+fi
+
+# Check that the import/export stuff really works as expected
+# first because it's used as the reset method between
+# following checks.
+echo "Testing that export/import cycle preserves state"
+HEADER_STATE=`$BIN header get`
+EVENT_LIST=`$BIN event get list`
+$BIN journal export $JOURNAL_FILE
+$BIN journal import $JOURNAL_FILE
+NEW_HEADER_STATE=`$BIN header get`
+NEW_EVENT_LIST=`$BIN event get list`
+
+if [ ! "$HEADER_STATE" = "$NEW_HEADER_STATE" ] ; then
+    echo "Import failed to preserve header state"
+    echo $HEADER_STATE
+    echo $NEW_HEADER_STATE
+    exit -1
+fi
+
+if [ ! "$EVENT_LIST" = "$NEW_EVENT_LIST" ] ; then
+    echo "Import failed to preserve event state"
+    echo $EVENT_LIST
+    echo $NEW_EVENT_LIST
+    exit -1
+fi
+
+echo "Testing 'journal' commands..."
+
+# Simplest thing: print the vital statistics of the journal
+$BIN journal inspect
+$BIN header get
+
+# Make a copy of the journal in its original state
+$BIN journal export $JOURNAL_FILE
+if [ ! -s $JOURNAL_FILE ] ; then
+    echo "Export to $JOURNAL_FILE failed"
+    exit -1
+fi
+
+# Can we execute a journal reset?
+$BIN journal reset
+$BIN journal inspect
+$BIN header get
+
+echo "Rolling back journal to original state..."
+$BIN journal import $JOURNAL_FILE
+
+echo "Testing 'header' commands..."
+$BIN header get
+$BIN header set write_pos 123
+$BIN header set expire_pos 123
+$BIN header set trimmed_pos 123
+
+echo "Rolling back journal to original state..."
+$BIN journal import $JOURNAL_FILE
+
+echo "Testing 'event' commands..."
+$BIN event get summary
+$BIN event get --type=UPDATE --path=/ --inode=0 --frag=0x100 summary
+$BIN event get json --path $JSON_OUTPUT
+if [ ! -s $JSON_OUTPUT ] ; then
+    echo "Export to $JSON_OUTPUT failed"
+    exit -1
+fi
+$BIN event get binary --path $BINARY_OUTPUT
+if [ ! -s $BINARY_OUTPUT ] ; then
+    echo "Export to $BINARY_OUTPUT failed"
+    exit -1
+fi
+$BIN event recover_dentries summary
+$BIN event splice summary
+
+# Tests finish.
+# Metadata objects have been modified by the 'event recover_dentries' command.
+# Journal is no long consistent with respect to metadata objects (especially inotable).
+# To ensure mds successfully replays its journal, we need to do journal reset.
+$BIN journal reset
+cephfs-table-tool all reset session
+
diff --git a/qa/workunits/suites/dbench-short.sh b/qa/workunits/suites/dbench-short.sh
new file mode 100755
index 00000000..b0da0227
--- /dev/null
+++ b/qa/workunits/suites/dbench-short.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -e
+
+dbench 1
diff --git a/qa/workunits/suites/dbench.sh b/qa/workunits/suites/dbench.sh
new file mode 100755
index 00000000..32c893b4
--- /dev/null
+++ b/qa/workunits/suites/dbench.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+dbench 1
+dbench 10
diff --git a/qa/workunits/suites/ffsb.patch b/qa/workunits/suites/ffsb.patch
new file mode 100644
index 00000000..a5121759
--- /dev/null
+++ b/qa/workunits/suites/ffsb.patch
@@ -0,0 +1,12 @@
+diff -urp 1/parser.c 2/parser.c
+--- 1/parser.c	2008-10-28 04:17:05.000000000 +0800
++++ 2/parser.c	2018-06-26 20:25:59.000000000 +0800
+@@ -203,7 +203,7 @@ static char *get_optstr(char *buf, char 
+ 	len = strnlen(string, BUFSIZE);
+ 	sprintf(search_str, "%s=%%%ds\\n", string, BUFSIZE - len-1);
+ 	if (1 == sscanf(line, search_str, &temp)) {
+-		len = strnlen(temp, 4096);
++		len = strnlen(temp, 4095) + 1;
+ 		ret_buf = malloc(len);
+ 		strncpy(ret_buf, temp, len);
+ 		return ret_buf;
diff --git a/qa/workunits/suites/ffsb.sh b/qa/workunits/suites/ffsb.sh
new file mode 100755
index 00000000..e4451955
--- /dev/null
+++ b/qa/workunits/suites/ffsb.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+mydir=`dirname $0`
+
+wget http://download.ceph.com/qa/ffsb.tar.bz2
+tar jxvf ffsb.tar.bz2
+cd ffsb-6.0-rc2
+patch -p1 < $mydir/ffsb.patch
+./configure
+make
+cd ..
+mkdir tmp
+cd tmp
+
+for f in $mydir/*.ffsb
+do
+    ../ffsb-*/ffsb $f
+done
+cd ..
+rm -r tmp ffsb*
+
diff --git a/qa/workunits/suites/fio.sh b/qa/workunits/suites/fio.sh
new file mode 100755
index 00000000..ee69de81
--- /dev/null
+++ b/qa/workunits/suites/fio.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -x
+
+gen_fio_file() {
+  iter=$1
+  f=$2
+  cat > randio-$$-${iter}.fio <<EOF
+[randio]
+blocksize_range=32m:128m
+blocksize_unaligned=1
+filesize=10G:20G
+readwrite=randrw
+runtime=300
+size=20G
+filename=${f}
+EOF
+}
+
+sudo apt-get -y install fio
+for i in $(seq 1 20); do
+  fcount=$(ls donetestfile* 2>/dev/null | wc -l)
+  donef="foo"
+  fiof="bar"
+  if test ${fcount} -gt 0; then
+     # choose random file
+     r=$[ ${RANDOM} % ${fcount} ]
+     testfiles=( $(ls donetestfile*) )
+     donef=${testfiles[${r}]}
+     fiof=$(echo ${donef} | sed -e "s|done|fio|")
+     gen_fio_file $i ${fiof}
+  else
+     fiof=fiotestfile.$$.$i
+     donef=donetestfile.$$.$i
+     gen_fio_file $i ${fiof}
+  fi
+
+  sudo rm -f ${donef}
+  sudo fio randio-$$-$i.fio
+  sudo ln ${fiof} ${donef}
+  ls -la
+done
diff --git a/qa/workunits/suites/fsstress.sh b/qa/workunits/suites/fsstress.sh
new file mode 100755
index 00000000..e5da5b43
--- /dev/null
+++ b/qa/workunits/suites/fsstress.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -ex
+
+mkdir -p fsstress
+pushd fsstress
+wget -q -O ltp-full.tgz http://download.ceph.com/qa/ltp-full-20091231.tgz
+tar xzf ltp-full.tgz
+pushd ltp-full-20091231/testcases/kernel/fs/fsstress
+make
+BIN=$(readlink -f fsstress)
+popd
+popd
+
+T=$(mktemp -d -p .)
+"$BIN" -d "$T" -l 1 -n 1000 -p 10 -v
+rm -rf -- "$T"
diff --git a/qa/workunits/suites/fsx.sh b/qa/workunits/suites/fsx.sh
new file mode 100755
index 00000000..8a34806c
--- /dev/null
+++ b/qa/workunits/suites/fsx.sh
@@ -0,0 +1,16 @@
+#!/bin/sh -x
+
+set -e
+
+git clone git://git.ceph.com/xfstests.git
+cd xfstests
+git checkout b7fd3f05d6a7a320d13ff507eda2e5b183cae180
+make
+cd ..
+cp xfstests/ltp/fsx .
+
+OPTIONS="-z"  # don't use zero range calls; not supported by cephfs
+
+./fsx $OPTIONS  1MB -N 50000 -p 10000 -l 1048576
+./fsx $OPTIONS  10MB -N 50000 -p 10000 -l 10485760
+./fsx $OPTIONS 100MB -N 50000 -p 10000 -l 104857600
diff --git a/qa/workunits/suites/fsync-tester.sh b/qa/workunits/suites/fsync-tester.sh
new file mode 100755
index 00000000..345fbde8
--- /dev/null
+++ b/qa/workunits/suites/fsync-tester.sh
@@ -0,0 +1,12 @@
+#!/bin/sh -x
+
+set -e
+
+wget http://download.ceph.com/qa/fsync-tester.c
+gcc fsync-tester.c -o fsync-tester
+
+./fsync-tester
+
+echo $PATH
+whereis lsof
+lsof
diff --git a/qa/workunits/suites/iogen.sh b/qa/workunits/suites/iogen.sh
new file mode 100755
index 00000000..d92b8708
--- /dev/null
+++ b/qa/workunits/suites/iogen.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+echo "getting iogen"
+wget http://download.ceph.com/qa/iogen_3.1p0.tar
+tar -xvzf iogen_3.1p0.tar
+cd iogen_3.1p0
+echo "making iogen"
+make
+echo "running iogen"
+./iogen -n 5 -s 2g
+echo "sleep for 10 min"
+sleep 600
+echo "stopping iogen"
+./iogen -k
+
+echo "OK"
diff --git a/qa/workunits/suites/iozone-sync.sh b/qa/workunits/suites/iozone-sync.sh
new file mode 100755
index 00000000..a37962d3
--- /dev/null
+++ b/qa/workunits/suites/iozone-sync.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -e
+
+# basic tests of O_SYNC, O_DSYNC, O_RSYNC
+# test O_SYNC
+iozone -c -e -s 512M -r 1M -t 1 -F osync1 -i 0 -i 1 -o
+# test O_DSYNC
+iozone -c -e -s 512M -r 1M -t 1 -F odsync1 -i 0 -i 1 -+D
+# test O_RSYNC
+iozone -c -e -s 512M -r 1M -t 1 -F orsync1 -i 0 -i 1 -+r
+
+# test same file with O_SYNC in one process, buffered in the other
+# the sync test starts first, so the buffered test should blow
+# past it and 
+iozone -c -e -s 512M -r 1M -t 1 -F osync2 -i 0 -i 1 -o &
+sleep 1
+iozone -c -e -s 512M -r 256K -t 1 -F osync2 -i 0
+wait $!
+
+# test same file with O_SYNC from different threads
+iozone -c -e -s 512M -r 1M -t 2 -F osync3 -i 2 -o
diff --git a/qa/workunits/suites/iozone.sh b/qa/workunits/suites/iozone.sh
new file mode 100755
index 00000000..7dc50cb0
--- /dev/null
+++ b/qa/workunits/suites/iozone.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -ex
+
+iozone -c -e -s 1024M -r 16K -t 1 -F f1 -i 0 -i 1
+iozone -c -e -s 1024M -r 1M -t 1 -F f2 -i 0 -i 1
+iozone -c -e -s 10240M -r 1M -t 1 -F f3 -i 0 -i 1
diff --git a/qa/workunits/suites/pjd.sh b/qa/workunits/suites/pjd.sh
new file mode 100755
index 00000000..bd72f77f
--- /dev/null
+++ b/qa/workunits/suites/pjd.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+wget http://download.ceph.com/qa/pjd-fstest-20090130-RC-aclfixes.tgz
+tar zxvf pjd*.tgz
+cd pjd-fstest-20090130-RC
+make clean
+make
+cd ..
+mkdir tmp
+cd tmp
+# must be root!
+sudo prove -r -v --exec 'bash -x' ../pjd*/tests
+cd ..
+rm -rf tmp pjd*
+
diff --git a/qa/workunits/suites/random_write.32.ffsb b/qa/workunits/suites/random_write.32.ffsb
new file mode 100644
index 00000000..ba83e470
--- /dev/null
+++ b/qa/workunits/suites/random_write.32.ffsb
@@ -0,0 +1,48 @@
+# Large file random writes.
+# 1024 files, 100MB per file.
+
+time=300  # 5 min
+alignio=1
+
+[filesystem0]
+        location=.
+        num_files=128
+        min_filesize=104857600  # 100 MB
+        max_filesize=104857600
+        reuse=1
+[end0]
+
+[threadgroup0]
+        num_threads=32
+
+        write_random=1
+        write_weight=1
+
+        write_size=5242880  # 5 MB
+        write_blocksize=4096
+
+        [stats]
+                enable_stats=1
+                enable_range=1
+
+                msec_range    0.00      0.01
+                msec_range    0.01      0.02
+                msec_range    0.02      0.05
+                msec_range    0.05      0.10
+                msec_range    0.10      0.20
+                msec_range    0.20      0.50
+                msec_range    0.50      1.00
+                msec_range    1.00      2.00
+                msec_range    2.00      5.00
+                msec_range    5.00     10.00
+                msec_range   10.00     20.00
+                msec_range   20.00     50.00
+                msec_range   50.00    100.00
+                msec_range  100.00    200.00
+                msec_range  200.00    500.00
+                msec_range  500.00   1000.00
+                msec_range 1000.00   2000.00
+                msec_range 2000.00   5000.00
+                msec_range 5000.00  10000.00
+        [end]
+[end0]
diff --git a/qa/workunits/suites/wac.sh b/qa/workunits/suites/wac.sh
new file mode 100755
index 00000000..a6114885
--- /dev/null
+++ b/qa/workunits/suites/wac.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -ex
+
+wget http://download.ceph.com/qa/wac.c
+gcc -o wac wac.c
+set +e
+timeout 5m ./wac -l 65536 -n 64 -r wac-test
+RET=$?
+set -e
+[[ $RET -eq 124 ]]
+echo OK
diff --git a/qa/workunits/true.sh b/qa/workunits/true.sh
new file mode 100755
index 00000000..296ef781
--- /dev/null
+++ b/qa/workunits/true.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+true