Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

10 files changed, 2241 insertions, 0 deletions

diff --git a/src/test/cli-integration/balancer/misplaced.t b/src/test/cli-integration/balancer/misplaced.t
new file mode 100644
index 000000000..050cceb64
--- /dev/null
+++ b/src/test/cli-integration/balancer/misplaced.t
@@ -0,0 +1,30 @@
+  $ ceph balancer off
+  $ ceph balancer mode none
+  $ ceph osd pool create balancer_opt 128
+  pool 'balancer_opt' created
+  $ ceph osd pool application enable balancer_opt rados
+  enabled application 'rados' on pool 'balancer_opt'
+  $ rados bench -p balancer_opt 50 write --no-cleanup > /dev/null
+  $ ceph balancer on
+  $ ceph balancer mode crush-compat
+  $ ceph balancer ls
+  []
+  $ ceph config set osd.* target_max_misplaced_ratio .07
+  $ ceph balancer eval
+  current cluster score [0-9]*\.?[0-9]+.* (re)
+# Turn off active balancer to use manual commands
+  $ ceph balancer off
+  $ ceph balancer optimize test_plan balancer_opt
+  $ ceph balancer ls
+  [
+      "test_plan"
+  ]
+  $ ceph balancer execute test_plan
+  $ ceph balancer eval
+  current cluster score [0-9]*\.?[0-9]+.* (re)
+# Plan is gone after execution ?
+  $ ceph balancer execute test_plan
+  Error ENOENT: plan test_plan not found
+  [2]
+  $ ceph osd pool rm balancer_opt balancer_opt --yes-i-really-really-mean-it
+  pool 'balancer_opt' removed
diff --git a/src/test/cli-integration/rbd/defaults.t b/src/test/cli-integration/rbd/defaults.t
new file mode 100644
index 000000000..fe69997ce
--- /dev/null
+++ b/src/test/cli-integration/rbd/defaults.t
@@ -0,0 +1,310 @@
+Plain create with various options specified via usual cli arguments
+===================================================================
+  $ rbd create -s 1 test
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rb.0.*",  (glob)
+      "format": 1, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 1, 
+      "order": 22, 
+      "size": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 --object-size 1M test
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rb.0.*",  (glob)
+      "format": 1, 
+      "name": "test", 
+      "object_size": 1048576, 
+      "objects": 1, 
+      "order": 20, 
+      "size": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1G --object-size 4K test
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rb.0.*",  (glob)
+      "format": 1, 
+      "name": "test", 
+      "object_size": 4096, 
+      "objects": 262144, 
+      "order": 12, 
+      "size": 1073741824
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --image-format 2
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 1, 
+      "order": 22, 
+      "size": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1G test --image-format 2
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 256, 
+      "order": 22, 
+      "size": 1073741824
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --image-format 2 --object-size 1M
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 1048576, 
+      "objects": 1, 
+      "order": 20, 
+      "size": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --image-format 2 --stripe-unit 1048576 --stripe-count 8
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 1, 
+      "order": 22, 
+      "size": 1048576, 
+      "stripe_count": 8, 
+      "stripe_unit": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --image-format 2 --stripe-unit 1048576B --stripe-count 8
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 1, 
+      "order": 22, 
+      "size": 1048576, 
+      "stripe_count": 8, 
+      "stripe_unit": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1G test --image-format 2 --stripe-unit 4K --stripe-count 8
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 256, 
+      "order": 22, 
+      "size": 1073741824, 
+      "stripe_count": 8, 
+      "stripe_unit": 4096
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1G test --image-format 2 --stripe-unit 1M --stripe-count 8
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 256, 
+      "order": 22, 
+      "size": 1073741824, 
+      "stripe_count": 8, 
+      "stripe_unit": 1048576
+  }
+  $ rbd rm test --no-progress
+
+Format 2 Usual arguments with custom rbd_default_* params
+=========================================================
+  $ rbd create -s 1 test --image-format 2 --stripe-unit 1048576 --stripe-count 8 --rbd-default-order 21
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 2097152, 
+      "objects": 1, 
+      "order": 21, 
+      "size": 1048576, 
+      "stripe_count": 8, 
+      "stripe_unit": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --image-format 2 --stripe-unit 1048576 --stripe-count 8 --object-size 8M --rbd-default-order 20
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 8388608, 
+      "objects": 1, 
+      "order": 23, 
+      "size": 1048576, 
+      "stripe_count": 8, 
+      "stripe_unit": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --image-format 2 --rbd-default-stripe-unit 1048576 --rbd-default-stripe-count 8
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 1, 
+      "order": 22, 
+      "size": 1048576, 
+      "stripe_count": 8, 
+      "stripe_unit": 1048576
+  }
+  $ rbd rm test --no-progress
+
+Format 1 Usual arguments with custom rbd_default_* params
+=========================================================
+  $ rbd create -s 1 test --rbd-default-order 20
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rb.0.*",  (glob)
+      "format": 1, 
+      "name": "test", 
+      "object_size": 1048576, 
+      "objects": 1, 
+      "order": 20, 
+      "size": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --rbd-default-format 2
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 1, 
+      "order": 22, 
+      "size": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --rbd-default-format 2 --rbd-default-order 20
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 1048576, 
+      "objects": 1, 
+      "order": 20, 
+      "size": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --rbd-default-format 2 --rbd-default-order 20 --rbd-default-features 1
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 1048576, 
+      "objects": 1, 
+      "order": 20, 
+      "size": 1048576
+  }
+  $ rbd rm test --no-progress
+  $ rbd create -s 1 test --rbd-default-format 2 --stripe-unit 1048576 --stripe-count 8
+  $ rbd info test --format json | python3 -mjson.tool | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "features": [
+          "layering", 
+          "striping", 
+          "exclusive"
+      ], 
+      "format": 2, 
+      "name": "test", 
+      "object_size": 4194304, 
+      "objects": 1, 
+      "order": 22, 
+      "size": 1048576, 
+      "stripe_count": 8, 
+      "stripe_unit": 1048576
+  }
+  $ rbd rm test --no-progress
diff --git a/src/test/cli-integration/rbd/formatted-output.t b/src/test/cli-integration/rbd/formatted-output.t
new file mode 100644
index 000000000..778cc2198
--- /dev/null
+++ b/src/test/cli-integration/rbd/formatted-output.t
@@ -0,0 +1,1171 @@
+ls on empty pool never containing images
+========================================
+  $ ceph osd pool create rbd_other
+  pool 'rbd_other' created
+  $ rbd pool init rbd_other
+  $ rados -p rbd rm rbd_directory >/dev/null 2>&1 || true
+  $ rbd ls
+  $ rbd ls --format json
+  []
+  $ rbd ls --format xml
+  <images></images>
+
+create
+=======
+  $ RBD_FORCE_ALLOW_V1=1 rbd create -s 1024 --image-format 1 foo --log-to-stderr=false
+  rbd: image format 1 is deprecated
+  $ rbd create -s 512 --image-format 2 bar
+  $ rbd create -s 2048 --image-format 2 --image-feature layering baz
+  $ RBD_FORCE_ALLOW_V1=1 rbd create -s 1 --image-format 1 quux --log-to-stderr=false
+  rbd: image format 1 is deprecated
+  $ rbd create -s 1G --image-format 2 quuy
+
+snapshot
+========
+  $ rbd snap create bar@snap --no-progress
+  $ rbd resize -s 1024 --no-progress bar
+  $ rbd resize -s 2G --no-progress quuy
+  $ rbd snap create bar@snap2 --no-progress
+  $ rbd snap create foo@snap --no-progress
+
+clone
+=====
+  $ rbd snap protect bar@snap
+  $ rbd clone --image-feature layering,exclusive-lock,object-map,fast-diff bar@snap rbd_other/child
+  $ rbd snap create rbd_other/child@snap --no-progress
+  $ rbd flatten rbd_other/child 2> /dev/null
+  $ rbd bench rbd_other/child --io-type write --io-pattern seq --io-total 1B > /dev/null 2>&1
+  $ rbd clone bar@snap rbd_other/deep-flatten-child
+  $ rbd snap create rbd_other/deep-flatten-child@snap --no-progress
+  $ rbd flatten rbd_other/deep-flatten-child 2> /dev/null
+
+lock
+====
+  $ rbd lock add quux id
+  $ rbd lock add baz id1 --shared tag
+  $ rbd lock add baz id2 --shared tag
+  $ rbd lock add baz id3 --shared tag
+
+test formatting
+===============
+  $ rbd children foo@snap
+  $ rbd children bar@snap
+  rbd_other/child
+  $ rbd children bar@snap2
+TODO: figure out why .* does not match the block_name_prefix line in rbd info.
+For now, use a more inclusive regex.
+  $ rbd info foo
+  rbd image 'foo':
+  \tsize 1 GiB in 256 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 1 (esc)
+  [^^]+ (re)
+  \tformat: 1 (esc)
+  $ rbd info foo --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rb.0.*",  (glob)
+      "format": 1, 
+      "id": "", 
+      "name": "foo", 
+      "object_size": 4194304, 
+      "objects": 256, 
+      "order": 22, 
+      "size": 1073741824, 
+      "snapshot_count": 1
+  }
+  $ rbd info foo --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>foo</name>
+    <id/>
+    <size>1073741824</size>
+    <objects>256</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>1</snapshot_count>
+    <block_name_prefix>rb.0.*</block_name_prefix> (glob)
+    <format>1</format>
+  </image>
+  $ rbd info foo@snap
+  rbd image 'foo':
+  \tsize 1 GiB in 256 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 1 (esc)
+  [^^]+ (re)
+  \tformat: 1 (esc)
+  \tprotected: False (esc)
+  $ rbd info foo@snap --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rb.0.*",  (glob)
+      "format": 1, 
+      "id": "", 
+      "name": "foo", 
+      "object_size": 4194304, 
+      "objects": 256, 
+      "order": 22, 
+      "protected": "false", 
+      "size": 1073741824, 
+      "snapshot_count": 1
+  }
+  $ rbd info foo@snap --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>foo</name>
+    <id/>
+    <size>1073741824</size>
+    <objects>256</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>1</snapshot_count>
+    <block_name_prefix>rb.0*</block_name_prefix> (glob)
+    <format>1</format>
+    <protected>false</protected>
+  </image>
+  $ rbd info bar
+  rbd image 'bar':
+  \tsize 1 GiB in 256 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 2 (esc)
+  \tid:* (glob)
+  [^^]+ (re)
+  \tformat: 2 (esc)
+  \tfeatures: layering, exclusive-lock, object-map, fast-diff, deep-flatten (esc)
+  \top_features:  (esc)
+  \tflags:  (esc)
+  \tcreate_timestamp:* (glob)
+  \taccess_timestamp:* (glob)
+  \tmodify_timestamp:* (glob)
+  $ rbd info bar --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "access_timestamp": "*",  (glob)
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "create_timestamp": "*",  (glob)
+      "features": [
+          "layering", 
+          "exclusive-lock", 
+          "object-map", 
+          "fast-diff", 
+          "deep-flatten"
+      ], 
+      "flags": [], 
+      "format": 2, 
+      "id": "*",  (glob)
+      "modify_timestamp": "*",  (glob)
+      "name": "bar", 
+      "object_size": 4194304, 
+      "objects": 256, 
+      "op_features": [], 
+      "order": 22, 
+      "size": 1073741824, 
+      "snapshot_count": 2
+  }
+  $ rbd info bar --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>bar</name>
+    <id>*</id> (glob)
+    <size>1073741824</size>
+    <objects>256</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>2</snapshot_count>
+    <block_name_prefix>rbd_data.*</block_name_prefix> (glob)
+    <format>2</format>
+    <features>
+      <feature>layering</feature>
+      <feature>exclusive-lock</feature>
+      <feature>object-map</feature>
+      <feature>fast-diff</feature>
+      <feature>deep-flatten</feature>
+    </features>
+    <op_features/>
+    <flags/>
+    <create_timestamp>*</create_timestamp> (glob)
+    <access_timestamp>*</access_timestamp> (glob)
+    <modify_timestamp>*</modify_timestamp> (glob)
+  </image>
+  $ rbd info bar@snap
+  rbd image 'bar':
+  \tsize 512 MiB in 128 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 2 (esc)
+  \tid:* (glob)
+  [^^]+ (re)
+  \tformat: 2 (esc)
+  \tfeatures: layering, exclusive-lock, object-map, fast-diff, deep-flatten (esc)
+  \top_features:  (esc)
+  \tflags:  (esc)
+  \tcreate_timestamp:* (glob)
+  \taccess_timestamp:* (glob)
+  \tmodify_timestamp:* (glob)
+  \tprotected: True (esc)
+  $ rbd info bar@snap --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "access_timestamp": "*",  (glob)
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "create_timestamp": "*",  (glob)
+      "features": [
+          "layering", 
+          "exclusive-lock", 
+          "object-map", 
+          "fast-diff", 
+          "deep-flatten"
+      ], 
+      "flags": [], 
+      "format": 2, 
+      "id": "*",  (glob)
+      "modify_timestamp": "*",  (glob)
+      "name": "bar", 
+      "object_size": 4194304, 
+      "objects": 128, 
+      "op_features": [], 
+      "order": 22, 
+      "protected": "true", 
+      "size": 536870912, 
+      "snapshot_count": 2
+  }
+  $ rbd info bar@snap --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>bar</name>
+    <id>*</id> (glob)
+    <size>536870912</size>
+    <objects>128</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>2</snapshot_count>
+    <block_name_prefix>rbd_data.*</block_name_prefix> (glob)
+    <format>2</format>
+    <features>
+      <feature>layering</feature>
+      <feature>exclusive-lock</feature>
+      <feature>object-map</feature>
+      <feature>fast-diff</feature>
+      <feature>deep-flatten</feature>
+    </features>
+    <op_features/>
+    <flags/>
+    <create_timestamp>*</create_timestamp> (glob)
+    <access_timestamp>*</access_timestamp> (glob)
+    <modify_timestamp>*</modify_timestamp> (glob)
+    <protected>true</protected>
+  </image>
+  $ rbd info bar@snap2
+  rbd image 'bar':
+  \tsize 1 GiB in 256 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 2 (esc)
+  \tid:* (glob)
+  [^^]+ (re)
+  \tformat: 2 (esc)
+  \tfeatures: layering, exclusive-lock, object-map, fast-diff, deep-flatten (esc)
+  \top_features:  (esc)
+  \tflags:  (esc)
+  \tcreate_timestamp:* (glob)
+  \taccess_timestamp:* (glob)
+  \tmodify_timestamp:* (glob)
+  \tprotected: False (esc)
+  $ rbd info bar@snap2 --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "access_timestamp": "*",  (glob)
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "create_timestamp": "*",  (glob)
+      "features": [
+          "layering", 
+          "exclusive-lock", 
+          "object-map", 
+          "fast-diff", 
+          "deep-flatten"
+      ], 
+      "flags": [], 
+      "format": 2, 
+      "id": "*",  (glob)
+      "modify_timestamp": "*",  (glob)
+      "name": "bar", 
+      "object_size": 4194304, 
+      "objects": 256, 
+      "op_features": [], 
+      "order": 22, 
+      "protected": "false", 
+      "size": 1073741824, 
+      "snapshot_count": 2
+  }
+  $ rbd info bar@snap2 --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>bar</name>
+    <id>*</id> (glob)
+    <size>1073741824</size>
+    <objects>256</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>2</snapshot_count>
+    <block_name_prefix>rbd_data.*</block_name_prefix> (glob)
+    <format>2</format>
+    <features>
+      <feature>layering</feature>
+      <feature>exclusive-lock</feature>
+      <feature>object-map</feature>
+      <feature>fast-diff</feature>
+      <feature>deep-flatten</feature>
+    </features>
+    <op_features/>
+    <flags/>
+    <create_timestamp>*</create_timestamp> (glob)
+    <access_timestamp>*</access_timestamp> (glob)
+    <modify_timestamp>*</modify_timestamp> (glob)
+    <protected>false</protected>
+  </image>
+  $ rbd info baz
+  rbd image 'baz':
+  \tsize 2 GiB in 512 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 0 (esc)
+  \tid:* (glob)
+  [^^]+ (re)
+  \tformat: 2 (esc)
+  \tfeatures: layering (esc)
+  \top_features:  (esc)
+  \tflags:  (esc)
+  \tcreate_timestamp:* (glob)
+  \taccess_timestamp:* (glob)
+  \tmodify_timestamp:* (glob)
+  $ rbd info baz --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "access_timestamp": "*",  (glob)
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "create_timestamp": "*",  (glob)
+      "features": [
+          "layering"
+      ], 
+      "flags": [], 
+      "format": 2, 
+      "id": "*",  (glob)
+      "modify_timestamp": "*",  (glob)
+      "name": "baz", 
+      "object_size": 4194304, 
+      "objects": 512, 
+      "op_features": [], 
+      "order": 22, 
+      "size": 2147483648, 
+      "snapshot_count": 0
+  }
+  $ rbd info baz --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>baz</name>
+    <id>*</id> (glob)
+    <size>2147483648</size>
+    <objects>512</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>0</snapshot_count>
+    <block_name_prefix>rbd_data.*</block_name_prefix> (glob)
+    <format>2</format>
+    <features>
+      <feature>layering</feature>
+    </features>
+    <op_features/>
+    <flags/>
+    <create_timestamp>*</create_timestamp> (glob)
+    <access_timestamp>*</access_timestamp> (glob)
+    <modify_timestamp>*</modify_timestamp> (glob)
+  </image>
+  $ rbd info quux
+  rbd image 'quux':
+  \tsize 1 MiB in 1 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 0 (esc)
+  [^^]+ (re)
+  \tformat: 1 (esc)
+  $ rbd info quux --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "block_name_prefix": "rb.0.*",  (glob)
+      "format": 1, 
+      "id": "", 
+      "name": "quux", 
+      "object_size": 4194304, 
+      "objects": 1, 
+      "order": 22, 
+      "size": 1048576, 
+      "snapshot_count": 0
+  }
+  $ rbd info quux --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>quux</name>
+    <id/>
+    <size>1048576</size>
+    <objects>1</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>0</snapshot_count>
+    <block_name_prefix>rb.0.*</block_name_prefix> (glob)
+    <format>1</format>
+  </image>
+  $ rbd info rbd_other/child
+  rbd image 'child':
+  \tsize 512 MiB in 128 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 1 (esc)
+  \tid:* (glob)
+  [^^]+ (re)
+  \tformat: 2 (esc)
+  \tfeatures: layering, exclusive-lock, object-map, fast-diff (esc)
+  \top_features:  (esc)
+  \tflags:  (esc)
+  \tcreate_timestamp:* (glob)
+  \taccess_timestamp:* (glob)
+  \tmodify_timestamp:* (glob)
+  $ rbd info rbd_other/child --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "access_timestamp": "*",  (glob)
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "create_timestamp": "*",  (glob)
+      "features": [
+          "layering", 
+          "exclusive-lock", 
+          "object-map", 
+          "fast-diff"
+      ], 
+      "flags": [], 
+      "format": 2, 
+      "id": "*",  (glob)
+      "modify_timestamp": "*",  (glob)
+      "name": "child", 
+      "object_size": 4194304, 
+      "objects": 128, 
+      "op_features": [], 
+      "order": 22, 
+      "size": 536870912, 
+      "snapshot_count": 1
+  }
+  $ rbd info rbd_other/child --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>child</name>
+    <id>*</id> (glob)
+    <size>536870912</size>
+    <objects>128</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>1</snapshot_count>
+    <block_name_prefix>rbd_data.*</block_name_prefix> (glob)
+    <format>2</format>
+    <features>
+      <feature>layering</feature>
+      <feature>exclusive-lock</feature>
+      <feature>object-map</feature>
+      <feature>fast-diff</feature>
+    </features>
+    <op_features/>
+    <flags/>
+    <create_timestamp>*</create_timestamp> (glob)
+    <access_timestamp>*</access_timestamp> (glob)
+    <modify_timestamp>*</modify_timestamp> (glob)
+  </image>
+  $ rbd info rbd_other/child@snap
+  rbd image 'child':
+  \tsize 512 MiB in 128 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 1 (esc)
+  \tid:* (glob)
+  [^^]+ (re)
+  \tformat: 2 (esc)
+  \tfeatures: layering, exclusive-lock, object-map, fast-diff (esc)
+  \top_features:  (esc)
+  \tflags:  (esc)
+  \tcreate_timestamp:* (glob)
+  \taccess_timestamp:* (glob)
+  \tmodify_timestamp:* (glob)
+  \tprotected: False (esc)
+  \tparent: rbd/bar@snap (esc)
+  \toverlap: 512 MiB (esc)
+  $ rbd info rbd_other/child@snap --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "access_timestamp": "*",  (glob)
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "create_timestamp": "*",  (glob)
+      "features": [
+          "layering", 
+          "exclusive-lock", 
+          "object-map", 
+          "fast-diff"
+      ], 
+      "flags": [], 
+      "format": 2, 
+      "id": "*",  (glob)
+      "modify_timestamp": "*",  (glob)
+      "name": "child", 
+      "object_size": 4194304, 
+      "objects": 128, 
+      "op_features": [], 
+      "order": 22, 
+      "parent": {
+          "id": "*",  (glob)
+          "image": "bar", 
+          "overlap": 536870912, 
+          "pool": "rbd", 
+          "pool_namespace": "", 
+          "snapshot": "snap", 
+          "trash": false
+      }, 
+      "protected": "false", 
+      "size": 536870912, 
+      "snapshot_count": 1
+  }
+  $ rbd info rbd_other/child@snap --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>child</name>
+    <id>*</id> (glob)
+    <size>536870912</size>
+    <objects>128</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>1</snapshot_count>
+    <block_name_prefix>rbd_data.*</block_name_prefix> (glob)
+    <format>2</format>
+    <features>
+      <feature>layering</feature>
+      <feature>exclusive-lock</feature>
+      <feature>object-map</feature>
+      <feature>fast-diff</feature>
+    </features>
+    <op_features/>
+    <flags/>
+    <create_timestamp>*</create_timestamp> (glob)
+    <access_timestamp>*</access_timestamp> (glob)
+    <modify_timestamp>*</modify_timestamp> (glob)
+    <protected>false</protected>
+    <parent>
+      <pool>rbd</pool>
+      <pool_namespace/>
+      <image>bar</image>
+      <id>*</id> (glob)
+      <snapshot>snap</snapshot>
+      <trash>false</trash>
+      <overlap>536870912</overlap>
+    </parent>
+  </image>
+  $ rbd info rbd_other/deep-flatten-child
+  rbd image 'deep-flatten-child':
+  \tsize 512 MiB in 128 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 1 (esc)
+  \tid:* (glob)
+  [^^]+ (re)
+  \tformat: 2 (esc)
+  \tfeatures: layering, exclusive-lock, object-map, fast-diff, deep-flatten (esc)
+  \top_features:  (esc)
+  \tflags:  (esc)
+  \tcreate_timestamp:* (glob)
+  \taccess_timestamp:* (glob)
+  \tmodify_timestamp:* (glob)
+  $ rbd info rbd_other/deep-flatten-child --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "access_timestamp": "*",  (glob)
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "create_timestamp": "*",  (glob)
+      "features": [
+          "layering", 
+          "exclusive-lock", 
+          "object-map", 
+          "fast-diff", 
+          "deep-flatten"
+      ], 
+      "flags": [], 
+      "format": 2, 
+      "id": "*",  (glob)
+      "modify_timestamp": "*",  (glob)
+      "name": "deep-flatten-child", 
+      "object_size": 4194304, 
+      "objects": 128, 
+      "op_features": [], 
+      "order": 22, 
+      "size": 536870912, 
+      "snapshot_count": 1
+  }
+  $ rbd info rbd_other/deep-flatten-child --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>deep-flatten-child</name>
+    <id>*</id> (glob)
+    <size>536870912</size>
+    <objects>128</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>1</snapshot_count>
+    <block_name_prefix>rbd_data.*</block_name_prefix> (glob)
+    <format>2</format>
+    <features>
+      <feature>layering</feature>
+      <feature>exclusive-lock</feature>
+      <feature>object-map</feature>
+      <feature>fast-diff</feature>
+      <feature>deep-flatten</feature>
+    </features>
+    <op_features/>
+    <flags/>
+    <create_timestamp>*</create_timestamp> (glob)
+    <access_timestamp>*</access_timestamp> (glob)
+    <modify_timestamp>*</modify_timestamp> (glob)
+  </image>
+  $ rbd info rbd_other/deep-flatten-child@snap
+  rbd image 'deep-flatten-child':
+  \tsize 512 MiB in 128 objects (esc)
+  \torder 22 (4 MiB objects) (esc)
+  \tsnapshot_count: 1 (esc)
+  \tid:* (glob)
+  [^^]+ (re)
+  \tformat: 2 (esc)
+  \tfeatures: layering, exclusive-lock, object-map, fast-diff, deep-flatten (esc)
+  \top_features:  (esc)
+  \tflags:  (esc)
+  \tcreate_timestamp:* (glob)
+  \taccess_timestamp:* (glob)
+  \tmodify_timestamp:* (glob)
+  \tprotected: False (esc)
+  $ rbd info rbd_other/deep-flatten-child@snap --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "access_timestamp": "*",  (glob)
+      "block_name_prefix": "rbd_data.*",  (glob)
+      "create_timestamp": "*",  (glob)
+      "features": [
+          "layering", 
+          "exclusive-lock", 
+          "object-map", 
+          "fast-diff", 
+          "deep-flatten"
+      ], 
+      "flags": [], 
+      "format": 2, 
+      "id": "*",  (glob)
+      "modify_timestamp": "*",  (glob)
+      "name": "deep-flatten-child", 
+      "object_size": 4194304, 
+      "objects": 128, 
+      "op_features": [], 
+      "order": 22, 
+      "protected": "false", 
+      "size": 536870912, 
+      "snapshot_count": 1
+  }
+  $ rbd info rbd_other/deep-flatten-child@snap --format xml | xmlstarlet format -s 2 -o || true
+  <image>
+    <name>deep-flatten-child</name>
+    <id>*</id> (glob)
+    <size>536870912</size>
+    <objects>128</objects>
+    <order>22</order>
+    <object_size>4194304</object_size>
+    <snapshot_count>1</snapshot_count>
+    <block_name_prefix>rbd_data.*</block_name_prefix> (glob)
+    <format>2</format>
+    <features>
+      <feature>layering</feature>
+      <feature>exclusive-lock</feature>
+      <feature>object-map</feature>
+      <feature>fast-diff</feature>
+      <feature>deep-flatten</feature>
+    </features>
+    <op_features/>
+    <flags/>
+    <create_timestamp>*</create_timestamp> (glob)
+    <access_timestamp>*</access_timestamp> (glob)
+    <modify_timestamp>*</modify_timestamp> (glob)
+    <protected>false</protected>
+  </image>
+  $ rbd list
+  foo
+  quux
+  bar
+  baz
+  quuy
+  $ rbd list --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      "foo", 
+      "quux", 
+      "bar", 
+      "baz", 
+      "quuy"
+  ]
+  $ rbd list --format xml | xmlstarlet format -s 2 -o || true
+  <images>
+    <name>foo</name>
+    <name>quux</name>
+    <name>bar</name>
+    <name>baz</name>
+    <name>quuy</name>
+  </images>
+  $ rbd list -l
+  NAME       SIZE     PARENT  FMT  PROT  LOCK
+  foo          1 GiB            1            
+  foo@snap     1 GiB            1            
+  quux         1 MiB            1        excl
+  bar          1 GiB            2            
+  bar@snap   512 MiB            2  yes       
+  bar@snap2    1 GiB            2            
+  baz          2 GiB            2        shr 
+  quuy         2 GiB            2            
+  $ rbd list -l --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      {
+          "format": 1, 
+          "id": "", 
+          "image": "foo", 
+          "size": 1073741824
+      }, 
+      {
+          "format": 1, 
+          "id": "", 
+          "image": "foo", 
+          "protected": "false", 
+          "size": 1073741824, 
+          "snapshot": "snap", 
+          "snapshot_id": * (glob)
+      }, 
+      {
+          "format": 1, 
+          "id": "", 
+          "image": "quux", 
+          "lock_type": "exclusive", 
+          "size": 1048576
+      }, 
+      {
+          "format": 2, 
+          "id": "*",  (glob)
+          "image": "bar", 
+          "size": 1073741824
+      }, 
+      {
+          "format": 2, 
+          "id": "*",  (glob)
+          "image": "bar", 
+          "protected": "true", 
+          "size": 536870912, 
+          "snapshot": "snap", 
+          "snapshot_id": * (glob)
+      }, 
+      {
+          "format": 2, 
+          "id": "*",  (glob)
+          "image": "bar", 
+          "protected": "false", 
+          "size": 1073741824, 
+          "snapshot": "snap2", 
+          "snapshot_id": * (glob)
+      }, 
+      {
+          "format": 2, 
+          "id": "*",  (glob)
+          "image": "baz", 
+          "lock_type": "shared", 
+          "size": 2147483648
+      }, 
+      {
+          "format": 2, 
+          "id": "*",  (glob)
+          "image": "quuy", 
+          "size": 2147483648
+      }
+  ]
+  $ rbd list -l --format xml | xmlstarlet format -s 2 -o || true
+  <images>
+    <image>
+      <image>foo</image>
+      <id/>
+      <size>1073741824</size>
+      <format>1</format>
+    </image>
+    <snapshot>
+      <image>foo</image>
+      <id/>
+      <snapshot>snap</snapshot>
+      <snapshot_id>*</snapshot_id> (glob)
+      <size>1073741824</size>
+      <format>1</format>
+      <protected>false</protected>
+    </snapshot>
+    <image>
+      <image>quux</image>
+      <id/>
+      <size>1048576</size>
+      <format>1</format>
+      <lock_type>exclusive</lock_type>
+    </image>
+    <image>
+      <image>bar</image>
+      <id>*</id> (glob)
+      <size>1073741824</size>
+      <format>2</format>
+    </image>
+    <snapshot>
+      <image>bar</image>
+      <id>*</id> (glob)
+      <snapshot>snap</snapshot>
+      <snapshot_id>*</snapshot_id> (glob)
+      <size>536870912</size>
+      <format>2</format>
+      <protected>true</protected>
+    </snapshot>
+    <snapshot>
+      <image>bar</image>
+      <id>*</id> (glob)
+      <snapshot>snap2</snapshot>
+      <snapshot_id>*</snapshot_id> (glob)
+      <size>1073741824</size>
+      <format>2</format>
+      <protected>false</protected>
+    </snapshot>
+    <image>
+      <image>baz</image>
+      <id>*</id> (glob)
+      <size>2147483648</size>
+      <format>2</format>
+      <lock_type>shared</lock_type>
+    </image>
+    <image>
+      <image>quuy</image>
+      <id>*</id> (glob)
+      <size>2147483648</size>
+      <format>2</format>
+    </image>
+  </images>
+  $ rbd list rbd_other
+  child
+  deep-flatten-child
+  $ rbd list rbd_other --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      "child", 
+      "deep-flatten-child"
+  ]
+  $ rbd list rbd_other --format xml | xmlstarlet format -s 2 -o || true
+  <images>
+    <name>child</name>
+    <name>deep-flatten-child</name>
+  </images>
+  $ rbd list rbd_other -l
+  NAME                     SIZE     PARENT        FMT  PROT  LOCK
+  child                    512 MiB                  2            
+  child@snap               512 MiB  rbd/bar@snap    2            
+  deep-flatten-child       512 MiB                  2            
+  deep-flatten-child@snap  512 MiB                  2            
+  $ rbd list rbd_other -l --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      {
+          "format": 2, 
+          "id": "*",  (glob)
+          "image": "child", 
+          "size": 536870912
+      }, 
+      {
+          "format": 2, 
+          "id": "*",  (glob)
+          "image": "child", 
+          "parent": {
+              "image": "bar", 
+              "pool": "rbd", 
+              "pool_namespace": "", 
+              "snapshot": "snap"
+          }, 
+          "protected": "false", 
+          "size": 536870912, 
+          "snapshot": "snap", 
+          "snapshot_id": * (glob)
+      }, 
+      {
+          "format": 2, 
+          "id": "*",  (glob)
+          "image": "deep-flatten-child", 
+          "size": 536870912
+      }, 
+      {
+          "format": 2, 
+          "id": "*",  (glob)
+          "image": "deep-flatten-child", 
+          "protected": "false", 
+          "size": 536870912, 
+          "snapshot": "snap", 
+          "snapshot_id": * (glob)
+      }
+  ]
+  $ rbd list rbd_other -l --format xml | xmlstarlet format -s 2 -o || true
+  <images>
+    <image>
+      <image>child</image>
+      <id>*</id> (glob)
+      <size>536870912</size>
+      <format>2</format>
+    </image>
+    <snapshot>
+      <image>child</image>
+      <id>*</id> (glob)
+      <snapshot>snap</snapshot>
+      <snapshot_id>*</snapshot_id> (glob)
+      <size>536870912</size>
+      <parent>
+        <pool>rbd</pool>
+        <pool_namespace/>
+        <image>bar</image>
+        <snapshot>snap</snapshot>
+      </parent>
+      <format>2</format>
+      <protected>false</protected>
+    </snapshot>
+    <image>
+      <image>deep-flatten-child</image>
+      <id>*</id> (glob)
+      <size>536870912</size>
+      <format>2</format>
+    </image>
+    <snapshot>
+      <image>deep-flatten-child</image>
+      <id>*</id> (glob)
+      <snapshot>snap</snapshot>
+      <snapshot_id>*</snapshot_id> (glob)
+      <size>536870912</size>
+      <format>2</format>
+      <protected>false</protected>
+    </snapshot>
+  </images>
+  $ rbd lock list foo
+  $ rbd lock list foo --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  []
+  $ rbd lock list foo --format xml | xmlstarlet format -s 2 -o || true
+  <locks/>
+  $ rbd lock list quux
+  There is 1 exclusive lock on this image.
+  Locker*ID*Address* (glob)
+  client.* id * (glob)
+  $ rbd lock list quux --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      {
+          "address": "*",  (glob)
+          "id": "id", 
+          "locker": "client.*" (glob)
+      }
+  ]
+  $ rbd lock list quux --format xml | xmlstarlet format -s 2 -o || true
+  <locks>
+    <lock>
+      <id>id</id>
+      <locker>client.*</locker> (glob)
+      <address>*</address> (glob)
+    </lock>
+  </locks>
+  $ rbd lock list baz
+  There are 3 shared locks on this image.
+  Lock tag: tag
+  Locker*ID*Address* (glob)
+  client.*id[123].* (re)
+  client.*id[123].* (re)
+  client.*id[123].* (re)
+  $ rbd lock list baz --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      {
+          "address": "*",  (glob)
+          "id": "id*",  (glob)
+          "locker": "client.*" (glob)
+      }, 
+      {
+          "address": "*",  (glob)
+          "id": "id*",  (glob)
+          "locker": "client.*" (glob)
+      }, 
+      {
+          "address": "*",  (glob)
+          "id": "id*",  (glob)
+          "locker": "client.*" (glob)
+      }
+  ]
+  $ rbd lock list baz --format xml | xmlstarlet format -s 2 -o || true
+  <locks>
+    <lock>
+      <id>id*</id> (glob)
+      <locker>client.*</locker> (glob)
+      <address>*</address> (glob)
+    </lock>
+    <lock>
+      <id>id*</id> (glob)
+      <locker>client.*</locker> (glob)
+      <address>*</address> (glob)
+    </lock>
+    <lock>
+      <id>id*</id> (glob)
+      <locker>client.*</locker> (glob)
+      <address>*</address> (glob)
+    </lock>
+  </locks>
+  $ rbd snap list foo
+  SNAPID*NAME*SIZE*PROTECTED*TIMESTAMP* (glob)
+  *snap*1 GiB* (glob)
+  $ rbd snap list foo --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      {
+          "id": *,  (glob)
+          "name": "snap", 
+          "protected": "false", 
+          "size": 1073741824, 
+          "timestamp": ""
+      }
+  ]
+  $ rbd snap list foo --format xml | xmlstarlet format -s 2 -o || true
+  <snapshots>
+    <snapshot>
+      <id>*</id> (glob)
+      <name>snap</name>
+      <size>1073741824</size>
+      <protected>false</protected>
+      <timestamp/>
+    </snapshot>
+  </snapshots>
+  $ rbd snap list bar
+  SNAPID*NAME*SIZE*PROTECTED*TIMESTAMP* (glob)
+  *snap*512 MiB*yes* (glob)
+  *snap2*1 GiB* (glob)
+  $ rbd snap list bar --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      {
+          "id": *,  (glob)
+          "name": "snap", 
+          "protected": "true", 
+          "size": 536870912, 
+          "timestamp": * (glob)
+      }, 
+      {
+          "id": *,  (glob)
+          "name": "snap2", 
+          "protected": "false", 
+          "size": 1073741824, 
+          "timestamp": * (glob)
+      }
+  ]
+  $ rbd snap list bar --format xml | xmlstarlet format -s 2 -o || true
+  <snapshots>
+    <snapshot>
+      <id>*</id> (glob)
+      <name>snap</name>
+      <size>536870912</size>
+      <protected>true</protected>
+      <timestamp>*</timestamp> (glob)
+    </snapshot>
+    <snapshot>
+      <id>*</id> (glob)
+      <name>snap2</name>
+      <size>1073741824</size>
+      <protected>false</protected>
+      <timestamp>*</timestamp> (glob)
+    </snapshot>
+  </snapshots>
+  $ rbd snap list baz
+  $ rbd snap list baz --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  []
+  $ rbd snap list baz --format xml | xmlstarlet format -s 2 -o || true
+  <snapshots/>
+  $ rbd snap list rbd_other/child
+  SNAPID*NAME*SIZE*PROTECTED*TIMESTAMP* (glob)
+  *snap*512 MiB* (glob)
+  $ rbd snap list rbd_other/child --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      {
+          "id": *,  (glob)
+          "name": "snap", 
+          "protected": "false", 
+          "size": 536870912, 
+          "timestamp": * (glob)
+      }
+  ]
+  $ rbd snap list rbd_other/child --format xml | xmlstarlet format -s 2 -o || true
+  <snapshots>
+    <snapshot>
+      <id>*</id> (glob)
+      <name>snap</name>
+      <size>536870912</size>
+      <protected>false</protected>
+      <timestamp>*</timestamp> (glob)
+    </snapshot>
+  </snapshots>
+  $ rbd disk-usage --pool rbd_other 2>/dev/null
+  NAME                     PROVISIONED  USED 
+  child@snap                   512 MiB    0 B
+  child                        512 MiB  4 MiB
+  deep-flatten-child@snap      512 MiB    0 B
+  deep-flatten-child           512 MiB    0 B
+  <TOTAL>                        1 GiB  4 MiB
+  $ rbd disk-usage --pool rbd_other --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  {
+      "images": [
+          {
+              "id": "*",  (glob)
+              "name": "child", 
+              "provisioned_size": 536870912, 
+              "snapshot": "snap", 
+              "snapshot_id": *,  (glob)
+              "used_size": 0
+          }, 
+          {
+              "id": "*",  (glob)
+              "name": "child", 
+              "provisioned_size": 536870912, 
+              "used_size": 4194304
+          }, 
+          {
+              "id": "*",  (glob)
+              "name": "deep-flatten-child", 
+              "provisioned_size": 536870912, 
+              "snapshot": "snap", 
+              "snapshot_id": *,  (glob)
+              "used_size": 0
+          }, 
+          {
+              "id": "*",  (glob)
+              "name": "deep-flatten-child", 
+              "provisioned_size": 536870912, 
+              "used_size": 0
+          }
+      ], 
+      "total_provisioned_size": 1073741824, 
+      "total_used_size": 4194304
+  }
+  $ rbd disk-usage --pool rbd_other --format xml | xmlstarlet format -s 2 -o || true
+  <stats>
+    <images>
+      <image>
+        <name>child</name>
+        <id>*</id> (glob)
+        <snapshot>snap</snapshot>
+        <snapshot_id>*</snapshot_id> (glob)
+        <provisioned_size>536870912</provisioned_size>
+        <used_size>0</used_size>
+      </image>
+      <image>
+        <name>child</name>
+        <id>*</id> (glob)
+        <provisioned_size>536870912</provisioned_size>
+        <used_size>4194304</used_size>
+      </image>
+      <image>
+        <name>deep-flatten-child</name>
+        <id>*</id> (glob)
+        <snapshot>snap</snapshot>
+        <snapshot_id>*</snapshot_id> (glob)
+        <provisioned_size>536870912</provisioned_size>
+        <used_size>0</used_size>
+      </image>
+      <image>
+        <name>deep-flatten-child</name>
+        <id>*</id> (glob)
+        <provisioned_size>536870912</provisioned_size>
+        <used_size>0</used_size>
+      </image>
+    </images>
+    <total_provisioned_size>1073741824</total_provisioned_size>
+    <total_used_size>4194304</total_used_size>
+  </stats>
+
+# cleanup
+  $ rbd snap remove --no-progress rbd_other/deep-flatten-child@snap
+  $ rbd snap remove --no-progress rbd_other/child@snap
+  $ rbd snap unprotect bar@snap
+  $ rbd snap purge bar 2> /dev/null
+  $ rbd snap purge foo 2> /dev/null
+  $ rbd rm rbd_other/deep-flatten-child 2> /dev/null
+  $ rbd rm rbd_other/child 2> /dev/null
+  $ rbd rm foo 2> /dev/null
+  $ rbd rm bar 2> /dev/null
+  $ rbd rm quux 2> /dev/null
+  $ rbd rm quuy 2> /dev/null
+  $ rbd rm baz 2> /dev/null
+  $ ceph osd pool delete rbd_other rbd_other --yes-i-really-really-mean-it
+  pool 'rbd_other' removed
diff --git a/src/test/cli-integration/rbd/gwcli_create.t b/src/test/cli-integration/rbd/gwcli_create.t
new file mode 100644
index 000000000..e23ed895e
--- /dev/null
+++ b/src/test/cli-integration/rbd/gwcli_create.t
@@ -0,0 +1,74 @@
+Dismiss the "could not load preferences file .gwcli/prefs.bin" warning
+======================================================================
+  $ sudo gwcli ls >/dev/null 2>&1
+
+Create a datapool/block0 disk
+=============================
+  $ sudo gwcli disks/ create pool=datapool image=block0 size=300M wwn=36001405da17b74481464e9fa968746d3
+  $ sudo gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  300M, Disks: 1]
+  $ sudo gwcli ls disks/ | grep 'o- datapool' | awk -F'[' '{print $2}'
+  datapool (300M)]
+  $ sudo gwcli ls disks/ | grep 'o- block0' | awk -F'[' '{print $2}'
+  datapool/block0 (Unknown, 300M)]
+
+Create the target IQN
+=====================
+  $ sudo gwcli iscsi-targets/ create target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}'
+  DiscoveryAuth: None, Targets: 1]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw' | awk -F'[' '{print $2}'
+  Auth: None, Gateways: 0]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  Disks: 0]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}'
+  Up: 0/0, Portals: 0]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- host-groups' | awk -F'[' '{print $2}'
+  Groups : 0]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+  Auth: ACL_ENABLED, Hosts: 0]
+
+Create the first gateway
+========================
+  $ HOST=`python3 -c "import socket; print(socket.getfqdn())"`
+  > IP=`hostname -i | awk '{print $1}'`
+  > sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}'
+  Up: 1/1, Portals: 1]
+
+Create the second gateway
+========================
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+  > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then
+  >   HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"`
+  >   sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST
+  > fi
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $4}'`
+  > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then
+  >   HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"`
+  >   sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST
+  > fi
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}'
+  Up: 2/2, Portals: 2]
+
+Attach the disk
+===============
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ add disk=datapool/block0
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  Disks: 1]
+
+Create a host
+=============
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts create client_iqn=iqn.1994-05.com.redhat:client
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+  Auth: ACL_ENABLED, Hosts: 1]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}'
+  Auth: None, Disks: 0(0.00Y)]
+
+Map the LUN
+===========
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts/iqn.1994-05.com.redhat:client disk disk=datapool/block0
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+  Auth: ACL_ENABLED, Hosts: 1]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}'
+  Auth: None, Disks: 1(300M)]
diff --git a/src/test/cli-integration/rbd/gwcli_delete.t b/src/test/cli-integration/rbd/gwcli_delete.t
new file mode 100644
index 000000000..5aef9d0e9
--- /dev/null
+++ b/src/test/cli-integration/rbd/gwcli_delete.t
@@ -0,0 +1,27 @@
+Dismiss the "could not load preferences file .gwcli/prefs.bin" warning
+======================================================================
+  $ sudo gwcli ls >/dev/null 2>&1
+
+Delete the host
+===============
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts delete client_iqn=iqn.1994-05.com.redhat:client
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+  Auth: ACL_ENABLED, Hosts: 0]
+
+Delete the iscsi-targets disk
+=============================
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ delete disk=datapool/block0
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  Disks: 0]
+
+Delete the target IQN
+=====================
+  $ sudo gwcli iscsi-targets/ delete target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}'
+  DiscoveryAuth: None, Targets: 0]
+
+Delete the disks
+================
+  $ sudo gwcli disks/ delete image_id=datapool/block0
+  $ sudo gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  0.00Y, Disks: 0]
diff --git a/src/test/cli-integration/rbd/iscsi_client.t b/src/test/cli-integration/rbd/iscsi_client.t
new file mode 100644
index 000000000..f636d540d
--- /dev/null
+++ b/src/test/cli-integration/rbd/iscsi_client.t
@@ -0,0 +1,30 @@
+Login to the target
+===================
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+  > sudo iscsiadm -m discovery -t st -p $IP -l 2&> /dev/null
+  $ sleep 10
+  $ sudo ls /dev/disk/by-path/ |grep 'iscsi-iqn.2003-01.com.redhat.iscsi-gw:ceph-gw' |wc -l
+  2
+
+Make filesystem
+===============
+  $ device=`sudo multipath -l | grep 'LIO-ORG,TCMU device' | awk '{print $1}'`
+  > sudo mkfs.xfs /dev/mapper/$device -f | grep 'meta-data=/dev/mapper/mpath' | awk '{print $2}'
+  isize=512
+
+Write/Read test
+===============
+  $ device=`sudo multipath -l | grep 'LIO-ORG,TCMU device' | awk '{print $1}'`
+  > sudo dd if=/dev/random of=/tmp/iscsi_tmpfile bs=1 count=1K status=none
+  > sudo dd if=/tmp/iscsi_tmpfile of=/dev/mapper/$device bs=1 count=1K status=none
+  > sudo dd if=/dev/mapper/$device of=/tmp/iscsi_tmpfile1 bs=1 count=1K status=none
+  $ sudo diff /tmp/iscsi_tmpfile /tmp/iscsi_tmpfile1
+
+Logout the targets
+==================
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+  > sudo iscsiadm -m node -T iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -p $IP:3260 -u | grep 'successful' | awk -F']' '{print $2}'
+   successful.
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $4}'`
+  > sudo iscsiadm -m node -T iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -p $IP:3260 -u | grep 'successful' | awk -F']' '{print $2}'
+   successful.
diff --git a/src/test/cli-integration/rbd/rest_api_create.t b/src/test/cli-integration/rbd/rest_api_create.t
new file mode 100644
index 000000000..d0308eef3
--- /dev/null
+++ b/src/test/cli-integration/rbd/rest_api_create.t
@@ -0,0 +1,44 @@
+Create a datapool/block1 disk
+=============================
+  $ sudo curl --user admin:admin -d mode=create -d size=300M -X PUT http://127.0.0.1:5000/api/disk/datapool/block1 -s
+  {"message":"disk create/update successful"}
+
+Create the target IQN
+=====================
+  $ sudo curl --user admin:admin -X PUT http://127.0.0.1:5000/api/target/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -s
+  {"message":"Target defined successfully"}
+
+Create the first gateway
+========================
+  $ HOST=`python3 -c "import socket; print(socket.getfqdn())"`
+  > IP=`hostname -i | awk '{print $1}'`
+  > sudo curl --user admin:admin -d ip_address=$IP -X PUT http://127.0.0.1:5000/api/gateway/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/$HOST -s
+  {"message":"Gateway creation successful"}
+
+Create the second gateway
+========================
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+  > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then
+  >   HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"`
+  >   sudo curl --user admin:admin -d ip_address=$IP -X PUT http://127.0.0.1:5000/api/gateway/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/$HOST -s
+  > else
+  >   IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $4}'`
+  >   HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"`
+  >   sudo curl --user admin:admin -d ip_address=$IP -X PUT http://127.0.0.1:5000/api/gateway/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/$HOST -s
+  > fi
+  {"message":"Gateway creation successful"}
+
+Attach the disk
+===============
+  $ sudo curl --user admin:admin -d disk=datapool/block1 -X PUT http://127.0.0.1:5000/api/targetlun/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -s
+  {"message":"Target LUN mapping updated successfully"}
+
+Create a host
+=============
+  $ sudo curl --user admin:admin -X PUT http://127.0.0.1:5000/api/client/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/iqn.1994-05.com.redhat:client -s
+  {"message":"client create/update successful"}
+
+Map the LUN
+===========
+  $ sudo curl --user admin:admin -d disk=datapool/block1 -X PUT http://127.0.0.1:5000/api/clientlun/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/iqn.1994-05.com.redhat:client -s
+  {"message":"client masking update successful"}
diff --git a/src/test/cli-integration/rbd/rest_api_delete.t b/src/test/cli-integration/rbd/rest_api_delete.t
new file mode 100644
index 000000000..0e66d25d9
--- /dev/null
+++ b/src/test/cli-integration/rbd/rest_api_delete.t
@@ -0,0 +1,19 @@
+Delete the host
+===============
+  $ sudo curl --user admin:admin -X DELETE http://127.0.0.1:5000/api/client/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/iqn.1994-05.com.redhat:client -s
+  {"message":"client delete successful"}
+
+Delete the iscsi-targets disk
+=============================
+  $ sudo curl --user admin:admin -d disk=datapool/block1 -X DELETE http://127.0.0.1:5000/api/targetlun/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -s
+  {"message":"Target LUN mapping updated successfully"}
+
+Delete the target IQN
+=====================
+  $ sudo curl --user admin:admin -X DELETE http://127.0.0.1:5000/api/target/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -s
+  {"message":"Target deleted."}
+
+Delete the disks
+================
+  $ sudo curl --user admin:admin -X DELETE http://127.0.0.1:5000/api/disk/datapool/block1 -s
+  {"message":"disk map deletion successful"}
diff --git a/src/test/cli-integration/rbd/snap-diff.t b/src/test/cli-integration/rbd/snap-diff.t
new file mode 100644
index 000000000..1ca2fb04d
--- /dev/null
+++ b/src/test/cli-integration/rbd/snap-diff.t
@@ -0,0 +1,48 @@
+  $ ceph osd pool create xrbddiff1
+  pool 'xrbddiff1' created
+  $ rbd pool init xrbddiff1
+  $ rbd create --thick-provision --size 1M xrbddiff1/xtestdiff1 --no-progress
+  $ rbd diff xrbddiff1/xtestdiff1 --format json
+  [{"offset":0,"length":1048576,"exists":"true"}]
+  $ rbd rm xrbddiff1/xtestdiff1 --no-progress
+  $ rbd create --size 1M xrbddiff1/xtestdiff1
+  $ rbd diff xrbddiff1/xtestdiff1 --format json
+  []
+  $ rbd snap create xrbddiff1/xtestdiff1 --snap=allzeroes --no-progress
+  $ rbd diff xrbddiff1/xtestdiff1 --format json
+  []
+  $ rbd diff --from-snap=allzeroes xrbddiff1/xtestdiff1 --format json
+  []
+  $ rbd bench --io-type write --io-size 1M --io-total 1M xrbddiff1/xtestdiff1 > /dev/null 2>&1
+  $ rbd diff xrbddiff1/xtestdiff1 --format json
+  [{"offset":0,"length":1048576,"exists":"true"}]
+  $ rbd diff --from-snap=allzeroes xrbddiff1/xtestdiff1 --format json
+  [{"offset":0,"length":1048576,"exists":"true"}]
+  $ rbd snap create xrbddiff1/xtestdiff1 --snap=snap1 --no-progress
+  $ rbd snap list xrbddiff1/xtestdiff1 --format json | python3 -mjson.tool --sort-keys | sed 's/,$/, /'
+  [
+      {
+          "id": *,  (glob)
+          "name": "allzeroes", 
+          "protected": "false", 
+          "size": 1048576, 
+          "timestamp": * (glob)
+      }, 
+      {
+          "id": *,  (glob)
+          "name": "snap1", 
+          "protected": "false", 
+          "size": 1048576, 
+          "timestamp": * (glob)
+      }
+  ]
+  $ rbd diff --from-snap=snap1 xrbddiff1/xtestdiff1 --format json
+  []
+  $ rbd snap rollback xrbddiff1/xtestdiff1@snap1 --no-progress
+  $ rbd diff --from-snap=snap1 xrbddiff1/xtestdiff1 --format json
+  []
+  $ rbd snap rollback xrbddiff1/xtestdiff1@allzeroes --no-progress
+  $ rbd diff --from-snap=allzeroes xrbddiff1/xtestdiff1 --format json
+  [{"offset":0,"length":1048576,"exists":"false"}]
+  $ ceph osd pool rm xrbddiff1 xrbddiff1 --yes-i-really-really-mean-it
+  pool 'xrbddiff1' removed
diff --git a/src/test/cli-integration/rbd/unmap.t b/src/test/cli-integration/rbd/unmap.t
new file mode 100644
index 000000000..3fdac43d7
--- /dev/null
+++ b/src/test/cli-integration/rbd/unmap.t
@@ -0,0 +1,488 @@
+
+Setup
+=====
+
+  $ rbd create --size 1 img
+  $ rbd snap create img@snap --no-progress
+  $ rbd create --size 1 anotherimg
+  $ ceph osd pool create custom >/dev/null 2>&1
+  $ rbd pool init custom
+  $ rbd create --size 1 custom/img
+  $ rbd snap create custom/img@snap --no-progress
+  $ rbd snap create custom/img@anothersnap --no-progress
+
+Spell out device instead of using $DEV - sfdisk is not a joke.
+
+  $ DEV=$(sudo rbd device map img)
+  $ cat <<EOF | sudo sfdisk /dev/rbd[01] >/dev/null 2>&1
+  > unit: sectors
+  > /dev/rbd0p1 : start=        2, size=        2, Id=83
+  > /dev/rbd0p2 : start=        5, size=     2043, Id= 5
+  > /dev/rbd0p3 : start=        0, size=        0, Id= 0
+  > /dev/rbd0p4 : start=        0, size=        0, Id= 0
+  > /dev/rbd0p5 : start=        7, size=        2, Id=83
+  > EOF
+
+
+Unmap by device
+===============
+
+Unmap by device (img is already mapped):
+
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap $DEV
+  $ rbd device list
+
+Unmap by device partition:
+
+  $ DEV=$(sudo rbd device map img)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap ${DEV}p1
+  $ rbd device list
+
+  $ DEV=$(sudo rbd device map img)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap ${DEV}p5
+  $ rbd device list
+
+Not a block device - random junk prefixed with /dev/ (so it's not
+interpreted as a spec):
+
+  $ sudo rbd device unmap /dev/foobar
+  rbd: '/dev/foobar' is not a block device
+  rbd: unmap failed: (22) Invalid argument
+  [22]
+
+Not a block device - device that's just been unmapped:
+
+  $ DEV=$(sudo rbd device map img)
+  $ sudo rbd device unmap $DEV
+  $ sudo rbd device unmap $DEV
+  rbd: '/dev/rbd?' is not a block device (glob)
+  rbd: unmap failed: (22) Invalid argument
+  [22]
+
+A block device, but not rbd:
+
+  $ sudo rbd device unmap /dev/[sv]da
+  rbd: '/dev/?da' is not an rbd device (glob)
+  rbd: unmap failed: (22) Invalid argument
+  [22]
+
+
+Unmap by spec
+=============
+
+img:
+
+  $ sudo rbd device map img
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap img
+  $ rbd device list
+
+  $ sudo rbd device map img
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  $ sudo rbd --image img device unmap
+  $ rbd device list
+
+img@snap:
+
+  $ sudo rbd device map img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap img@snap
+  $ rbd device list
+
+  $ sudo rbd device map img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd --snap snap device unmap img
+  $ rbd device list
+
+  $ sudo rbd device map img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd --image img --snap snap device unmap
+  $ rbd device list
+
+pool/img@snap, default pool:
+
+  $ sudo rbd device map rbd/img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap rbd/img@snap
+  $ rbd device list
+
+  $ sudo rbd device map rbd/img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd --pool rbd device unmap img@snap
+  $ rbd device list
+
+  $ sudo rbd device map rbd/img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd --pool rbd --snap snap device unmap img
+  $ rbd device list
+
+  $ sudo rbd device map rbd/img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd --pool rbd --image img --snap snap device unmap
+  $ rbd device list
+
+pool/img@snap, custom pool:
+
+  $ sudo rbd device map custom/img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   custom             img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap custom/img@snap
+  $ rbd device list
+
+  $ sudo rbd device map custom/img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   custom             img    snap  /dev/rbd? (glob)
+  $ sudo rbd --pool custom device unmap img@snap
+  $ rbd device list
+
+  $ sudo rbd device map custom/img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   custom             img    snap  /dev/rbd? (glob)
+  $ sudo rbd --pool custom --snap snap device unmap img
+  $ rbd device list
+
+  $ sudo rbd device map custom/img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   custom             img    snap  /dev/rbd? (glob)
+  $ sudo rbd --pool custom --image img --snap snap device unmap
+  $ rbd device list
+
+Not a mapped spec - random junk (which gets interpreted as a spec):
+
+  $ sudo rbd device unmap foobar
+  rbd: rbd/foobar: not a mapped image or snapshot
+  rbd: unmap failed: (22) Invalid argument
+  [22]
+
+  $ sudo rbd --image foobar device unmap
+  rbd: rbd/foobar: not a mapped image or snapshot
+  rbd: unmap failed: (22) Invalid argument
+  [22]
+
+Not a mapped spec - spec that's just been unmapped:
+
+  $ sudo rbd device map img
+  /dev/rbd? (glob)
+  $ sudo rbd device unmap img
+  $ sudo rbd device unmap img
+  rbd: rbd/img: not a mapped image or snapshot
+  rbd: unmap failed: (22) Invalid argument
+  [22]
+
+  $ sudo rbd device map img@snap
+  /dev/rbd? (glob)
+  $ sudo rbd device unmap img@snap
+  $ sudo rbd device unmap img@snap
+  rbd: rbd/img@snap: not a mapped image or snapshot
+  rbd: unmap failed: (22) Invalid argument
+  [22]
+
+Need an arg:
+
+  $ sudo rbd device unmap
+  rbd: unmap requires either image name or device path
+  [22]
+
+
+Two images
+==========
+
+Unmap img first:
+
+  $ sudo rbd device map img
+  /dev/rbd? (glob)
+  $ sudo rbd device map anotherimg
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image       snap  device   
+  ?   rbd              img         -     /dev/rbd? (glob)
+  ?   rbd              anotherimg  -     /dev/rbd? (glob)
+  $ sudo rbd device unmap img
+  $ rbd device list
+  id  pool  namespace  image       snap  device   
+  ?   rbd              anotherimg  -     /dev/rbd? (glob)
+  $ sudo rbd device unmap anotherimg
+  $ rbd device list
+
+Unmap anotherimg first:
+
+  $ sudo rbd device map img
+  /dev/rbd? (glob)
+  $ sudo rbd device map anotherimg
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image       snap  device   
+  ?   rbd              img         -     /dev/rbd? (glob)
+  ?   rbd              anotherimg  -     /dev/rbd? (glob)
+  $ sudo rbd device unmap anotherimg
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap img
+  $ rbd device list
+
+
+Image and its snap
+==================
+
+Unmap the image first:
+
+  $ sudo rbd device map img
+  /dev/rbd? (glob)
+  $ sudo rbd device map img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap img
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap img@snap
+  $ rbd device list
+
+Unmap the snap first:
+
+  $ sudo rbd device map img
+  /dev/rbd? (glob)
+  $ sudo rbd device map img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap img@snap
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap img
+  $ rbd device list
+
+
+Two snaps of the same image
+===========================
+
+Unmap snap first:
+
+  $ sudo rbd device map custom/img@snap
+  /dev/rbd? (glob)
+  $ sudo rbd device map custom/img@anothersnap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap         device   
+  ?   custom             img    snap         /dev/rbd? (glob)
+  ?   custom             img    anothersnap  /dev/rbd? (glob)
+  $ sudo rbd device unmap custom/img@snap
+  $ rbd device list
+  id  pool    namespace  image  snap         device   
+  ?   custom             img    anothersnap  /dev/rbd? (glob)
+  $ sudo rbd device unmap custom/img@anothersnap
+  $ rbd device list
+
+Unmap anothersnap first:
+
+  $ sudo rbd device map custom/img@snap
+  /dev/rbd? (glob)
+  $ sudo rbd device map custom/img@anothersnap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap         device   
+  ?   custom             img    snap         /dev/rbd? (glob)
+  ?   custom             img    anothersnap  /dev/rbd? (glob)
+  $ sudo rbd device unmap custom/img@anothersnap
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   custom             img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap custom/img@snap
+  $ rbd device list
+
+
+Same img and snap in different pools
+====================================
+
+img:
+
+  $ sudo rbd device map img
+  /dev/rbd? (glob)
+  $ sudo rbd device map custom/img
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   rbd                img    -     /dev/rbd? (glob)
+  ?   custom             img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap img
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   custom             img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap custom/img
+  $ rbd device list
+
+img@snap:
+
+  $ sudo rbd device map img@snap
+  /dev/rbd? (glob)
+  $ sudo rbd device map custom/img@snap
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   rbd                img    snap  /dev/rbd? (glob)
+  ?   custom             img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap custom/img@snap
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap img@snap
+  $ rbd device list
+
+
+Same spec mapped twice
+======================
+
+img:
+
+  $ sudo rbd device map img
+  /dev/rbd? (glob)
+  $ sudo rbd device map img
+  rbd: warning: image already mapped as /dev/rbd? (glob)
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  ?   rbd              img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap img
+  rbd: rbd/img: mapped more than once, unmapping /dev/rbd? only (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    -     /dev/rbd? (glob)
+  $ sudo rbd device unmap img
+  $ rbd device list
+
+img@snap:
+
+  $ sudo rbd device map img@snap
+  /dev/rbd? (glob)
+  $ sudo rbd device map img@snap
+  rbd: warning: image already mapped as /dev/rbd? (glob)
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap img@snap
+  rbd: rbd/img@snap: mapped more than once, unmapping /dev/rbd? only (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap img@snap
+  $ rbd device list
+
+pool/img@snap, default pool:
+
+  $ sudo rbd device map rbd/img@snap
+  /dev/rbd? (glob)
+  $ sudo rbd device map rbd/img@snap
+  rbd: warning: image already mapped as /dev/rbd? (glob)
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap rbd/img@snap
+  rbd: rbd/img@snap: mapped more than once, unmapping /dev/rbd? only (glob)
+  $ rbd device list
+  id  pool  namespace  image  snap  device   
+  ?   rbd              img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap rbd/img@snap
+  $ rbd device list
+
+pool/img@snap, custom pool:
+
+  $ sudo rbd device map custom/img@snap
+  /dev/rbd? (glob)
+  $ sudo rbd device map custom/img@snap
+  rbd: warning: image already mapped as /dev/rbd? (glob)
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   custom             img    snap  /dev/rbd? (glob)
+  ?   custom             img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap custom/img@snap
+  rbd: custom/img@snap: mapped more than once, unmapping /dev/rbd? only (glob)
+  $ rbd device list
+  id  pool    namespace  image  snap  device   
+  ?   custom             img    snap  /dev/rbd? (glob)
+  $ sudo rbd device unmap custom/img@snap
+  $ rbd device list
+
+
+Odd names
+=========
+
+  $ ceph osd pool create foo\* >/dev/null 2>&1
+  $ rbd pool init foo\*
+  $ rbd create --size 1 foo\*/[0.0.0.0]
+  $ rbd snap create foo\*/[0.0.0.0]@\?bar --no-progress
+  $ sudo rbd device map foo\*/[0.0.0.0]@\?bar
+  /dev/rbd? (glob)
+  $ rbd device list
+  id  pool  namespace  image      snap  device   
+  ?   foo*             [0.0.0.0]  ?bar  /dev/rbd? (glob)
+  $ sudo rbd device unmap foo\*/[0.0.0.0]@\?bar
+  $ rbd device list
+  $ ceph osd pool delete foo\* foo\* --yes-i-really-really-mean-it >/dev/null 2>&1
+
+
+Teardown
+========
+
+  $ ceph osd pool delete custom custom --yes-i-really-really-mean-it >/dev/null 2>&1
+  $ rbd snap purge anotherimg >/dev/null 2>&1
+  $ rbd rm anotherimg >/dev/null 2>&1
+  $ rbd snap purge img >/dev/null 2>&1
+  $ rbd rm img >/dev/null 2>&1
+