Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

38 files changed, 15413 insertions, 0 deletions

diff --git a/src/tools/rbd/action/Bench.cc b/src/tools/rbd/action/Bench.cc
new file mode 100644
index 00000000..27843c7b
--- /dev/null
+++ b/src/tools/rbd/action/Bench.cc
@@ -0,0 +1,539 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include "common/strtol.h"
+#include "common/Cond.h"
+#include "common/Mutex.h"
+#include "global/signal_handler.h"
+#include <atomic>
+#include <chrono>
+#include <iostream>
+#include <boost/accumulators/accumulators.hpp>
+#include <boost/accumulators/statistics/stats.hpp>
+#include <boost/accumulators/statistics/rolling_sum.hpp>
+#include <boost/program_options.hpp>
+
+using namespace std::chrono;
+
+static std::atomic<bool> terminating;
+static void handle_signal(int signum)
+{
+  ceph_assert(signum == SIGINT || signum == SIGTERM);
+  terminating = true;
+}
+
+namespace rbd {
+namespace action {
+namespace bench {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+namespace {
+
+enum io_type_t {
+  IO_TYPE_READ = 0,
+  IO_TYPE_WRITE,
+  IO_TYPE_RW,
+
+  IO_TYPE_NUM,
+};
+
+struct IOType {};
+struct Size {};
+struct IOPattern {};
+
+void validate(boost::any& v, const std::vector<std::string>& values,
+              Size *target_type, int) {
+  po::validators::check_first_occurrence(v);
+  const std::string &s = po::validators::get_single_string(values);
+
+  std::string parse_error;
+  uint64_t size = strict_iecstrtoll(s.c_str(), &parse_error);
+  if (!parse_error.empty()) {
+    throw po::validation_error(po::validation_error::invalid_option_value);
+  }
+  v = boost::any(size);
+}
+
+void validate(boost::any& v, const std::vector<std::string>& values,
+              IOPattern *target_type, int) {
+  po::validators::check_first_occurrence(v);
+  const std::string &s = po::validators::get_single_string(values);
+  if (s == "rand") {
+    v = boost::any(true);
+  } else if (s == "seq") {
+    v = boost::any(false);
+  } else {
+    throw po::validation_error(po::validation_error::invalid_option_value);
+  }
+}
+
+io_type_t get_io_type(string io_type_string) {
+  if (io_type_string == "read")
+    return IO_TYPE_READ;
+  else if (io_type_string == "write")
+    return IO_TYPE_WRITE;
+  else if (io_type_string == "readwrite" || io_type_string == "rw")
+    return IO_TYPE_RW;
+  else
+    return IO_TYPE_NUM;
+}
+
+void validate(boost::any& v, const std::vector<std::string>& values,
+              IOType *target_type, int) {
+  po::validators::check_first_occurrence(v);
+  const std::string &s = po::validators::get_single_string(values);
+  io_type_t io_type = get_io_type(s);
+  if (io_type >= IO_TYPE_NUM)
+    throw po::validation_error(po::validation_error::invalid_option_value);
+  else
+    v = boost::any(io_type);
+}
+
+} // anonymous namespace
+
+static void rbd_bencher_completion(void *c, void *pc);
+struct rbd_bencher;
+
+struct bencher_completer {
+  rbd_bencher *bencher;
+  bufferlist *bl;
+
+public:
+  bencher_completer(rbd_bencher *bencher, bufferlist *bl)
+    : bencher(bencher), bl(bl)
+  { }
+
+  ~bencher_completer()
+  {
+    if (bl)
+      delete bl;
+  }
+};
+
+struct rbd_bencher {
+  librbd::Image *image;
+  Mutex lock;
+  Cond cond;
+  int in_flight;
+  io_type_t io_type;
+  uint64_t io_size;
+  bufferlist write_bl;
+
+  explicit rbd_bencher(librbd::Image *i, io_type_t io_type, uint64_t io_size)
+    : image(i),
+      lock("rbd_bencher::lock"),
+      in_flight(0),
+      io_type(io_type),
+      io_size(io_size)
+  {
+    if (io_type == IO_TYPE_WRITE || io_type == IO_TYPE_RW) {
+      bufferptr bp(io_size);
+      memset(bp.c_str(), rand() & 0xff, io_size);
+      write_bl.push_back(bp);
+    }
+  }
+
+  void start_io(int max, uint64_t off, uint64_t len, int op_flags, bool read_flag)
+  {
+    {
+      Mutex::Locker l(lock);
+      in_flight++;
+    }
+
+    librbd::RBD::AioCompletion *c;
+    if (read_flag) {
+      bufferlist *read_bl = new bufferlist();
+      c = new librbd::RBD::AioCompletion((void *)(new bencher_completer(this, read_bl)),
+					 rbd_bencher_completion);
+      image->aio_read2(off, len, *read_bl, c, op_flags);
+    } else {
+      c = new librbd::RBD::AioCompletion((void *)(new bencher_completer(this, NULL)),
+					 rbd_bencher_completion);
+      image->aio_write2(off, len, write_bl, c, op_flags);
+    }
+  }
+
+  int wait_for(int max, bool interrupt_on_terminating) {
+    Mutex::Locker l(lock);
+    while (in_flight > max && !(terminating && interrupt_on_terminating)) {
+      utime_t dur;
+      dur.set_from_double(.2);
+      cond.WaitInterval(lock, dur);
+    }
+
+    return terminating ? -EINTR : 0;
+  }
+
+};
+
+void rbd_bencher_completion(void *vc, void *pc)
+{
+  librbd::RBD::AioCompletion *c = (librbd::RBD::AioCompletion *)vc;
+  bencher_completer *bc = static_cast<bencher_completer *>(pc);
+  rbd_bencher *b = bc->bencher;
+  //cout << "complete " << c << std::endl;
+  int ret = c->get_return_value();
+  if (b->io_type == IO_TYPE_WRITE && ret != 0) {
+    cout << "write error: " << cpp_strerror(ret) << std::endl;
+    exit(ret < 0 ? -ret : ret);
+  } else if (b->io_type == IO_TYPE_READ && (unsigned int)ret != b->io_size) {
+    cout << "read error: " << cpp_strerror(ret) << std::endl;
+    exit(ret < 0 ? -ret : ret);
+  }
+  b->lock.Lock();
+  b->in_flight--;
+  b->cond.Signal();
+  b->lock.Unlock();
+  c->release();
+  delete bc;
+}
+
+bool should_read(uint64_t read_proportion)
+{
+  uint64_t rand_num = rand() % 100;
+
+  if (rand_num < read_proportion)
+    return true;
+  else
+    return false;
+}
+
+int do_bench(librbd::Image& image, io_type_t io_type,
+		   uint64_t io_size, uint64_t io_threads,
+		   uint64_t io_bytes, bool random, uint64_t read_proportion)
+{
+  uint64_t size = 0;
+  image.size(&size);
+  if (io_size > size) {
+    std::cerr << "rbd: io-size " << byte_u_t(io_size) << " "
+              << "larger than image size " << byte_u_t(size) << std::endl;
+    return -EINVAL;
+  }
+
+  if (io_size > std::numeric_limits<uint32_t>::max()) {
+    std::cerr << "rbd: io-size should be less than 4G" << std::endl;
+    return -EINVAL;
+  }
+
+  int r = image.flush();
+  if (r < 0 && (r != -EROFS || io_type != IO_TYPE_READ)) {
+    std::cerr << "rbd: failed to flush: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  rbd_bencher b(&image, io_type, io_size);
+
+  std::cout << "bench "
+       << " type " << (io_type == IO_TYPE_READ ? "read" :
+                       io_type == IO_TYPE_WRITE ? "write" : "readwrite")
+       << (io_type == IO_TYPE_RW ? " read:write=" +
+           to_string(read_proportion) + ":" + to_string(100 - read_proportion) : "")
+       << " io_size " << io_size
+       << " io_threads " << io_threads
+       << " bytes " << io_bytes
+       << " pattern " << (random ? "random" : "sequential")
+       << std::endl;
+
+  srand(time(NULL) % (unsigned long) -1);
+
+  coarse_mono_time start = coarse_mono_clock::now();
+  chrono::duration<double> last = chrono::duration<double>::zero();
+  unsigned ios = 0;
+
+  vector<uint64_t> thread_offset;
+  uint64_t i;
+  uint64_t start_pos;
+
+  uint64_t unit_len = size/io_size/io_threads;
+  // disturb all thread's offset
+  for (i = 0; i < io_threads; i++) {
+    if (random) {
+      start_pos = (rand() % (size / io_size)) * io_size;
+    } else {
+      start_pos = unit_len * i * io_size;
+    }
+    thread_offset.push_back(start_pos);
+  }
+
+  const int WINDOW_SIZE = 5;
+  typedef boost::accumulators::accumulator_set<
+    double, boost::accumulators::stats<
+      boost::accumulators::tag::rolling_sum> > RollingSum;
+
+  RollingSum time_acc(
+    boost::accumulators::tag::rolling_window::window_size = WINDOW_SIZE);
+  RollingSum ios_acc(
+    boost::accumulators::tag::rolling_window::window_size = WINDOW_SIZE);
+  RollingSum off_acc(
+    boost::accumulators::tag::rolling_window::window_size = WINDOW_SIZE);
+  uint64_t cur_ios = 0;
+  uint64_t cur_off = 0;
+
+  int op_flags;
+  if  (random) {
+    op_flags = LIBRADOS_OP_FLAG_FADVISE_RANDOM;
+  } else {
+    op_flags = LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL;
+  }
+
+  printf("  SEC       OPS   OPS/SEC   BYTES/SEC\n");
+  uint64_t off;
+  int read_ops = 0;
+  int write_ops = 0;
+
+  for (off = 0; off < io_bytes; ) {
+    // Issue I/O
+    i = 0;
+    int r = 0;
+    while (i < io_threads && off < io_bytes) {
+      bool read_flag = should_read(read_proportion);
+
+      r = b.wait_for(io_threads - 1, true);
+      if (r < 0) {
+        break;
+      }
+      b.start_io(io_threads, thread_offset[i], io_size, op_flags, read_flag);
+
+      ++i;
+      ++ios;
+      off += io_size;
+
+      ++cur_ios;
+      cur_off += io_size;
+
+      if (read_flag)
+        read_ops++;
+      else
+        write_ops++;
+    }
+
+    if (r < 0) {
+      break;
+    }
+
+    // Set the thread_offsets of next I/O
+    for (i = 0; i < io_threads; ++i) {
+      if (random) {
+        thread_offset[i] = (rand() % (size / io_size)) * io_size;
+        continue;
+      }
+      if (off < (io_size * unit_len * io_threads) ) {
+        thread_offset[i] += io_size;
+      } else {
+        // thread_offset is adjusted to the chunks unassigned to threads.
+        thread_offset[i] = off + (i * io_size);
+      }
+      if (thread_offset[i] + io_size > size)
+        thread_offset[i] = unit_len * i * io_size;
+    }
+
+    coarse_mono_time now = coarse_mono_clock::now();
+    chrono::duration<double> elapsed = now - start;
+    if (last == chrono::duration<double>::zero()) {
+      last = elapsed;
+    } else if ((int)elapsed.count() != (int)last.count()) {
+      time_acc((elapsed - last).count());
+      ios_acc(static_cast<double>(cur_ios));
+      off_acc(static_cast<double>(cur_off));
+      cur_ios = 0;
+      cur_off = 0;
+
+      double time_sum = boost::accumulators::rolling_sum(time_acc);
+      printf("%5d  %8d  %8.2lf  %8.2lf\n",
+             (int)elapsed.count(),
+             (int)(ios - io_threads),
+             boost::accumulators::rolling_sum(ios_acc) / time_sum,
+             boost::accumulators::rolling_sum(off_acc) / time_sum);
+      last = elapsed;
+    }
+  }
+  b.wait_for(0, false);
+
+  if (io_type != IO_TYPE_READ) {
+    r = image.flush();
+    if (r < 0) {
+      std::cerr << "rbd: failed to flush at the end: " << cpp_strerror(r)
+                << std::endl;
+    }
+  }
+
+  coarse_mono_time now = coarse_mono_clock::now();
+  chrono::duration<double> elapsed = now - start;
+
+  printf("elapsed: %5d  ops: %8d  ops/sec: %8.2lf  bytes/sec: %8.2lf\n",
+         (int)elapsed.count(), ios, (double)ios / elapsed.count(),
+         (double)off / elapsed.count());
+
+  if (io_type == IO_TYPE_RW) {
+    printf("read_ops: %5d   read_ops/sec: %8.2lf   read_bytes/sec: %8.2lf\n",
+           read_ops, (double)read_ops / elapsed.count(),
+           (double)read_ops * io_size / elapsed.count());
+
+    printf("write_ops: %5d   write_ops/sec: %8.2lf   write_bytes/sec: %8.2lf\n",
+           write_ops, (double)write_ops / elapsed.count(),
+           (double)write_ops * io_size / elapsed.count());
+  }
+
+  return 0;
+}
+
+void add_bench_common_options(po::options_description *positional,
+			      po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+
+  options->add_options()
+    ("io-size", po::value<Size>(), "IO size (in B/K/M/G/T) [default: 4K]")
+    ("io-threads", po::value<uint32_t>(), "ios in flight [default: 16]")
+    ("io-total", po::value<Size>(), "total size for IO (in B/K/M/G/T) [default: 1G]")
+    ("io-pattern", po::value<IOPattern>(), "IO pattern (rand or seq) [default: seq]")
+    ("rw-mix-read", po::value<uint64_t>(), "read proportion in readwrite (<= 100) [default: 50]");
+}
+
+void get_arguments_for_write(po::options_description *positional,
+                             po::options_description *options) {
+  add_bench_common_options(positional, options);
+}
+
+void get_arguments_for_bench(po::options_description *positional,
+                             po::options_description *options) {
+  add_bench_common_options(positional, options);
+
+  options->add_options()
+    ("io-type", po::value<IOType>()->required(), "IO type (read , write, or readwrite(rw))");
+}
+
+int bench_execute(const po::variables_map &vm, io_type_t bench_io_type) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  utils::SnapshotPresence snap_presence = utils::SNAPSHOT_PRESENCE_NONE;
+  if (bench_io_type == IO_TYPE_READ)
+    snap_presence = utils::SNAPSHOT_PRESENCE_PERMITTED;
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, snap_presence, utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  uint64_t bench_io_size;
+  if (vm.count("io-size")) {
+    bench_io_size = vm["io-size"].as<uint64_t>();
+  } else {
+    bench_io_size = 4096;
+  }
+  if (bench_io_size == 0) {
+    std::cerr << "rbd: --io-size should be greater than zero." << std::endl;
+    return -EINVAL;
+  }
+
+  uint32_t bench_io_threads;
+  if (vm.count("io-threads")) {
+    bench_io_threads = vm["io-threads"].as<uint32_t>();
+  } else {
+    bench_io_threads = 16;
+  }
+  if (bench_io_threads == 0) {
+    std::cerr << "rbd: --io-threads should be greater than zero." << std::endl;
+    return -EINVAL;
+  }
+
+  uint64_t bench_bytes;
+  if (vm.count("io-total")) {
+    bench_bytes = vm["io-total"].as<uint64_t>();
+  } else {
+    bench_bytes = 1 << 30;
+  }
+
+  bool bench_random;
+  if (vm.count("io-pattern")) {
+    bench_random = vm["io-pattern"].as<bool>();
+  } else {
+    bench_random = false;
+  }
+
+  uint64_t bench_read_proportion;
+  if (bench_io_type == IO_TYPE_READ) {
+    bench_read_proportion = 100;
+  } else if (bench_io_type == IO_TYPE_WRITE) {
+    bench_read_proportion = 0;
+  } else {
+    if (vm.count("rw-mix-read")) {
+      bench_read_proportion = vm["rw-mix-read"].as<uint64_t>();
+    } else {
+      bench_read_proportion = 50;
+    }
+
+    if (bench_read_proportion > 100) {
+      std::cerr << "rbd: --rw-mix-read should not be larger than 100." << std::endl;
+      return -EINVAL;
+    }
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "",
+                                 snap_name, false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  init_async_signal_handler();
+  register_async_signal_handler(SIGHUP, sighup_handler);
+  register_async_signal_handler_oneshot(SIGINT, handle_signal);
+  register_async_signal_handler_oneshot(SIGTERM, handle_signal);
+
+  r = do_bench(image, bench_io_type, bench_io_size, bench_io_threads,
+		     bench_bytes, bench_random, bench_read_proportion);
+
+  unregister_async_signal_handler(SIGHUP, sighup_handler);
+  unregister_async_signal_handler(SIGINT, handle_signal);
+  unregister_async_signal_handler(SIGTERM, handle_signal);
+  shutdown_async_signal_handler();
+
+  if (r < 0) {
+    std::cerr << "bench failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+int execute_for_write(const po::variables_map &vm,
+                      const std::vector<std::string> &ceph_global_init_args) {
+  std::cerr << "rbd: bench-write is deprecated, use rbd bench --io-type write ..." << std::endl;
+  return bench_execute(vm, IO_TYPE_WRITE);
+}
+
+int execute_for_bench(const po::variables_map &vm,
+                      const std::vector<std::string> &ceph_global_init_args) {
+  io_type_t bench_io_type;
+  if (vm.count("io-type")) {
+    bench_io_type = vm["io-type"].as<io_type_t>();
+  } else {
+    std::cerr << "rbd: --io-type must be specified." << std::endl;
+    return -EINVAL;
+  }
+
+  return bench_execute(vm, bench_io_type);
+}
+
+Shell::Action action_write(
+  {"bench-write"}, {}, "Simple write benchmark. (Deprecated, please use `rbd bench --io-type write` instead.)",
+		   "", &get_arguments_for_write, &execute_for_write, false);
+
+Shell::Action action_bench(
+  {"bench"}, {}, "Simple benchmark.", "", &get_arguments_for_bench, &execute_for_bench);
+
+} // namespace bench
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Children.cc b/src/tools/rbd/action/Children.cc
new file mode 100644
index 00000000..f459e92b
--- /dev/null
+++ b/src/tools/rbd/action/Children.cc
@@ -0,0 +1,166 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace children {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+int do_list_children(librados::IoCtx &io_ctx, librbd::Image &image,
+                     bool all_flag, bool descendants_flag, Formatter *f)
+{
+  std::vector<librbd::linked_image_spec_t> children;
+  librbd::RBD rbd;
+  int r;
+  if (descendants_flag) {
+    r = image.list_descendants(&children);
+  } else {
+    r = image.list_children3(&children);
+  }
+  if (r < 0)
+    return r;
+
+  if (f)
+    f->open_array_section("children");
+
+  for (auto& child : children) {
+    bool trash = child.trash;
+    if (f) {
+      if (all_flag) {
+        f->open_object_section("child");
+        f->dump_string("pool", child.pool_name);
+        f->dump_string("pool_namespace", child.pool_namespace);
+        f->dump_string("image", child.image_name);
+        f->dump_string("id", child.image_id);
+        f->dump_bool("trash", child.trash);
+        f->close_section();
+      } else if (!trash) {
+        f->open_object_section("child");
+        f->dump_string("pool", child.pool_name);
+        f->dump_string("pool_namespace", child.pool_namespace);
+        f->dump_string("image", child.image_name);
+        f->close_section();
+      }
+    } else if (all_flag || !trash) {
+      if (child.pool_name.empty()) {
+        std::cout << "(child missing " << child.pool_id << "/";
+      } else {
+        std::cout << child.pool_name << "/";
+      }
+      if (!child.pool_namespace.empty()) {
+        std::cout << child.pool_namespace << "/";
+      }
+      if (child.image_name.empty()) {
+        std::cout << child.image_id << ")";
+      } else {
+        std::cout << child.image_name;
+        if (trash) {
+          std::cout << " (trash " << child.image_id << ")";
+	}
+      }
+      std::cout << std::endl;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  }
+
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_NONE);
+  at::add_snap_id_option(options);
+  options->add_options()
+    ("all,a", po::bool_switch(), "list all children (include trash)");
+  options->add_options()
+    ("descendants", po::bool_switch(), "include all descendants");
+  at::add_format_options(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  uint64_t snap_id = LIBRADOS_SNAP_HEAD;
+  if (vm.count(at::SNAPSHOT_ID)) {
+    snap_id = vm[at::SNAPSHOT_ID].as<uint64_t>();
+  }
+
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  if (snap_id != LIBRADOS_SNAP_HEAD && !snap_name.empty()) {
+    std::cerr << "rbd: trying to access snapshot using both name and id."
+              << std::endl;
+    return -EINVAL;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  if (!snap_name.empty()) {
+    r = image.snap_set(snap_name.c_str());
+  } else if (snap_id != LIBRADOS_SNAP_HEAD) {
+    r = image.snap_set_by_id(snap_id);
+  }
+  if (r == -ENOENT) {
+    std::cerr << "rbd: snapshot does not exist." << std::endl;
+    return r;
+  } else if (r < 0) {
+    std::cerr << "rbd: error setting snapshot: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  r = do_list_children(io_ctx, image, vm["all"].as<bool>(),
+                       vm["descendants"].as<bool>(), formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: listing children failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"children"}, {}, "Display children of an image or its snapshot.", "",
+  &get_arguments, &execute);
+
+} // namespace children
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Clone.cc b/src/tools/rbd/action/Clone.cc
new file mode 100644
index 00000000..6406c957
--- /dev/null
+++ b/src/tools/rbd/action/Clone.cc
@@ -0,0 +1,99 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace clone {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+int do_clone(librbd::RBD &rbd, librados::IoCtx &p_ioctx,
+             const char *p_name, const char *p_snapname,
+             librados::IoCtx &c_ioctx, const char *c_name,
+             librbd::ImageOptions& opts) {
+  return rbd.clone3(p_ioctx, p_name, p_snapname, c_ioctx, c_name, opts);
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_snap_spec_options(positional, options, at::ARGUMENT_MODIFIER_SOURCE);
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
+  at::add_create_image_options(options, false);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_SOURCE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_REQUIRED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string dst_pool_name;
+  std::string dst_namespace_name;
+  std::string dst_image_name;
+  std::string dst_snap_name;
+  r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, &dst_pool_name,
+    &dst_namespace_name, &dst_image_name, &dst_snap_name, true,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::ImageOptions opts;
+  r = utils::get_image_options(vm, false, &opts);
+  if (r < 0) {
+    return r;
+  }
+  opts.set(RBD_IMAGE_OPTION_FORMAT, static_cast<uint64_t>(2));
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::IoCtx dst_io_ctx;
+  r = utils::init_io_ctx(rados, dst_pool_name, dst_namespace_name, &dst_io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = do_clone(rbd, io_ctx, image_name.c_str(), snap_name.c_str(), dst_io_ctx,
+               dst_image_name.c_str(), opts);
+  if (r == -EXDEV) {
+    std::cerr << "rbd: clone v2 required for cross-namespace clones."
+              << std::endl;
+    return r;
+  } else if (r < 0) {
+    std::cerr << "rbd: clone error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"clone"}, {}, "Clone a snapshot into a CoW child image.",
+  at::get_long_features_help(), &get_arguments, &execute);
+
+} // namespace clone
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Config.cc b/src/tools/rbd/action/Config.cc
new file mode 100644
index 00000000..2868c7ad
--- /dev/null
+++ b/src/tools/rbd/action/Config.cc
@@ -0,0 +1,890 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include "common/ceph_context.h"
+#include "common/ceph_json.h"
+#include "common/escape.h"
+#include "common/errno.h"
+#include "common/options.h"
+#include "global/global_context.h"
+#include "include/stringify.h"
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+
+#include <iostream>
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace config {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+namespace {
+
+const std::string METADATA_CONF_PREFIX = "conf_";
+const uint32_t MAX_KEYS = 64;
+
+void add_config_entity_option(
+    boost::program_options::options_description *positional) {
+  positional->add_options()
+    ("config-entity", "config entity (global, client, client.<id>)");
+}
+
+void add_pool_option(boost::program_options::options_description *positional) {
+  positional->add_options()
+    ("pool-name", "pool name");
+}
+
+void add_key_option(po::options_description *positional) {
+  positional->add_options()
+    ("key", "config key");
+}
+
+int get_config_entity(const po::variables_map &vm, std::string *config_entity) {
+  *config_entity = utils::get_positional_argument(vm, 0);
+
+  if (*config_entity != "global" && *config_entity != "client" &&
+      !boost::starts_with(*config_entity, ("client."))) {
+    std::cerr << "rbd: invalid config entity: " << *config_entity
+              << " (must be global, client or client.<id>)" << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+int get_pool(const po::variables_map &vm, std::string *pool_name) {
+  *pool_name = utils::get_positional_argument(vm, 0);
+  if (pool_name->empty()) {
+    std::cerr << "rbd: pool name was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+int get_key(const po::variables_map &vm, size_t *arg_index,
+            std::string *key) {
+  *key = utils::get_positional_argument(vm, *arg_index);
+  if (key->empty()) {
+    std::cerr << "rbd: config key was not specified" << std::endl;
+    return -EINVAL;
+  } else {
+    ++(*arg_index);
+  }
+
+  if (!boost::starts_with(*key, "rbd_")) {
+    std::cerr << "rbd: not rbd option: " << *key << std::endl;
+    return -EINVAL;
+  }
+
+  std::string value;
+  int r = g_ceph_context->_conf.get_val(key->c_str(), &value);
+  if (r < 0) {
+    std::cerr << "rbd: invalid config key: " << *key << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+std::ostream& operator<<(std::ostream& os,
+                         const librbd::config_source_t& source) {
+  switch (source) {
+  case RBD_CONFIG_SOURCE_CONFIG:
+    os << "config";
+    break;
+  case RBD_CONFIG_SOURCE_POOL:
+    os << "pool";
+    break;
+  case RBD_CONFIG_SOURCE_IMAGE:
+    os << "image";
+    break;
+  default:
+    os << "unknown (" << static_cast<uint32_t>(source) << ")";
+    break;
+  }
+  return os;
+}
+
+int config_global_list(
+    librados::Rados &rados, const std::string &config_entity,
+    std::map<std::string, std::pair<std::string, std::string>> *options) {
+  bool client_id_config_entity =
+    boost::starts_with(config_entity, ("client."));
+  std::string cmd =
+    "{"
+      "\"prefix\": \"config dump\", "
+      "\"format\": \"json\" "
+    "}";
+  bufferlist in_bl;
+  bufferlist out_bl;
+  std::string ss;
+  int r = rados.mon_command(cmd, in_bl, &out_bl, &ss);
+  if (r < 0) {
+    std::cerr << "rbd: error reading config: " << ss << std::endl;
+    return r;
+  }
+
+  json_spirit::mValue json_root;
+  if (!json_spirit::read(out_bl.to_str(), json_root)) {
+    std::cerr << "rbd: error parsing config dump" << std::endl;
+    return -EINVAL;
+  }
+
+  try {
+    auto &json_array = json_root.get_array();
+    for (auto& e : json_array) {
+      auto &json_obj = e.get_obj();
+      std::string section;
+      std::string name;
+      std::string value;
+
+      for (auto &pairs : json_obj) {
+        if (pairs.first == "section") {
+          section = pairs.second.get_str();
+        } else if (pairs.first == "name") {
+          name = pairs.second.get_str();
+        } else if (pairs.first == "value") {
+          value = pairs.second.get_str();
+        }
+      }
+
+      if (!boost::starts_with(name, "rbd_")) {
+        continue;
+      }
+      if (section != "global" && section != "client" &&
+          (!client_id_config_entity || section != config_entity)) {
+        continue;
+      }
+      if (config_entity == "global" && section != "global") {
+        continue;
+      }
+      auto it = options->find(name);
+      if (it == options->end()) {
+        (*options)[name] = {value, section};
+        continue;
+      }
+      if (section == "client") {
+        if (it->second.second == "global") {
+          it->second = {value, section};
+        }
+      } else if (client_id_config_entity) {
+        it->second = {value, section};
+      }
+    }
+  } catch (std::runtime_error &e) {
+    std::cerr << "rbd: error parsing config dump: " << e.what() << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+} // anonymous namespace
+
+void get_global_get_arguments(po::options_description *positional,
+                              po::options_description *options) {
+  add_config_entity_option(positional);
+  add_key_option(positional);
+}
+
+int execute_global_get(const po::variables_map &vm,
+                       const std::vector<std::string> &ceph_global_init_args) {
+  std::string config_entity;
+  int r = get_config_entity(vm, &config_entity);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  size_t arg_index = 1;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  r = utils::init_rados(&rados);
+  if (r < 0) {
+    return r;
+  }
+
+  std::map<std::string, std::pair<std::string, std::string>> options;
+  r = config_global_list(rados, config_entity, &options);
+  if (r < 0) {
+    return r;
+  }
+
+  auto it = options.find(key);
+
+  if (it == options.end() || it->second.second != config_entity) {
+    std::cerr << "rbd: " << key << " is not set" << std::endl;
+    return -ENOENT;
+  }
+
+  std::cout << it->second.first << std::endl;
+  return 0;
+}
+
+void get_global_set_arguments(po::options_description *positional,
+                              po::options_description *options) {
+  add_config_entity_option(positional);
+  add_key_option(positional);
+  positional->add_options()
+    ("value", "config value");
+}
+
+int execute_global_set(const po::variables_map &vm,
+                     const std::vector<std::string> &ceph_global_init_args) {
+  std::string config_entity;
+  int r = get_config_entity(vm, &config_entity);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  size_t arg_index = 1;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  r = utils::init_rados(&rados);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string value = utils::get_positional_argument(vm, 2);
+  std::string cmd =
+    "{"
+      "\"prefix\": \"config set\", "
+      "\"who\": \"" + stringify(json_stream_escaper(config_entity)) + "\", "
+      "\"name\": \"" + key + "\", "
+      "\"value\": \"" + stringify(json_stream_escaper(value)) + "\""
+    "}";
+  bufferlist in_bl;
+  std::string ss;
+  r = rados.mon_command(cmd, in_bl, nullptr, &ss);
+  if (r < 0) {
+    std::cerr << "rbd: error setting " << key << ": " << ss << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_global_remove_arguments(po::options_description *positional,
+                                 po::options_description *options) {
+  add_config_entity_option(positional);
+  add_key_option(positional);
+}
+
+int execute_global_remove(
+    const po::variables_map &vm,
+    const std::vector<std::string> &ceph_global_init_args) {
+  std::string config_entity;
+  int r = get_config_entity(vm, &config_entity);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  size_t arg_index = 1;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  r = utils::init_rados(&rados);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string cmd =
+    "{"
+      "\"prefix\": \"config rm\", "
+      "\"who\": \"" + stringify(json_stream_escaper(config_entity)) + "\", "
+      "\"name\": \"" + key + "\""
+    "}";
+  bufferlist in_bl;
+  std::string ss;
+  r = rados.mon_command(cmd, in_bl, nullptr, &ss);
+  if (r < 0) {
+    std::cerr << "rbd: error removing " << key << ": " << ss << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_global_list_arguments(po::options_description *positional,
+                             po::options_description *options) {
+  add_config_entity_option(positional);
+  at::add_format_options(options);
+}
+
+int execute_global_list(const po::variables_map &vm,
+                        const std::vector<std::string> &ceph_global_init_args) {
+  std::string config_entity;
+  int r = get_config_entity(vm, &config_entity);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter f;
+  r = utils::get_formatter(vm, &f);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  r = utils::init_rados(&rados);
+  if (r < 0) {
+    return r;
+  }
+
+  std::map<std::string, std::pair<std::string, std::string>> options;
+  r = config_global_list(rados, config_entity, &options);
+  if (r < 0) {
+    return r;
+  }
+
+  if (options.empty() && !f) {
+    return 0;
+  }
+
+  TextTable tbl;
+
+  if (f) {
+    f->open_array_section("config");
+  } else {
+    tbl.define_column("Name", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("Value", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("Section", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  for (const auto &it : options) {
+    if (f) {
+      f->open_object_section("option");
+      f->dump_string("name", it.first);
+      f->dump_string("value", it.second.first);
+      f->dump_string("section", it.second.second);
+      f->close_section();
+    } else {
+      tbl << it.first << it.second.first << it.second.second
+          << TextTable::endrow;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  } else {
+    std::cout << tbl;
+  }
+
+  return 0;
+}
+
+void get_pool_get_arguments(po::options_description *positional,
+                            po::options_description *options) {
+  add_pool_option(positional);
+  add_key_option(positional);
+}
+
+int execute_pool_get(const po::variables_map &vm,
+                     const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  int r = get_pool(vm, &pool_name);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  size_t arg_index = 1;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  std::string value;
+
+  r = rbd.pool_metadata_get(io_ctx, METADATA_CONF_PREFIX + key, &value);
+  if (r < 0) {
+    if (r == -ENOENT) {
+      std::cerr << "rbd: " << key << " is not set" << std::endl;
+    } else {
+      std::cerr << "rbd: failed to get " << key << ": " << cpp_strerror(r)
+                << std::endl;
+    }
+    return r;
+  }
+
+  std::cout << value << std::endl;
+  return 0;
+}
+
+void get_pool_set_arguments(po::options_description *positional,
+                            po::options_description *options) {
+  add_pool_option(positional);
+  add_key_option(positional);
+  positional->add_options()
+    ("value", "config value");
+}
+
+int execute_pool_set(const po::variables_map &vm,
+                     const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  int r = get_pool(vm, &pool_name);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  size_t arg_index = 1;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string value = utils::get_positional_argument(vm, 2);
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.pool_metadata_set(io_ctx, METADATA_CONF_PREFIX + key, value);
+  if (r < 0) {
+    std::cerr << "rbd: failed to set " << key << ": " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_pool_remove_arguments(po::options_description *positional,
+                               po::options_description *options) {
+  add_pool_option(positional);
+  add_key_option(positional);
+}
+
+int execute_pool_remove(const po::variables_map &vm,
+                        const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  int r = get_pool(vm, &pool_name);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  size_t arg_index = 1;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.pool_metadata_remove(io_ctx, METADATA_CONF_PREFIX + key);
+  if (r < 0) {
+    std::cerr << "rbd: failed to remove " << key << ": " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_pool_list_arguments(po::options_description *positional,
+                             po::options_description *options) {
+  add_pool_option(positional);
+  at::add_format_options(options);
+}
+
+int execute_pool_list(const po::variables_map &vm,
+                      const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  int r = get_pool(vm, &pool_name);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter f;
+  r = utils::get_formatter(vm, &f);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  TextTable tbl;
+  librbd::RBD rbd;
+  std::vector<librbd::config_option_t> options;
+
+  r = rbd.config_list(io_ctx, &options);
+  if (r < 0) {
+    std::cerr << "rbd: failed to list config: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  if (f) {
+    f->open_array_section("config");
+  } else {
+    tbl.define_column("Name", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("Value", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("Source", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  for (auto &option : options) {
+    if (f) {
+      f->open_object_section("option");
+      f->dump_string("name", option.name);
+      f->dump_string("value", option.value);
+      f->dump_stream("source") << option.source;
+      f->close_section();
+    } else {
+      std::ostringstream source;
+      source << option.source;
+      tbl << option.name << option.value << source.str() << TextTable::endrow;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  } else {
+    std::cout << tbl;
+  }
+
+  return 0;
+}
+
+void get_image_get_arguments(po::options_description *positional,
+                             po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  add_key_option(positional);
+}
+
+int execute_image_get(const po::variables_map &vm,
+                      const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string value;
+
+  r = image.metadata_get(METADATA_CONF_PREFIX + key, &value);
+  if (r < 0) {
+    if (r == -ENOENT) {
+      std::cerr << "rbd: " << key << " is not set" << std::endl;
+    } else {
+      std::cerr << "rbd: failed to get " << key << ": " << cpp_strerror(r)
+                << std::endl;
+    }
+    return r;
+  }
+
+  std::cout << value << std::endl;
+  return 0;
+}
+
+void get_image_set_arguments(po::options_description *positional,
+                             po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  add_key_option(positional);
+  positional->add_options()
+    ("value", "config value");
+}
+
+int execute_image_set(const po::variables_map &vm,
+                      const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string value = utils::get_positional_argument(vm, arg_index);
+  if (value.empty()) {
+    std::cerr << "rbd: image config value was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = image.metadata_set(METADATA_CONF_PREFIX + key, value);
+  if (r < 0) {
+    std::cerr << "rbd: failed to set " << key << ": " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_image_remove_arguments(po::options_description *positional,
+                                po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  add_key_option(positional);
+}
+
+int execute_image_remove(
+    const po::variables_map &vm,
+    const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = image.metadata_remove(METADATA_CONF_PREFIX + key);
+  if (r < 0) {
+    std::cerr << "rbd: failed to remove " << key << ": " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_image_list_arguments(po::options_description *positional,
+                              po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_format_options(options);
+}
+
+int execute_image_list(const po::variables_map &vm,
+                       const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter f;
+  r = utils::get_formatter(vm, &f);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  TextTable tbl;
+  std::vector<librbd::config_option_t> options;
+
+  r = image.config_list(&options);
+  if (r < 0) {
+    std::cerr << "rbd: failed to list config: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  if (options.empty()) {
+    if (f == nullptr) {
+      std::cout << "There are no values" << std::endl;
+    }
+    return 0;
+  }
+
+  if (f) {
+    f->open_array_section("config");
+  } else {
+    tbl.define_column("Name", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("Value", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("Source", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  for (auto &option : options) {
+    if (f) {
+      f->open_object_section("option");
+      f->dump_string("name", option.name);
+      f->dump_string("value", option.value);
+      f->dump_stream("source") << option.source;
+      f->close_section();
+    } else {
+      std::ostringstream source;
+      source << option.source;
+      tbl << option.name << option.value << source.str() << TextTable::endrow;
+    }
+  }
+
+  if (f == nullptr) {
+    bool single = (options.size() == 1);
+    std::cout << "There " << (single ? "is" : "are") << " " << options.size()
+              << " " << (single ? "value" : "values") << ":" << std::endl;
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  } else {
+    std::cout << tbl;
+  }
+
+  return 0;
+}
+
+Shell::Action action_global_get(
+  {"config", "global", "get"}, {},
+   "Get a global-level configuration override.", "",
+   &get_global_get_arguments, &execute_global_get);
+Shell::Action action_global_set(
+  {"config", "global", "set"}, {},
+   "Set a global-level configuration override.", "",
+   &get_global_set_arguments, &execute_global_set);
+Shell::Action action_global_remove(
+  {"config", "global", "remove"}, {"config", "global", "rm"},
+   "Remove a global-level configuration override.", "",
+   &get_global_remove_arguments, &execute_global_remove);
+Shell::Action action_global_list(
+  {"config", "global", "list"}, {"config", "global", "ls"},
+   "List global-level configuration overrides.", "",
+   &get_global_list_arguments, &execute_global_list);
+
+Shell::Action action_pool_get(
+  {"config", "pool", "get"}, {}, "Get a pool-level configuration override.", "",
+   &get_pool_get_arguments, &execute_pool_get);
+Shell::Action action_pool_set(
+  {"config", "pool", "set"}, {}, "Set a pool-level configuration override.", "",
+   &get_pool_set_arguments, &execute_pool_set);
+Shell::Action action_pool_remove(
+  {"config", "pool", "remove"}, {"config", "pool", "rm"},
+   "Remove a pool-level configuration override.", "",
+   &get_pool_remove_arguments, &execute_pool_remove);
+Shell::Action action_pool_list(
+  {"config", "pool", "list"}, {"config", "pool", "ls"},
+   "List pool-level configuration overrides.", "",
+   &get_pool_list_arguments, &execute_pool_list);
+
+Shell::Action action_image_get(
+  {"config", "image", "get"}, {}, "Get an image-level configuration override.",
+   "", &get_image_get_arguments, &execute_image_get);
+Shell::Action action_image_set(
+  {"config", "image", "set"}, {}, "Set an image-level configuration override.",
+   "", &get_image_set_arguments, &execute_image_set);
+Shell::Action action_image_remove(
+  {"config", "image", "remove"}, {"config", "image", "rm"},
+   "Remove an image-level configuration override.", "",
+   &get_image_remove_arguments, &execute_image_remove);
+Shell::Action action_image_list(
+  {"config", "image", "list"}, {"config", "image", "ls"},
+   "List image-level configuration overrides.", "",
+   &get_image_list_arguments, &execute_image_list);
+
+} // namespace config
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Copy.cc b/src/tools/rbd/action/Copy.cc
new file mode 100644
index 00000000..9a248437
--- /dev/null
+++ b/src/tools/rbd/action/Copy.cc
@@ -0,0 +1,195 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace copy {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_copy(librbd::Image &src, librados::IoCtx& dest_pp,
+		   const char *destname, librbd::ImageOptions& opts,
+		   bool no_progress,
+		   size_t sparse_size)
+{
+  utils::ProgressContext pc("Image copy", no_progress);
+  int r = src.copy_with_progress4(dest_pp, destname, opts, pc, sparse_size);
+  if (r < 0){
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_SOURCE);
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
+  at::add_create_image_options(options, false);
+  at::add_sparse_size_option(options);
+  at::add_no_progress_option(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_SOURCE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string dst_pool_name;
+  std::string dst_namespace_name;
+  std::string dst_image_name;
+  std::string dst_snap_name;
+  r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, &dst_pool_name,
+    &dst_namespace_name, &dst_image_name, &dst_snap_name, true,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::ImageOptions opts;
+  r = utils::get_image_options(vm, false, &opts);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "",
+                                 snap_name, true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::IoCtx dst_io_ctx;
+  r = utils::init_io_ctx(rados, dst_pool_name, dst_namespace_name, &dst_io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  size_t sparse_size = utils::RBD_DEFAULT_SPARSE_SIZE;
+  if (vm.count(at::IMAGE_SPARSE_SIZE)) {
+    sparse_size = vm[at::IMAGE_SPARSE_SIZE].as<size_t>();
+  }
+  r = do_copy(image, dst_io_ctx, dst_image_name.c_str(), opts,
+              vm[at::NO_PROGRESS].as<bool>(), sparse_size);
+  if (r < 0) {
+    std::cerr << "rbd: copy failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"copy"}, {"cp"}, "Copy src image to dest.", at::get_long_features_help(),
+  &get_arguments, &execute);
+
+static int do_deep_copy(librbd::Image &src, librados::IoCtx& dest_pp,
+		        const char *destname, librbd::ImageOptions& opts,
+		        bool no_progress)
+{
+  utils::ProgressContext pc("Image deep copy", no_progress);
+  int r = src.deep_copy_with_progress(dest_pp, destname, opts, pc);
+  if (r < 0){
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+void get_arguments_deep(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_SOURCE);
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
+  at::add_create_image_options(options, false);
+  at::add_flatten_option(options);
+  at::add_no_progress_option(options);
+}
+
+int execute_deep(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_SOURCE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string dst_pool_name;
+  std::string dst_namespace_name;
+  std::string dst_image_name;
+  std::string dst_snap_name;
+  r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, &dst_pool_name,
+    &dst_namespace_name, &dst_image_name, &dst_snap_name, true,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::ImageOptions opts;
+  r = utils::get_image_options(vm, false, &opts);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "",
+                                 snap_name, true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::IoCtx dst_io_ctx;
+  r = utils::init_io_ctx(rados, dst_pool_name, dst_namespace_name, &dst_io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_deep_copy(image, dst_io_ctx, dst_image_name.c_str(), opts,
+                   vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: deep copy failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action_deep(
+  {"deep", "copy"}, {"deep", "cp"}, "Deep copy src image to dest.",
+  at::get_long_features_help(), &get_arguments_deep, &execute_deep);
+
+} // namespace copy
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Create.cc b/src/tools/rbd/action/Create.cc
new file mode 100644
index 00000000..99efa0b5
--- /dev/null
+++ b/src/tools/rbd/action/Create.cc
@@ -0,0 +1,264 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+#include "common/Cond.h"
+#include "common/Mutex.h"
+
+namespace rbd {
+namespace action {
+namespace create {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_create(librbd::RBD &rbd, librados::IoCtx& io_ctx,
+                     const char *imgname, uint64_t size,
+		     librbd::ImageOptions& opts) {
+  return rbd.create4(io_ctx, imgname, size, opts);
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_create_image_options(options, true);
+  options->add_options()
+    (at::IMAGE_THICK_PROVISION.c_str(), po::bool_switch(), "fully allocate storage and zero image");
+  at::add_size_option(options);
+  at::add_no_progress_option(options);
+}
+
+void thick_provision_writer_completion(rbd_completion_t, void *);
+
+struct thick_provision_writer {
+  librbd::Image *image;
+  Mutex lock;
+  Cond cond;
+  bufferlist bl;
+  uint64_t chunk_size;
+  const int block_size;
+  uint64_t concurr;
+  struct {
+    uint64_t in_flight;
+    int io_error;
+  } io_status;
+
+  // Constructor
+  explicit thick_provision_writer(librbd::Image *i, librbd::ImageOptions &o)
+    : image(i),
+      lock("thick_provision_writer::lock"),
+      block_size(512) // 512 Bytes
+  {
+    // If error cases occur, the code is aborted, because
+    // constructor cannot return error value.
+    ceph_assert(g_ceph_context != nullptr);
+    bl.append_zero(block_size);
+
+    librbd::image_info_t info;
+    int r = image->stat(info, sizeof(info));
+    ceph_assert(r >= 0);
+    uint64_t order;
+    if (info.order == 0) {
+      order = g_conf().get_val<uint64_t>("rbd_default_order");
+    } else {
+      order = info.order;
+    }
+    chunk_size = (1ull << order);
+    if (image->get_stripe_unit() < chunk_size) {
+      chunk_size = image->get_stripe_unit();
+    }
+
+    concurr = g_conf().get_val<uint64_t>("rbd_concurrent_management_ops");
+    io_status.in_flight = 0;
+    io_status.io_error = 0;
+  }
+
+  int start_io(uint64_t write_offset)
+  {
+    {
+      Mutex::Locker l(lock);
+      io_status.in_flight++;
+      if (io_status.in_flight > concurr) {
+        io_status.in_flight--;
+        return -EINVAL;
+      }
+    }
+
+    librbd::RBD::AioCompletion *c;
+    c = new librbd::RBD::AioCompletion(this, thick_provision_writer_completion);
+    int r;
+    r = image->aio_writesame(write_offset, chunk_size, bl, c, LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL);
+    if (r < 0) {
+      Mutex::Locker l(lock);
+      io_status.io_error = r;
+    }
+    return r;
+  }
+
+  int wait_for(uint64_t max) {
+    Mutex::Locker l(lock);
+    int r = io_status.io_error;
+
+    while (io_status.in_flight > max) {
+      utime_t dur;
+      dur.set_from_double(.2);
+      cond.WaitInterval(lock, dur);
+    }
+    return r;
+  }
+};
+
+void thick_provision_writer_completion(rbd_completion_t rc, void *pc) {
+  librbd::RBD::AioCompletion *ac = (librbd::RBD::AioCompletion *)rc;
+  thick_provision_writer *tc = static_cast<thick_provision_writer *>(pc);
+
+  int r = ac->get_return_value();
+  tc->lock.Lock();
+  if (r < 0 &&  tc->io_status.io_error >= 0) {
+    tc->io_status.io_error = r;
+  }
+  tc->io_status.in_flight--;
+  tc->cond.Signal();
+  tc->lock.Unlock();
+  ac->release();
+}
+
+int write_data(librbd::Image &image, librbd::ImageOptions &opts,
+               bool no_progress) {
+  uint64_t image_size;
+  int r = 0;
+  utils::ProgressContext pc("Thick provisioning", no_progress);
+
+  if (image.size(&image_size) != 0) {
+    return -EINVAL;
+  }
+
+  thick_provision_writer tpw(&image, opts);
+  uint64_t off;
+  uint64_t i;
+  for (off = 0; off < image_size;) {
+    i = 0;
+    while (i < tpw.concurr && off < image_size) {
+      tpw.wait_for(tpw.concurr - 1);
+      r = tpw.start_io(off);
+      if (r != 0) {
+        goto err_writesame;
+      }
+      ++i;
+      off += tpw.chunk_size;
+      if(off > image_size) {
+        off = image_size;
+      }
+      pc.update_progress(off, image_size);
+    }
+  }
+
+  tpw.wait_for(0);
+  r = image.flush();
+  if (r < 0) {
+    std::cerr << "rbd: failed to flush at the end: " << cpp_strerror(r)
+              << std::endl;
+    goto err_writesame;
+  }
+  pc.finish();
+
+  return r;
+
+err_writesame:
+  tpw.wait_for(0);
+  pc.fail();
+
+  return r;
+}
+
+int thick_write(const std::string &image_name,librados::IoCtx &io_ctx,
+                librbd::ImageOptions &opts, bool no_progress) {
+  int r = 0;
+  librbd::Image image;
+
+  // To prevent writesame from discarding data, thick_write sets
+  // the rbd_discard_on_zeroed_write_same option to false.
+  ceph_assert(g_ceph_context != nullptr);
+  r = g_conf().set_val("rbd_discard_on_zeroed_write_same", "false");
+  ceph_assert(r == 0);
+  r = utils::open_image(io_ctx, image_name, false, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = write_data(image, opts, no_progress);
+
+  image.close();
+
+  return r;
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::ImageOptions opts;
+  r = utils::get_image_options(vm, true, &opts);
+  if (r < 0) {
+    return r;
+  }
+
+  uint64_t size;
+  r = utils::get_image_size(vm, &size);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = do_create(rbd, io_ctx, image_name.c_str(), size, opts);
+  if (!namespace_name.empty() && r == -ENOENT) {
+    std::cerr << "rbd: namespace not found - it must be created with "
+              << "'rbd namespace create' before creating an image."
+              << std::endl;
+    return r;
+  } else if (r < 0) {
+    std::cerr << "rbd: create error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  if (vm.count(at::IMAGE_THICK_PROVISION) && vm[at::IMAGE_THICK_PROVISION].as<bool>()) {
+    r = thick_write(image_name, io_ctx, opts, vm[at::NO_PROGRESS].as<bool>());
+    if (r < 0) {
+      std::cerr << "rbd: image created but error encountered during thick provisioning: "
+                << cpp_strerror(r) << std::endl;
+      return r;
+    }
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"create"}, {}, "Create an empty image.", at::get_long_features_help(),
+  &get_arguments, &execute);
+
+} // namespace create
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Device.cc b/src/tools/rbd/action/Device.cc
new file mode 100644
index 00000000..3fdf2ef5
--- /dev/null
+++ b/src/tools/rbd/action/Device.cc
@@ -0,0 +1,185 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "acconfig.h"
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+
+#include <boost/program_options.hpp>
+
+#include "include/ceph_assert.h"
+
+namespace rbd {
+namespace action {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+#define DECLARE_DEVICE_OPERATIONS(ns)                                   \
+  namespace ns {                                                        \
+  int execute_list(const po::variables_map &vm,                         \
+                   const std::vector<std::string> &ceph_global_args);   \
+  int execute_map(const po::variables_map &vm,                          \
+                  const std::vector<std::string> &ceph_global_args);    \
+  int execute_unmap(const po::variables_map &vm,                        \
+                    const std::vector<std::string> &ceph_global_args);  \
+  }
+
+DECLARE_DEVICE_OPERATIONS(ggate);
+DECLARE_DEVICE_OPERATIONS(kernel);
+DECLARE_DEVICE_OPERATIONS(nbd);
+
+namespace device {
+
+namespace {
+
+struct DeviceOperations {
+  int (*execute_list)(const po::variables_map &vm,
+                      const std::vector<std::string> &ceph_global_args);
+  int (*execute_map)(const po::variables_map &vm,
+                     const std::vector<std::string> &ceph_global_args);
+  int (*execute_unmap)(const po::variables_map &vm,
+                       const std::vector<std::string> &ceph_global_args);
+};
+
+const DeviceOperations ggate_operations = {
+  ggate::execute_list,
+  ggate::execute_map,
+  ggate::execute_unmap,
+};
+
+const DeviceOperations krbd_operations = {
+  kernel::execute_list,
+  kernel::execute_map,
+  kernel::execute_unmap,
+};
+
+const DeviceOperations nbd_operations = {
+  nbd::execute_list,
+  nbd::execute_map,
+  nbd::execute_unmap,
+};
+
+enum device_type_t {
+  DEVICE_TYPE_GGATE,
+  DEVICE_TYPE_KRBD,
+  DEVICE_TYPE_NBD,
+};
+
+struct DeviceType {};
+
+void validate(boost::any& v, const std::vector<std::string>& values,
+              DeviceType *target_type, int) {
+  po::validators::check_first_occurrence(v);
+  const std::string &s = po::validators::get_single_string(values);
+  if (s == "ggate") {
+    v = boost::any(DEVICE_TYPE_GGATE);
+  } else if (s == "krbd") {
+    v = boost::any(DEVICE_TYPE_KRBD);
+  } else if (s == "nbd") {
+    v = boost::any(DEVICE_TYPE_NBD);
+  } else {
+    throw po::validation_error(po::validation_error::invalid_option_value);
+  }
+}
+
+void add_device_type_option(po::options_description *options) {
+  options->add_options()
+    ("device-type,t", po::value<DeviceType>(),
+     "device type [ggate, krbd (default), nbd]");
+}
+
+void add_device_specific_options(po::options_description *options) {
+  options->add_options()
+    ("options,o", po::value<std::vector<std::string>>(),
+     "device specific options");
+}
+
+device_type_t get_device_type(const po::variables_map &vm) {
+  if (vm.count("device-type")) {
+    return vm["device-type"].as<device_type_t>();
+  }
+  return DEVICE_TYPE_KRBD;
+}
+
+const DeviceOperations *get_device_operations(const po::variables_map &vm) {
+  switch (get_device_type(vm)) {
+  case DEVICE_TYPE_GGATE:
+    return &ggate_operations;
+  case DEVICE_TYPE_KRBD:
+    return &krbd_operations;
+  case DEVICE_TYPE_NBD:
+    return &nbd_operations;
+  default:
+    ceph_abort();
+    return nullptr;
+  }
+}
+
+} // anonymous namespace
+
+void get_list_arguments(po::options_description *positional,
+                        po::options_description *options) {
+  add_device_type_option(options);
+  at::add_format_options(options);
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  return (*get_device_operations(vm)->execute_list)(vm, ceph_global_init_args);
+}
+
+void get_map_arguments(po::options_description *positional,
+                       po::options_description *options) {
+  add_device_type_option(options);
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_NONE);
+  options->add_options()
+    ("read-only", po::bool_switch(), "map read-only")
+    ("exclusive", po::bool_switch(), "disable automatic exclusive lock transitions");
+  add_device_specific_options(options);
+}
+
+int execute_map(const po::variables_map &vm,
+                const std::vector<std::string> &ceph_global_init_args) {
+  return (*get_device_operations(vm)->execute_map)(vm, ceph_global_init_args);
+}
+
+void get_unmap_arguments(po::options_description *positional,
+                         po::options_description *options) {
+  add_device_type_option(options);
+  positional->add_options()
+    ("image-or-snap-or-device-spec",
+     "image, snapshot, or device specification\n"
+     "[<pool-name>/]<image-name>[@<snapshot-name>] or <device-path>");
+  at::add_pool_option(options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_snap_option(options, at::ARGUMENT_MODIFIER_NONE);
+  add_device_specific_options(options);
+}
+
+int execute_unmap(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+  return (*get_device_operations(vm)->execute_unmap)(vm, ceph_global_init_args);
+}
+
+Shell::SwitchArguments switched_arguments({"read-only", "exclusive"});
+Shell::Action action_list(
+  {"device", "list"}, {"showmapped"}, "List mapped rbd images.", "",
+  &get_list_arguments, &execute_list);
+// yet another alias for list command
+Shell::Action action_ls(
+  {"device", "ls"}, {}, "List mapped rbd images.", "",
+  &get_list_arguments, &execute_list, false);
+
+Shell::Action action_map(
+  {"device", "map"}, {"map"}, "Map an image to a block device.", "",
+  &get_map_arguments, &execute_map);
+
+Shell::Action action_unmap(
+  {"device", "unmap"}, {"unmap"}, "Unmap a rbd device.", "",
+  &get_unmap_arguments, &execute_unmap);
+
+} // namespace device
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Diff.cc b/src/tools/rbd/action/Diff.cc
new file mode 100644
index 00000000..3729469c
--- /dev/null
+++ b/src/tools/rbd/action/Diff.cc
@@ -0,0 +1,143 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace diff {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+struct output_method {
+  output_method() : f(NULL), t(NULL), empty(true) {}
+  Formatter *f;
+  TextTable *t;
+  bool empty;
+};
+
+static int diff_cb(uint64_t ofs, size_t len, int exists, void *arg)
+{
+  output_method *om = static_cast<output_method *>(arg);
+  om->empty = false;
+  if (om->f) {
+    om->f->open_object_section("extent");
+    om->f->dump_unsigned("offset", ofs);
+    om->f->dump_unsigned("length", len);
+    om->f->dump_string("exists", exists ? "true" : "false");
+    om->f->close_section();
+  } else {
+    ceph_assert(om->t);
+    *(om->t) << ofs << len << (exists ? "data" : "zero") << TextTable::endrow;
+  }
+  return 0;
+}
+
+static int do_diff(librbd::Image& image, const char *fromsnapname,
+                   bool whole_object, Formatter *f)
+{
+  int r;
+  librbd::image_info_t info;
+
+  r = image.stat(info, sizeof(info));
+  if (r < 0)
+    return r;
+
+  output_method om;
+  if (f) {
+    om.f = f;
+    f->open_array_section("extents");
+  } else {
+    om.t = new TextTable();
+    om.t->define_column("Offset", TextTable::LEFT, TextTable::LEFT);
+    om.t->define_column("Length", TextTable::LEFT, TextTable::LEFT);
+    om.t->define_column("Type", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  r = image.diff_iterate2(fromsnapname, 0, info.size, true, whole_object,
+                          diff_cb, &om);
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  } else {
+    if (!om.empty)
+      std::cout << *om.t;
+    delete om.t;
+  }
+  return r;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_NONE);
+  options->add_options()
+    (at::FROM_SNAPSHOT_NAME.c_str(), po::value<std::string>(),
+     "snapshot starting point")
+    (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object");
+  at::add_format_options(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string from_snap_name;
+  if (vm.count(at::FROM_SNAPSHOT_NAME)) {
+    from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as<std::string>();
+  }
+
+  bool diff_whole_object = vm[at::WHOLE_OBJECT].as<bool>();
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "",
+                                 snap_name, true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_diff(image, from_snap_name.empty() ? nullptr : from_snap_name.c_str(),
+              diff_whole_object, formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: diff error: " << cpp_strerror(r) << std::endl;
+    return -r;
+  }
+  return 0;
+}
+
+Shell::SwitchArguments switched_arguments({at::WHOLE_OBJECT});
+Shell::Action action(
+  {"diff"}, {},
+  "Print extents that differ since a previous snap, or image creation.", "",
+  &get_arguments, &execute);
+
+} // namespace diff
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/DiskUsage.cc b/src/tools/rbd/action/DiskUsage.cc
new file mode 100644
index 00000000..649f39a7
--- /dev/null
+++ b/src/tools/rbd/action/DiskUsage.cc
@@ -0,0 +1,341 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/types.h"
+#include "include/stringify.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include <algorithm>
+#include <iostream>
+#include <boost/bind.hpp>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace disk_usage {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int disk_usage_callback(uint64_t offset, size_t len, int exists,
+                               void *arg) {
+  uint64_t *used_size = reinterpret_cast<uint64_t *>(arg);
+  if (exists) {
+    (*used_size) += len;
+  }
+  return 0;
+}
+
+static int compute_image_disk_usage(const std::string& name,
+                                    const std::string& snap_name,
+                                    const std::string& from_snap_name,
+                                    librbd::Image &image, uint64_t size,
+                                    bool exact, TextTable& tbl, Formatter *f,
+                                    uint64_t *used_size) {
+  const char* from = NULL;
+  if (!from_snap_name.empty()) {
+    from = from_snap_name.c_str();
+  }
+
+  uint64_t flags;
+  int r = image.get_flags(&flags);
+  if (r < 0) {
+    std::cerr << "rbd: failed to retrieve image flags: " << cpp_strerror(r)
+         << std::endl;
+    return r;
+  }
+  if ((flags & RBD_FLAG_FAST_DIFF_INVALID) != 0) {
+    std::cerr << "warning: fast-diff map is invalid for " << name
+         << (snap_name.empty() ? "" : "@" + snap_name) << ". "
+         << "operation may be slow." << std::endl;
+  }
+
+  *used_size = 0;
+  r = image.diff_iterate2(from, 0, size, false, !exact,
+                          &disk_usage_callback, used_size);
+  if (r < 0) {
+    std::cerr << "rbd: failed to iterate diffs: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  if (f) {
+    f->open_object_section("image");
+    f->dump_string("name", name);
+    if (!snap_name.empty()) {
+      f->dump_string("snapshot", snap_name);
+    }
+    f->dump_unsigned("provisioned_size", size);
+    f->dump_unsigned("used_size" , *used_size);
+    f->close_section();
+  } else {
+    std::string full_name = name;
+    if (!snap_name.empty()) {
+      full_name += "@" + snap_name;
+    }
+    tbl << full_name
+        << stringify(byte_u_t(size))
+        << stringify(byte_u_t(*used_size))
+        << TextTable::endrow;
+  }
+  return 0;
+}
+
+static int do_disk_usage(librbd::RBD &rbd, librados::IoCtx &io_ctx,
+                         const char *imgname, const char *snapname,
+                         const char *from_snapname, bool exact, Formatter *f) {
+  std::vector<librbd::image_spec_t> images;
+  int r = rbd.list2(io_ctx, &images);
+  if (r == -ENOENT) {
+    r = 0;
+  } else if (r < 0) {
+    return r;
+  }
+
+  TextTable tbl;
+  if (f) {
+    f->open_object_section("stats");
+    f->open_array_section("images");
+  } else {
+    tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("PROVISIONED", TextTable::LEFT, TextTable::RIGHT);
+    tbl.define_column("USED", TextTable::LEFT, TextTable::RIGHT);
+  }
+
+  uint32_t count = 0;
+  uint64_t used_size = 0;
+  uint64_t total_prov = 0;
+  uint64_t total_used = 0;
+  uint64_t snap_id = CEPH_NOSNAP;
+  uint64_t from_id = CEPH_NOSNAP;
+  bool found = false;
+  for (auto& image_spec : images) {
+    if (imgname != NULL && image_spec.name != imgname) {
+      continue;
+    }
+    found = true;
+
+    librbd::Image image;
+    r = rbd.open_read_only(io_ctx, image, image_spec.name.c_str(), NULL);
+    if (r < 0) {
+      if (r != -ENOENT) {
+        std::cerr << "rbd: error opening " << image_spec.name << ": "
+                  << cpp_strerror(r) << std::endl;
+      }
+      continue;
+    }
+
+    uint64_t features;
+    r = image.features(&features);
+    if (r < 0) {
+      std::cerr << "rbd: failed to retrieve image features: " << cpp_strerror(r)
+                << std::endl;
+      goto out;
+    }
+    if ((features & RBD_FEATURE_FAST_DIFF) == 0) {
+      std::cerr << "warning: fast-diff map is not enabled for "
+                << image_spec.name << ". " << "operation may be slow."
+                << std::endl;
+    }
+
+    librbd::image_info_t info;
+    if (image.stat(info, sizeof(info)) < 0) {
+      r = -EINVAL;
+      goto out;
+    }
+
+    std::vector<librbd::snap_info_t> snap_list;
+    r = image.snap_list(snap_list);
+    if (r < 0) {
+      std::cerr << "rbd: error opening " << image_spec.name << " snapshots: "
+                << cpp_strerror(r) << std::endl;
+      continue;
+    }
+
+    snap_list.erase(remove_if(snap_list.begin(),
+                              snap_list.end(),
+                              boost::bind(utils::is_not_user_snap_namespace, &image, _1)),
+                    snap_list.end());
+
+    bool found_from_snap = (from_snapname == nullptr);
+    bool found_snap = (snapname == nullptr);
+    bool found_from = (from_snapname == nullptr);
+    std::string last_snap_name;
+    std::sort(snap_list.begin(), snap_list.end(),
+              boost::bind(&librbd::snap_info_t::id, _1) <
+                boost::bind(&librbd::snap_info_t::id, _2));
+    if (!found_snap || !found_from) {
+      for (auto &snap_info : snap_list) {
+        if (!found_snap && snap_info.name == snapname) {
+          snap_id = snap_info.id;
+          found_snap = true;
+        }
+        if (!found_from && snap_info.name == from_snapname) {
+          from_id = snap_info.id;
+          found_from = true;
+        }
+        if (found_snap && found_from) {
+          break;
+        }
+      }
+    }
+    if ((snapname != nullptr && snap_id == CEPH_NOSNAP) ||
+        (from_snapname != nullptr && from_id == CEPH_NOSNAP)) {
+      std::cerr << "specified snapshot is not found." << std::endl;
+      return -ENOENT;
+    }
+    if (snap_id != CEPH_NOSNAP && from_id != CEPH_NOSNAP) {
+      if (from_id == snap_id) {
+        // no diskusage.
+        return 0;
+      }
+      if (from_id >= snap_id) {
+        return -EINVAL;
+      }
+    }
+
+    for (std::vector<librbd::snap_info_t>::const_iterator snap =
+         snap_list.begin(); snap != snap_list.end(); ++snap) {
+      librbd::Image snap_image;
+      r = rbd.open_read_only(io_ctx, snap_image, image_spec.name.c_str(),
+                             snap->name.c_str());
+      if (r < 0) {
+        std::cerr << "rbd: error opening snapshot " << image_spec.name << "@"
+                  << snap->name << ": " << cpp_strerror(r) << std::endl;
+        goto out;
+      }
+
+      if (imgname == nullptr || found_from_snap ||
+         (found_from_snap && snapname != nullptr && snap->name == snapname)) {
+        r = compute_image_disk_usage(image_spec.name, snap->name,
+                                     last_snap_name, snap_image, snap->size,
+                                     exact, tbl, f, &used_size);
+        if (r < 0) {
+          goto out;
+        }
+
+        if (snapname != NULL) {
+          total_prov += snap->size;
+        }
+        total_used += used_size;
+        ++count;
+      }
+
+      if (!found_from_snap && from_snapname != nullptr &&
+          snap->name == from_snapname) {
+        found_from_snap = true;
+      }
+      if (snapname != nullptr && snap->name == snapname) {
+        break;
+      }
+      last_snap_name = snap->name;
+    }
+
+    if (snapname == NULL) {
+      r = compute_image_disk_usage(image_spec.name, "", last_snap_name, image,
+                                   info.size, exact, tbl, f, &used_size);
+      if (r < 0) {
+        goto out;
+      }
+      total_prov += info.size;
+      total_used += used_size;
+      ++count;
+    }
+  }
+  if (imgname != nullptr && !found) {
+    std::cerr << "specified image " << imgname << " is not found." << std::endl;
+    return -ENOENT;
+  }
+
+out:
+  if (f) {
+    f->close_section();
+    if (imgname == NULL) {
+      f->dump_unsigned("total_provisioned_size", total_prov);
+      f->dump_unsigned("total_used_size", total_used);
+    }
+    f->close_section();
+    f->flush(std::cout);
+  } else if (!images.empty()) {
+    if (count > 1) {
+      tbl << "<TOTAL>"
+          << stringify(byte_u_t(total_prov))
+          << stringify(byte_u_t(total_used))
+          << TextTable::endrow;
+    }
+    std::cout << tbl;
+  }
+
+  return r < 0 ? r : 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_NONE);
+  at::add_format_options(options);
+  options->add_options()
+    (at::FROM_SNAPSHOT_NAME.c_str(), po::value<std::string>(),
+     "snapshot starting point")
+    ("exact", po::bool_switch(), "compute exact disk usage (slow)");
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, vm.count(at::FROM_SNAPSHOT_NAME),
+    utils::SNAPSHOT_PRESENCE_PERMITTED, utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string from_snap_name;
+  if (vm.count(at::FROM_SNAPSHOT_NAME)) {
+    from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as<std::string>();
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  utils::disable_cache();
+
+  librbd::RBD rbd;
+  r = do_disk_usage(rbd, io_ctx,
+                    image_name.empty() ? nullptr: image_name.c_str(),
+                    snap_name.empty() ? nullptr : snap_name.c_str(),
+                    from_snap_name.empty() ? nullptr : from_snap_name.c_str(),
+                    vm["exact"].as<bool>(), formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: du failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"disk-usage"}, {"du"}, "Show disk usage stats for pool, image or snapshot.",
+  "", &get_arguments, &execute);
+
+} // namespace disk_usage
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc
new file mode 100644
index 00000000..b5b82f4c
--- /dev/null
+++ b/src/tools/rbd/action/Export.cc
@@ -0,0 +1,651 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/compat.h"
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/Context.h"
+#include "common/errno.h"
+#include "common/Throttle.h"
+#include "include/encoding.h"
+#include <iostream>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <boost/program_options.hpp>
+#include <boost/scope_exit.hpp>
+
+namespace rbd {
+namespace action {
+namespace export_full {
+
+struct ExportDiffContext {
+  librbd::Image *image;
+  int fd;
+  int export_format;
+  uint64_t totalsize;
+  utils::ProgressContext pc;
+  OrderedThrottle throttle;
+
+  ExportDiffContext(librbd::Image *i, int f, uint64_t t, int max_ops,
+                    bool no_progress, int eformat) :
+    image(i), fd(f), export_format(eformat), totalsize(t), pc("Exporting image", no_progress),
+    throttle(max_ops, true) {
+  }
+};
+
+class C_ExportDiff : public Context {
+public:
+  C_ExportDiff(ExportDiffContext *edc, uint64_t offset, uint64_t length,
+               bool exists, int export_format)
+    : m_export_diff_context(edc), m_offset(offset), m_length(length),
+      m_exists(exists), m_export_format(export_format) {
+  }
+
+  int send() {
+    if (m_export_diff_context->throttle.pending_error()) {
+      return m_export_diff_context->throttle.wait_for_ret();
+    }
+
+    C_OrderedThrottle *ctx = m_export_diff_context->throttle.start_op(this);
+    if (m_exists) {
+      librbd::RBD::AioCompletion *aio_completion =
+        new librbd::RBD::AioCompletion(ctx, &utils::aio_context_callback);
+
+      int op_flags = LIBRADOS_OP_FLAG_FADVISE_NOCACHE;
+      int r = m_export_diff_context->image->aio_read2(
+        m_offset, m_length, m_read_data, aio_completion, op_flags);
+      if (r < 0) {
+        aio_completion->release();
+        ctx->complete(r);
+      }
+    } else {
+      ctx->complete(0);
+    }
+    return 0;
+  }
+
+  static int export_diff_cb(uint64_t offset, size_t length, int exists,
+			    void *arg) {
+    ExportDiffContext *edc = reinterpret_cast<ExportDiffContext *>(arg);
+
+    C_ExportDiff *context = new C_ExportDiff(edc, offset, length, exists, edc->export_format);
+    return context->send();
+  }
+
+protected:
+  void finish(int r) override {
+    if (r >= 0) {
+      if (m_exists) {
+        m_exists = !m_read_data.is_zero();
+      }
+      r = write_extent(m_export_diff_context, m_offset, m_length, m_exists, m_export_format);
+      if (r == 0 && m_exists) {
+        r = m_read_data.write_fd(m_export_diff_context->fd);
+      }
+    }
+    m_export_diff_context->throttle.end_op(r);
+  }
+
+private:
+  ExportDiffContext *m_export_diff_context;
+  uint64_t m_offset;
+  uint64_t m_length;
+  bool m_exists;
+  int m_export_format;
+  bufferlist m_read_data;
+
+  static int write_extent(ExportDiffContext *edc, uint64_t offset,
+                          uint64_t length, bool exists, int export_format) {
+    // extent
+    bufferlist bl;
+    __u8 tag = exists ? RBD_DIFF_WRITE : RBD_DIFF_ZERO;
+    uint64_t len = 0;
+    encode(tag, bl);
+    if (export_format == 2) {
+      if (tag == RBD_DIFF_WRITE)
+	len = 8 + 8 + length;
+      else
+	len = 8 + 8;
+      encode(len, bl);
+    }
+    encode(offset, bl);
+    encode(length, bl);
+    int r = bl.write_fd(edc->fd);
+
+    edc->pc.update_progress(offset, edc->totalsize);
+    return r;
+  }
+};
+
+
+int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
+		   const char *endsnapname, bool whole_object,
+		   int fd, bool no_progress, int export_format)
+{
+  int r;
+  librbd::image_info_t info;
+
+  r = image.stat(info, sizeof(info));
+  if (r < 0)
+    return r;
+
+  {
+    // header
+    bufferlist bl;
+    if (export_format == 1)
+      bl.append(utils::RBD_DIFF_BANNER);
+    else
+      bl.append(utils::RBD_DIFF_BANNER_V2);
+
+    __u8 tag;
+    uint64_t len = 0;
+    if (fromsnapname) {
+      tag = RBD_DIFF_FROM_SNAP;
+      encode(tag, bl);
+      std::string from(fromsnapname);
+      if (export_format == 2) {
+	len = from.length() + 4;
+	encode(len, bl);
+      }
+      encode(from, bl);
+    }
+
+    if (endsnapname) {
+      tag = RBD_DIFF_TO_SNAP;
+      encode(tag, bl);
+      std::string to(endsnapname);
+      if (export_format == 2) {
+        len = to.length() + 4;
+        encode(len, bl);
+      }
+      encode(to, bl);
+    }
+
+    if (endsnapname && export_format == 2) {
+      tag = RBD_SNAP_PROTECTION_STATUS;
+      encode(tag, bl);
+      bool is_protected = false;
+      r = image.snap_is_protected(endsnapname, &is_protected);
+      if (r < 0) {
+        return r;
+      }
+      len = 8;
+      encode(len, bl);
+      encode(is_protected, bl);
+    }
+
+    tag = RBD_DIFF_IMAGE_SIZE;
+    encode(tag, bl);
+    uint64_t endsize = info.size;
+    if (export_format == 2) {
+      len = 8;
+      encode(len, bl);
+    }
+    encode(endsize, bl);
+
+    r = bl.write_fd(fd);
+    if (r < 0) {
+      return r;
+    }
+  }
+  ExportDiffContext edc(&image, fd, info.size,
+                        g_conf().get_val<uint64_t>("rbd_concurrent_management_ops"),
+                        no_progress, export_format);
+  r = image.diff_iterate2(fromsnapname, 0, info.size, true, whole_object,
+                          &C_ExportDiff::export_diff_cb, (void *)&edc);
+  if (r < 0) {
+    goto out;
+  }
+
+  r = edc.throttle.wait_for_ret();
+  if (r < 0) {
+    goto out;
+  }
+
+  {
+    __u8 tag = RBD_DIFF_END;
+    bufferlist bl;
+    encode(tag, bl);
+    r = bl.write_fd(fd);
+  }
+
+out:
+  if (r < 0)
+    edc.pc.fail();
+  else
+    edc.pc.finish();
+
+  return r;
+}
+
+int do_export_diff(librbd::Image& image, const char *fromsnapname,
+                const char *endsnapname, bool whole_object,
+                const char *path, bool no_progress)
+{
+  int r;
+  int fd;
+
+  if (strcmp(path, "-") == 0)
+    fd = STDOUT_FILENO;
+  else
+    fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
+  if (fd < 0)
+    return -errno;
+
+  r = do_export_diff_fd(image, fromsnapname, endsnapname, whole_object, fd, no_progress, 1);
+
+  if (fd != 1)
+    close(fd);
+  if (r < 0 && fd != 1) {
+    remove(path);
+  }
+
+  return r;
+}
+
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+void get_arguments_diff(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_SOURCE);
+  at::add_path_options(positional, options,
+                       "export file (or '-' for stdout)");
+  options->add_options()
+    (at::FROM_SNAPSHOT_NAME.c_str(), po::value<std::string>(),
+     "snapshot starting point")
+    (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object");
+  at::add_no_progress_option(options);
+}
+
+int execute_diff(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_SOURCE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string path;
+  r = utils::get_path(vm, &arg_index, &path);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string from_snap_name;
+  if (vm.count(at::FROM_SNAPSHOT_NAME)) {
+    from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as<std::string>();
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "",
+                                 snap_name, true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_export_diff(image,
+                     from_snap_name.empty() ? nullptr : from_snap_name.c_str(),
+                     snap_name.empty() ? nullptr : snap_name.c_str(),
+                     vm[at::WHOLE_OBJECT].as<bool>(), path.c_str(),
+                     vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: export-diff error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::SwitchArguments switched_arguments({at::WHOLE_OBJECT});
+Shell::Action action_diff(
+  {"export-diff"}, {}, "Export incremental diff to file.", "",
+  &get_arguments_diff, &execute_diff);
+
+class C_Export : public Context
+{
+public:
+  C_Export(OrderedThrottle &ordered_throttle, librbd::Image &image,
+	   uint64_t fd_offset, uint64_t offset, uint64_t length, int fd)
+    : m_throttle(ordered_throttle), m_image(image), m_dest_offset(fd_offset),
+      m_offset(offset), m_length(length), m_fd(fd)
+  {
+  }
+
+  void send()
+  {
+    auto ctx = m_throttle.start_op(this);
+    auto aio_completion = new librbd::RBD::AioCompletion(
+      ctx, &utils::aio_context_callback);
+    int op_flags = LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL |
+                   LIBRADOS_OP_FLAG_FADVISE_NOCACHE;
+    int r = m_image.aio_read2(m_offset, m_length, m_bufferlist,
+                              aio_completion, op_flags);
+    if (r < 0) {
+      cerr << "rbd: error requesting read from source image" << std::endl;
+      aio_completion->release();
+      m_throttle.end_op(r);
+    }
+  }
+
+  void finish(int r) override
+  {
+    BOOST_SCOPE_EXIT((&m_throttle) (&r))
+    {
+      m_throttle.end_op(r);
+    } BOOST_SCOPE_EXIT_END
+
+    if (r < 0) {
+      cerr << "rbd: error reading from source image at offset "
+           << m_offset << ": " << cpp_strerror(r) << std::endl;
+      return;
+    }
+
+    ceph_assert(m_bufferlist.length() == static_cast<size_t>(r));
+    if (m_fd != STDOUT_FILENO) {
+      if (m_bufferlist.is_zero()) {
+        return;
+      }
+
+      uint64_t chkret = lseek64(m_fd, m_dest_offset, SEEK_SET);
+      if (chkret != m_dest_offset) {
+        cerr << "rbd: error seeking destination image to offset "
+             << m_dest_offset << std::endl;
+        r = -errno;
+        return;
+      }
+    }
+
+    r = m_bufferlist.write_fd(m_fd);
+    if (r < 0) {
+      cerr << "rbd: error writing to destination image at offset "
+           << m_dest_offset << std::endl;
+    }
+  }
+
+private:
+  OrderedThrottle &m_throttle;
+  librbd::Image &m_image;
+  bufferlist m_bufferlist;
+  uint64_t m_dest_offset;
+  uint64_t m_offset;
+  uint64_t m_length;
+  int m_fd;
+};
+
+const uint32_t MAX_KEYS = 64;
+
+static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd,
+		        uint64_t period, int max_concurrent_ops, utils::ProgressContext &pc)
+{
+  int r = 0;
+  // header
+  bufferlist bl;
+  bl.append(utils::RBD_IMAGE_BANNER_V2);
+
+  __u8 tag;
+  uint64_t length;
+  // encode order
+  tag = RBD_EXPORT_IMAGE_ORDER;
+  length = 8;
+  encode(tag, bl);
+  encode(length, bl);
+  encode(uint64_t(info.order), bl);
+
+  // encode features
+  tag = RBD_EXPORT_IMAGE_FEATURES;
+  uint64_t features;
+  image.features(&features);
+  length = 8;
+  encode(tag, bl);
+  encode(length, bl);
+  encode(features, bl);
+
+  // encode stripe_unit and stripe_count
+  tag = RBD_EXPORT_IMAGE_STRIPE_UNIT;
+  uint64_t stripe_unit;
+  stripe_unit = image.get_stripe_unit();
+  length = 8;
+  encode(tag, bl);
+  encode(length, bl);
+  encode(stripe_unit, bl);
+
+  tag = RBD_EXPORT_IMAGE_STRIPE_COUNT;
+  uint64_t stripe_count;
+  stripe_count = image.get_stripe_count();
+  length = 8;
+  encode(tag, bl);
+  encode(length, bl);
+  encode(stripe_count, bl);
+
+  //retrieve metadata of image
+  std::map<std::string, string> imagemetas;
+  std::string last_key;
+  bool more_results = true;
+  while (more_results) {
+    std::map<std::string, bufferlist> pairs;
+    r = image.metadata_list(last_key, MAX_KEYS, &pairs);
+    if (r < 0) {
+      std::cerr << "failed to retrieve metadata of image : " << cpp_strerror(r)
+                << std::endl;
+      return r;
+    }
+
+    if (!pairs.empty()) {
+      last_key = pairs.rbegin()->first;
+
+      for (auto kv : pairs) {
+        std::string key = kv.first;
+        std::string val(kv.second.c_str(), kv.second.length());
+        imagemetas[key] = val;
+      }
+    }
+    more_results = (pairs.size() == MAX_KEYS);
+  }
+
+  //encode imageMeta key and value
+  for (std::map<std::string, string>::iterator it = imagemetas.begin();
+       it != imagemetas.end(); ++it) {
+    string key = it->first;
+    string value = it->second;
+
+    tag = RBD_EXPORT_IMAGE_META;
+    length = key.length() + value.length() + 4 * 2;
+    encode(tag, bl);
+    encode(length, bl);
+    encode(key, bl);
+    encode(value, bl);
+  }
+
+  // encode end tag
+  tag = RBD_EXPORT_IMAGE_END;
+  encode(tag, bl);
+
+  // write bl to fd.
+  r = bl.write_fd(fd);
+  if (r < 0) {
+    return r;
+  }
+
+  // header for snapshots
+  bl.clear();
+  bl.append(utils::RBD_IMAGE_DIFFS_BANNER_V2);
+
+  std::vector<librbd::snap_info_t> snaps;
+  r = image.snap_list(snaps);
+  if (r < 0) {
+    return r;
+  }
+
+  uint64_t diff_num = snaps.size() + 1;
+  encode(diff_num, bl);
+
+  r = bl.write_fd(fd);
+  if (r < 0) {
+    return r;
+  }
+
+  const char *last_snap = NULL;
+  for (size_t i = 0; i < snaps.size(); ++i) {
+    utils::snap_set(image, snaps[i].name.c_str());
+    r = do_export_diff_fd(image, last_snap, snaps[i].name.c_str(), false, fd, true, 2);
+    if (r < 0) {
+      return r;
+    }
+    pc.update_progress(i, snaps.size() + 1);
+    last_snap = snaps[i].name.c_str();
+  }
+  utils::snap_set(image, std::string(""));
+  r = do_export_diff_fd(image, last_snap, nullptr, false, fd, true, 2);
+  if (r < 0) {
+    return r;
+  }
+  pc.update_progress(snaps.size() + 1, snaps.size() + 1);
+  return r;
+}
+
+static int do_export_v1(librbd::Image& image, librbd::image_info_t &info,
+                        int fd, uint64_t period, int max_concurrent_ops,
+                        utils::ProgressContext &pc)
+{
+  int r = 0;
+  size_t file_size = 0;
+  OrderedThrottle throttle(max_concurrent_ops, false);
+  for (uint64_t offset = 0; offset < info.size; offset += period) {
+    if (throttle.pending_error()) {
+      break;
+    }
+
+    uint64_t length = min(period, info.size - offset);
+    C_Export *ctx = new C_Export(throttle, image, file_size + offset, offset,
+                                 length, fd);
+    ctx->send();
+
+    pc.update_progress(offset, info.size);
+  }
+
+  file_size += info.size;
+  r = throttle.wait_for_ret();
+  if (fd != 1) {
+    if (r >= 0) {
+      r = ftruncate(fd, file_size);
+      if (r < 0)
+	return r;
+
+      uint64_t chkret = lseek64(fd, file_size, SEEK_SET);
+      if (chkret != file_size)
+	r = errno;
+    }
+  }
+  return r;
+}
+
+static int do_export(librbd::Image& image, const char *path, bool no_progress,
+                     int export_format)
+{
+  librbd::image_info_t info;
+  int64_t r = image.stat(info, sizeof(info));
+  if (r < 0)
+    return r;
+
+  int fd;
+  int max_concurrent_ops = g_conf().get_val<uint64_t>("rbd_concurrent_management_ops");
+  bool to_stdout = (strcmp(path, "-") == 0);
+  if (to_stdout) {
+    fd = STDOUT_FILENO;
+  } else {
+    fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
+    if (fd < 0) {
+      return -errno;
+    }
+#ifdef HAVE_POSIX_FADVISE
+    posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+#endif
+  }
+
+  utils::ProgressContext pc("Exporting image", no_progress);
+  uint64_t period = image.get_stripe_count() * (1ull << info.order);
+
+  if (export_format == 1)
+    r = do_export_v1(image, info, fd, period, max_concurrent_ops, pc);
+  else
+    r = do_export_v2(image, info, fd, period, max_concurrent_ops, pc);
+
+  if (r < 0)
+    pc.fail();
+  else
+    pc.finish();
+  if (!to_stdout)
+    close(fd);
+  return r;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_SOURCE);
+  at::add_path_options(positional, options,
+                       "export file (or '-' for stdout)");
+  at::add_no_progress_option(options);
+  at::add_export_format_option(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_SOURCE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string path;
+  r = utils::get_path(vm, &arg_index, &path);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "",
+                                 snap_name, true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  int format = 1;
+  if (vm.count("export-format"))
+    format = vm["export-format"].as<uint64_t>();
+
+  r = do_export(image, path.c_str(), vm[at::NO_PROGRESS].as<bool>(), format);
+  if (r < 0) {
+    std::cerr << "rbd: export error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"export"}, {}, "Export image to file.", "", &get_arguments, &execute);
+
+} // namespace export_full
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Feature.cc b/src/tools/rbd/action/Feature.cc
new file mode 100644
index 00000000..13a7b6ea
--- /dev/null
+++ b/src/tools/rbd/action/Feature.cc
@@ -0,0 +1,116 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/stringify.h"
+#include "common/errno.h"
+#include <iostream>
+#include <map>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace feature {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options, bool enabled) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  positional->add_options()
+    ("features", po::value<at::ImageFeatures>()->multitoken(),
+     ("image features\n" + at::get_short_features_help(false)).c_str());
+  if (enabled) {
+    at::add_create_journal_options(options);
+  }
+}
+
+void get_arguments_disable(po::options_description *positional,
+			   po::options_description *options) {
+  get_arguments(positional, options, false);
+}
+
+void get_arguments_enable(po::options_description *positional,
+			  po::options_description *options) {
+  get_arguments(positional, options, true);
+}
+
+int execute(const po::variables_map &vm, bool enabled) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::ImageOptions opts;
+  r = utils::get_journal_options(vm, &opts);
+  if (r < 0) {
+    return r;
+  }
+
+  std::vector<std::string> feature_names;
+  if (vm.count(at::POSITIONAL_ARGUMENTS)) {
+    const std::vector<std::string> &args =
+      vm[at::POSITIONAL_ARGUMENTS].as<std::vector<std::string> >();
+    feature_names.insert(feature_names.end(), args.begin() + arg_index,
+                         args.end());
+  }
+
+  if (feature_names.empty()) {
+    std::cerr << "rbd: at least one feature name must be specified"
+              << std::endl;
+    return -EINVAL;
+  }
+
+  boost::any features_any(static_cast<uint64_t>(0));
+  at::ImageFeatures image_features;
+  at::validate(features_any, feature_names, &image_features, 0);
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = image.update_features(boost::any_cast<uint64_t>(features_any), enabled);
+  if (r < 0) {
+    std::cerr << "rbd: failed to update image features: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+int execute_disable(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  return execute(vm, false);
+}
+
+int execute_enable(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  return execute(vm, true);
+}
+
+Shell::Action action_disable(
+  {"feature", "disable"}, {}, "Disable the specified image feature.", "",
+  &get_arguments_disable, &execute_disable);
+Shell::Action action_enable(
+  {"feature", "enable"}, {}, "Enable the specified image feature.", "",
+  &get_arguments_enable, &execute_enable);
+
+} // namespace feature
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Flatten.cc b/src/tools/rbd/action/Flatten.cc
new file mode 100644
index 00000000..ec4e837a
--- /dev/null
+++ b/src/tools/rbd/action/Flatten.cc
@@ -0,0 +1,74 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace flatten {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_flatten(librbd::Image& image, bool no_progress)
+{
+  utils::ProgressContext pc("Image flatten", no_progress);
+  int r = image.flatten_with_progress(pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_no_progress_option(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_flatten(image, vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: flatten error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"flatten"}, {}, "Fill clone with parent data (make it independent).", "",
+  &get_arguments, &execute);
+
+} // namespace flatten
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Ggate.cc b/src/tools/rbd/action/Ggate.cc
new file mode 100644
index 00000000..61f77be2
--- /dev/null
+++ b/src/tools/rbd/action/Ggate.cc
@@ -0,0 +1,193 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <sys/param.h>
+#include <errno.h>
+#include <unistd.h>
+
+#include "include/stringify.h"
+#include "common/SubProcess.h"
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/program_options.hpp>
+
+#include <iostream>
+
+namespace rbd {
+namespace action {
+namespace ggate {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int call_ggate_cmd(const po::variables_map &vm,
+                          const std::vector<std::string> &args,
+                          const std::vector<std::string> &ceph_global_args) {
+  SubProcess process("rbd-ggate", SubProcess::KEEP, SubProcess::KEEP,
+                     SubProcess::KEEP);
+
+  for (auto &arg : ceph_global_args) {
+    process.add_cmd_arg(arg.c_str());
+  }
+
+  for (auto &arg : args) {
+    process.add_cmd_arg(arg.c_str());
+  }
+
+  if (process.spawn()) {
+    std::cerr << "rbd: failed to run rbd-ggate: " << process.err() << std::endl;
+    return -EINVAL;
+  } else if (process.join()) {
+    std::cerr << "rbd: rbd-ggate failed with error: " << process.err()
+              << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+int get_image_or_snap_spec(const po::variables_map &vm, std::string *spec) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string nspace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &nspace_name,
+    &image_name, &snap_name, true,
+    utils::SNAPSHOT_PRESENCE_PERMITTED, utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  spec->append(pool_name);
+  spec->append("/");
+  if (!nspace_name.empty()) {
+    spec->append(nspace_name);
+    spec->append("/");
+  }
+  spec->append(image_name);
+  if (!snap_name.empty()) {
+    spec->append("@");
+    spec->append(snap_name);
+  }
+
+  return 0;
+}
+
+int parse_options(const std::vector<std::string> &options,
+                  std::vector<std::string> *args) {
+  for (auto &opts : options) {
+    std::vector<std::string> args_;
+    boost::split(args_, opts, boost::is_any_of(","));
+    for (auto &o : args_) {
+      args->push_back("--" + o);
+    }
+  }
+
+  return 0;
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+#if !defined(__FreeBSD__)
+  std::cerr << "rbd: ggate is only supported on FreeBSD" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+  std::vector<std::string> args;
+
+  args.push_back("list");
+
+  if (vm.count("format")) {
+    args.push_back("--format");
+    args.push_back(vm["format"].as<at::Format>().value);
+  }
+  if (vm["pretty-format"].as<bool>()) {
+    args.push_back("--pretty-format");
+  }
+
+  return call_ggate_cmd(vm, args, ceph_global_init_args);
+}
+
+int execute_map(const po::variables_map &vm,
+                const std::vector<std::string> &ceph_global_init_args) {
+#if !defined(__FreeBSD__)
+  std::cerr << "rbd: ggate is only supported on FreeBSD" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+  std::vector<std::string> args;
+
+  args.push_back("map");
+  std::string img;
+  int r = get_image_or_snap_spec(vm, &img);
+  if (r < 0) {
+    return r;
+  }
+  args.push_back(img);
+
+  if (vm["read-only"].as<bool>()) {
+    args.push_back("--read-only");
+  }
+
+  if (vm["exclusive"].as<bool>()) {
+    args.push_back("--exclusive");
+  }
+
+  if (vm.count("options")) {
+    r = parse_options(vm["options"].as<std::vector<std::string>>(), &args);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  return call_ggate_cmd(vm, args, ceph_global_init_args);
+}
+
+int execute_unmap(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+#if !defined(__FreeBSD__)
+  std::cerr << "rbd: ggate is only supported on FreeBSD" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+  std::string device_name = utils::get_positional_argument(vm, 0);
+  if (!boost::starts_with(device_name, "/dev/")) {
+    device_name.clear();
+  }
+
+  std::string image_name;
+  if (device_name.empty()) {
+    int r = get_image_or_snap_spec(vm, &image_name);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  if (device_name.empty() && image_name.empty()) {
+    std::cerr << "rbd: unmap requires either image name or device path"
+              << std::endl;
+    return -EINVAL;
+  }
+
+  std::vector<std::string> args;
+
+  args.push_back("unmap");
+  args.push_back(device_name.empty() ? image_name : device_name);
+
+  if (vm.count("options")) {
+    int r = parse_options(vm["options"].as<std::vector<std::string>>(), &args);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  return call_ggate_cmd(vm, args, ceph_global_init_args);
+}
+
+} // namespace ggate
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Group.cc b/src/tools/rbd/action/Group.cc
new file mode 100644
index 00000000..8554ae3b
--- /dev/null
+++ b/src/tools/rbd/action/Group.cc
@@ -0,0 +1,904 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <iostream>
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/rbd_types.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+
+namespace rbd {
+namespace action {
+namespace group {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static const std::string GROUP_SPEC("group-spec");
+static const std::string GROUP_SNAP_SPEC("group-snap-spec");
+
+static const std::string GROUP_NAME("group");
+static const std::string DEST_GROUP_NAME("dest-group");
+
+static const std::string GROUP_POOL_NAME("group-" + at::POOL_NAME);
+static const std::string IMAGE_POOL_NAME("image-" + at::POOL_NAME);
+
+void add_group_option(po::options_description *opt,
+		      at::ArgumentModifier modifier) {
+  std::string name = GROUP_NAME;
+  std::string description = at::get_description_prefix(modifier) + "group name";
+  switch (modifier) {
+  case at::ARGUMENT_MODIFIER_NONE:
+  case at::ARGUMENT_MODIFIER_SOURCE:
+    break;
+  case at::ARGUMENT_MODIFIER_DEST:
+    name = DEST_GROUP_NAME;
+    break;
+  }
+
+  // TODO add validator
+  opt->add_options()
+    (name.c_str(), po::value<std::string>(), description.c_str());
+}
+
+void add_prefixed_pool_option(po::options_description *opt,
+                            const std::string &prefix) {
+  std::string name = prefix + "-" + at::POOL_NAME;
+  std::string description = prefix + " pool name";
+
+  opt->add_options()
+    (name.c_str(), po::value<std::string>(), description.c_str());
+}
+
+void add_prefixed_namespace_option(po::options_description *opt,
+                                   const std::string &prefix) {
+  std::string name = prefix + "-" + at::NAMESPACE_NAME;
+  std::string description = prefix + " namespace name";
+
+  opt->add_options()
+    (name.c_str(), po::value<std::string>(), description.c_str());
+}
+
+void add_group_spec_options(po::options_description *pos,
+			    po::options_description *opt,
+			    at::ArgumentModifier modifier,
+                            bool snap) {
+  at::add_pool_option(opt, modifier);
+  at::add_namespace_option(opt, modifier);
+  add_group_option(opt, modifier);
+  if (!snap) {
+    pos->add_options()
+      ((get_name_prefix(modifier) + GROUP_SPEC).c_str(),
+       (get_description_prefix(modifier) + "group specification\n" +
+         "(example: [<pool-name>/[<namespace>/]]<group-name>)").c_str());
+  } else {
+    add_snap_option(opt, modifier);
+    pos->add_options()
+      ((get_name_prefix(modifier) + GROUP_SNAP_SPEC).c_str(),
+       (get_description_prefix(modifier) + "group specification\n" +
+         "(example: [<pool-name>/[<namespace>/]]<group-name>@<snap-name>)").c_str());
+  }
+}
+
+int execute_create(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+
+  std::string pool_name;
+  std::string namespace_name;
+  std::string group_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, at::POOL_NAME, &pool_name,
+    &namespace_name, GROUP_NAME, "group", &group_name, nullptr, true,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+  librbd::RBD rbd;
+  r = rbd.group_create(io_ctx, group_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: create error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  std::string namespace_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, false, &pool_name,
+                                              &namespace_name, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+  Formatter *f = formatter.get();
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  std::vector<std::string> names;
+  r = rbd.group_list(io_ctx, &names);
+  if (r < 0)
+    return r;
+
+  if (f)
+    f->open_array_section("groups");
+  for (auto i : names) {
+    if (f)
+      f->dump_string("name", i);
+    else
+      std::cout << i << std::endl;
+  }
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  }
+
+  return 0;
+}
+
+int execute_remove(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+
+  std::string pool_name;
+  std::string namespace_name;
+  std::string group_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, at::POOL_NAME, &pool_name,
+    &namespace_name, GROUP_NAME, "group", &group_name, nullptr, true,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+  librbd::RBD rbd;
+
+  r = rbd.group_remove(io_ctx, group_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: remove error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+int execute_rename(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+
+  std::string pool_name;
+  std::string namespace_name;
+  std::string group_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, at::POOL_NAME, &pool_name,
+    &namespace_name, GROUP_NAME, "group", &group_name, nullptr, true,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string dest_pool_name;
+  std::string dest_namespace_name;
+  std::string dest_group_name;
+
+  r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, at::DEST_POOL_NAME,
+    &dest_pool_name, &dest_namespace_name, DEST_GROUP_NAME, "group",
+    &dest_group_name, nullptr, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  if (pool_name != dest_pool_name) {
+    std::cerr << "rbd: group rename across pools not supported" << std::endl
+              << "source pool: " << pool_name << ", dest pool: "
+              << dest_pool_name << std::endl;
+    return -EINVAL;
+  } else if (namespace_name != dest_namespace_name) {
+    std::cerr << "rbd: group rename across namespaces not supported"
+              << std::endl
+              << "source namespace: " << namespace_name << ", dest namespace: "
+              << dest_namespace_name << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.group_rename(io_ctx, group_name.c_str(),
+                       dest_group_name.c_str());
+
+  if (r < 0) {
+    std::cerr << "rbd: failed to rename group: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+int execute_add(const po::variables_map &vm,
+                const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  // Parse group data.
+  std::string group_pool_name;
+  std::string group_namespace_name;
+  std::string group_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, GROUP_POOL_NAME,
+    &group_pool_name, &group_namespace_name, GROUP_NAME, "group", &group_name,
+    nullptr, true, utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string image_pool_name;
+  std::string image_namespace_name;
+  std::string image_name;
+
+  r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, IMAGE_POOL_NAME,
+    &image_pool_name, &image_namespace_name, at::IMAGE_NAME, "image",
+    &image_name, nullptr, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  if (group_namespace_name != image_namespace_name) {
+    std::cerr << "rbd: group and image namespace must match." << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx cg_io_ctx;
+  r = utils::init(group_pool_name, group_namespace_name, &rados, &cg_io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::IoCtx image_io_ctx;
+  r = utils::init(image_pool_name, group_namespace_name, &rados, &image_io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.group_image_add(cg_io_ctx, group_name.c_str(),
+			  image_io_ctx, image_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: add image error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+int execute_remove_image(const po::variables_map &vm,
+                         const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+
+  std::string group_pool_name;
+  std::string group_namespace_name;
+  std::string group_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, GROUP_POOL_NAME,
+    &group_pool_name, &group_namespace_name, GROUP_NAME, "group", &group_name,
+    nullptr, true, utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string image_pool_name;
+  std::string image_namespace_name;
+  std::string image_name;
+  std::string image_id;
+
+  if (vm.count(at::IMAGE_ID)) {
+    image_id = vm[at::IMAGE_ID].as<std::string>();
+  }
+
+  r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, IMAGE_POOL_NAME,
+    &image_pool_name, &image_namespace_name, at::IMAGE_NAME, "image",
+    &image_name, nullptr, image_id.empty(), utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  if (group_namespace_name != image_namespace_name) {
+    std::cerr << "rbd: group and image namespace must match." << std::endl;
+    return -EINVAL;
+  } else if (!image_id.empty() && !image_name.empty()) {
+    std::cerr << "rbd: trying to access image using both name and id. "
+              << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx cg_io_ctx;
+  r = utils::init(group_pool_name, group_namespace_name, &rados, &cg_io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::IoCtx image_io_ctx;
+  r = utils::init(image_pool_name, group_namespace_name, &rados, &image_io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  if (image_id.empty()) {
+    r = rbd.group_image_remove(cg_io_ctx, group_name.c_str(),
+                               image_io_ctx, image_name.c_str());
+  } else {
+    r = rbd.group_image_remove_by_id(cg_io_ctx, group_name.c_str(),
+                                     image_io_ctx, image_id.c_str());
+  }
+  if (r < 0) {
+    std::cerr << "rbd: remove image error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+int execute_list_images(const po::variables_map &vm,
+                        const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string group_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, at::POOL_NAME, &pool_name,
+    &namespace_name, GROUP_NAME, "group", &group_name, nullptr, true,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+  Formatter *f = formatter.get();
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  std::vector<librbd::group_image_info_t> images;
+
+  r = rbd.group_image_list(io_ctx, group_name.c_str(), &images,
+                           sizeof(librbd::group_image_info_t));
+
+  if (r == -ENOENT)
+    r = 0;
+
+  if (r < 0)
+    return r;
+
+  std::sort(images.begin(), images.end(),
+    [](const librbd::group_image_info_t &lhs,
+       const librbd::group_image_info_t &rhs) {
+      if (lhs.pool != rhs.pool) {
+        return lhs.pool < rhs.pool;
+      }
+      return lhs.name < rhs.name;
+    }
+  );
+
+  if (f)
+    f->open_array_section("images");
+
+  for (auto image : images) {
+    std::string image_name = image.name;
+    int state = image.state;
+    std::string state_string;
+    if (RBD_GROUP_IMAGE_STATE_INCOMPLETE == state) {
+      state_string = "incomplete";
+    }
+
+    std::string pool_name = "";
+
+    librados::Rados rados(io_ctx);
+    librados::IoCtx pool_io_ctx;
+    r = rados.ioctx_create2(image.pool, pool_io_ctx);
+    if (r < 0) {
+      pool_name = "<missing image pool " + stringify(image.pool) + ">";
+    } else {
+      pool_name = pool_io_ctx.get_pool_name();
+    }
+
+    if (f) {
+      f->open_object_section("image");
+      f->dump_string("image", image_name);
+      f->dump_string("pool", pool_name);
+      f->dump_string("namespace", io_ctx.get_namespace());
+      f->dump_int("state", state);
+      f->close_section();
+    } else {
+      std::cout << pool_name << "/";
+      if (!io_ctx.get_namespace().empty()) {
+        std::cout << io_ctx.get_namespace() << "/";
+      }
+      std::cout << image_name << " " << state_string << std::endl;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  }
+
+  return 0;
+}
+
+int execute_group_snap_create(const po::variables_map &vm,
+                              const std::vector<std::string> &global_args) {
+  size_t arg_index = 0;
+
+  std::string pool_name;
+  std::string namespace_name;
+  std::string group_name;
+  std::string snap_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, at::POOL_NAME, &pool_name,
+    &namespace_name, GROUP_NAME, "group", &group_name, &snap_name, true,
+    utils::SNAPSHOT_PRESENCE_REQUIRED, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::IoCtx io_ctx;
+  librados::Rados rados;
+
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.group_snap_create(io_ctx, group_name.c_str(), snap_name.c_str());
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+int execute_group_snap_remove(const po::variables_map &vm,
+                              const std::vector<std::string> &global_args) {
+  size_t arg_index = 0;
+
+  std::string pool_name;
+  std::string namespace_name;
+  std::string group_name;
+  std::string snap_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, at::POOL_NAME, &pool_name,
+    &namespace_name, GROUP_NAME, "group", &group_name, &snap_name, true,
+    utils::SNAPSHOT_PRESENCE_REQUIRED, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::IoCtx io_ctx;
+  librados::Rados rados;
+
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.group_snap_remove(io_ctx, group_name.c_str(), snap_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: failed to remove group snapshot: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+int execute_group_snap_rename(const po::variables_map &vm,
+                              const std::vector<std::string> &global_args) {
+  size_t arg_index = 0;
+
+  std::string pool_name;
+  std::string namespace_name;
+  std::string group_name;
+  std::string source_snap_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, at::POOL_NAME, &pool_name,
+    &namespace_name, GROUP_NAME, "group", &group_name, &source_snap_name, true,
+    utils::SNAPSHOT_PRESENCE_REQUIRED, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string dest_snap_name;
+  if (vm.count(at::DEST_SNAPSHOT_NAME)) {
+    dest_snap_name = vm[at::DEST_SNAPSHOT_NAME].as<std::string>();
+  }
+
+  if (dest_snap_name.empty()) {
+    dest_snap_name = utils::get_positional_argument(vm, arg_index++);
+  }
+
+  if (dest_snap_name.empty()) {
+    std::cerr << "rbd: destination snapshot name was not specified"
+              << std::endl;
+    return -EINVAL;
+  }
+
+  r = utils::validate_snapshot_name(at::ARGUMENT_MODIFIER_DEST, dest_snap_name,
+                                    utils::SNAPSHOT_PRESENCE_REQUIRED,
+                                    utils::SPEC_VALIDATION_SNAP);
+  if (r < 0) {
+    return r;
+  }
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.group_snap_rename(io_ctx, group_name.c_str(),
+                            source_snap_name.c_str(), dest_snap_name.c_str());
+
+  if (r < 0) {
+    std::cerr << "rbd: failed to rename group snapshot: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+int execute_group_snap_list(const po::variables_map &vm,
+                            const std::vector<std::string> &ceph_global_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string group_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, at::POOL_NAME, &pool_name,
+    &namespace_name, GROUP_NAME, "group", &group_name, nullptr, true,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+  Formatter *f = formatter.get();
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  std::vector<librbd::group_snap_info_t> snaps;
+
+  r = rbd.group_snap_list(io_ctx, group_name.c_str(), &snaps,
+                          sizeof(librbd::group_snap_info_t));
+
+  if (r == -ENOENT) {
+    r = 0;
+  }
+  if (r < 0) {
+    return r;
+  }
+
+  TextTable t;
+  if (f) {
+    f->open_array_section("group_snaps");
+  } else {
+    t.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+    t.define_column("STATUS", TextTable::LEFT, TextTable::RIGHT);
+  }
+
+  for (auto i : snaps) {
+    std::string snap_name = i.name;
+    int state = i.state;
+    std::string state_string;
+    if (RBD_GROUP_SNAP_STATE_INCOMPLETE == state) {
+      state_string = "incomplete";
+    } else {
+      state_string = "ok";
+    }
+    if (r < 0) {
+      return r;
+    }
+    if (f) {
+      f->open_object_section("group_snap");
+      f->dump_string("snapshot", snap_name);
+      f->dump_string("state", state_string);
+      f->close_section();
+    } else {
+      t << snap_name << state_string << TextTable::endrow;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  } else if (snaps.size()) {
+    std::cout << t;
+  }
+  return 0;
+}
+
+int execute_group_snap_rollback(const po::variables_map &vm,
+                                const std::vector<std::string> &global_args) {
+  size_t arg_index = 0;
+
+  std::string group_name;
+  std::string namespace_name;
+  std::string pool_name;
+  std::string snap_name;
+
+  int r = utils::get_pool_generic_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, at::POOL_NAME, &pool_name,
+    &namespace_name, GROUP_NAME, "group", &group_name, &snap_name, true,
+    utils::SNAPSHOT_PRESENCE_REQUIRED, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::IoCtx io_ctx;
+  librados::Rados rados;
+
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  utils::ProgressContext pc("Rolling back to group snapshot",
+                            vm[at::NO_PROGRESS].as<bool>());
+  r = rbd.group_snap_rollback_with_progress(io_ctx, group_name.c_str(),
+                                            snap_name.c_str(), pc);
+  if (r < 0) {
+    pc.fail();
+    std::cerr << "rbd: rollback group to snapshot failed: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  pc.finish();
+  return 0;
+}
+
+void get_create_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE,
+                         false);
+}
+
+void get_remove_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE,
+                         false);
+}
+
+void get_list_arguments(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_pool_options(positional, options, true);
+  at::add_format_options(options);
+}
+
+void get_rename_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_SOURCE,
+                         false);
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST,
+                         false);
+}
+
+void get_add_arguments(po::options_description *positional,
+                       po::options_description *options) {
+  positional->add_options()
+    (GROUP_SPEC.c_str(),
+     "group specification\n"
+     "(example: [<pool-name>/[<namespace>/]]<group-name>)");
+
+  add_prefixed_pool_option(options, "group");
+  add_prefixed_namespace_option(options, "group");
+  add_group_option(options, at::ARGUMENT_MODIFIER_NONE);
+
+  positional->add_options()
+    (at::IMAGE_SPEC.c_str(),
+     "image specification\n"
+     "(example: [<pool-name>/[<namespace>/]]<image-name>)");
+
+  add_prefixed_pool_option(options, "image");
+  add_prefixed_namespace_option(options, "image");
+  at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE);
+
+  at::add_pool_option(options, at::ARGUMENT_MODIFIER_NONE,
+	       " unless overridden");
+}
+
+void get_remove_image_arguments(po::options_description *positional,
+                                po::options_description *options) {
+  positional->add_options()
+    (GROUP_SPEC.c_str(),
+     "group specification\n"
+     "(example: [<pool-name>/[<namespace>/]]<group-name>)");
+
+  add_prefixed_pool_option(options, "group");
+  add_prefixed_namespace_option(options, "group");
+  add_group_option(options, at::ARGUMENT_MODIFIER_NONE);
+
+  positional->add_options()
+    (at::IMAGE_SPEC.c_str(),
+     "image specification\n"
+     "(example: [<pool-name>/[<namespace>/]]<image-name>)");
+
+  add_prefixed_pool_option(options, "image");
+  add_prefixed_namespace_option(options, "image");
+  at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE);
+
+  at::add_pool_option(options, at::ARGUMENT_MODIFIER_NONE,
+	       " unless overridden");
+  at::add_image_id_option(options);
+}
+
+void get_list_images_arguments(po::options_description *positional,
+                               po::options_description *options) {
+  at::add_format_options(options);
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE,
+                         false);
+}
+
+void get_group_snap_create_arguments(po::options_description *positional,
+				  po::options_description *options) {
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE,
+                         true);
+}
+
+void get_group_snap_remove_arguments(po::options_description *positional,
+				  po::options_description *options) {
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE,
+                         true);
+}
+
+void get_group_snap_rename_arguments(po::options_description *positional,
+				     po::options_description *options) {
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE,
+                         true);
+
+  positional->add_options()
+    (at::DEST_SNAPSHOT_NAME.c_str(),
+     "destination snapshot name\n(example: <snapshot-name>)");
+  at::add_snap_option(options, at::ARGUMENT_MODIFIER_DEST);
+}
+
+void get_group_snap_list_arguments(po::options_description *positional,
+                             po::options_description *options) {
+  at::add_format_options(options);
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE,
+                         false);
+}
+
+void get_group_snap_rollback_arguments(po::options_description *positional,
+                                       po::options_description *options) {
+  at::add_no_progress_option(options);
+  add_group_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE,
+                         true);
+}
+
+Shell::Action action_create(
+  {"group", "create"}, {}, "Create a group.",
+  "", &get_create_arguments, &execute_create);
+Shell::Action action_remove(
+  {"group", "remove"}, {"group", "rm"}, "Delete a group.",
+  "", &get_remove_arguments, &execute_remove);
+Shell::Action action_list(
+  {"group", "list"}, {"group", "ls"}, "List rbd groups.",
+  "", &get_list_arguments, &execute_list);
+Shell::Action action_rename(
+  {"group", "rename"}, {}, "Rename a group within pool.",
+  "", &get_rename_arguments, &execute_rename);
+Shell::Action action_add(
+  {"group", "image", "add"}, {}, "Add an image to a group.",
+  "", &get_add_arguments, &execute_add);
+Shell::Action action_remove_image(
+  {"group", "image", "remove"}, {"group", "image", "rm"},
+  "Remove an image from a group.", "",
+  &get_remove_image_arguments, &execute_remove_image);
+Shell::Action action_list_images(
+  {"group", "image", "list"}, {"group", "image", "ls"},
+  "List images in a group.", "",
+  &get_list_images_arguments, &execute_list_images);
+Shell::Action action_group_snap_create(
+  {"group", "snap", "create"}, {}, "Make a snapshot of a group.",
+  "", &get_group_snap_create_arguments, &execute_group_snap_create);
+Shell::Action action_group_snap_remove(
+  {"group", "snap", "remove"}, {"group", "snap", "rm"},
+  "Remove a snapshot from a group.",
+  "", &get_group_snap_remove_arguments, &execute_group_snap_remove);
+Shell::Action action_group_snap_rename(
+  {"group", "snap", "rename"}, {}, "Rename group's snapshot.",
+  "", &get_group_snap_rename_arguments, &execute_group_snap_rename);
+Shell::Action action_group_snap_list(
+  {"group", "snap", "list"}, {"group", "snap", "ls"},
+  "List snapshots of a group.",
+  "", &get_group_snap_list_arguments, &execute_group_snap_list);
+Shell::Action action_group_snap_rollback(
+  {"group", "snap", "rollback"}, {},
+  "Rollback group to snapshot.",
+  "", &get_group_snap_rollback_arguments, &execute_group_snap_rollback);
+
+} // namespace group
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/ImageMeta.cc b/src/tools/rbd/action/ImageMeta.cc
new file mode 100644
index 00000000..20c4555d
--- /dev/null
+++ b/src/tools/rbd/action/ImageMeta.cc
@@ -0,0 +1,345 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace image_meta {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+namespace {
+
+void add_key_option(po::options_description *positional) {
+  positional->add_options()
+    ("key", "image meta key");
+}
+
+int get_key(const po::variables_map &vm, size_t *arg_index,
+            std::string *key) {
+  *key = utils::get_positional_argument(vm, *arg_index);
+  if (key->empty()) {
+    std::cerr << "rbd: metadata key was not specified" << std::endl;
+    return -EINVAL;
+  } else {
+    ++(*arg_index);
+  }
+  return 0;
+}
+
+const uint32_t MAX_KEYS = 64;
+
+} // anonymous namespace
+
+static int do_metadata_list(librbd::Image& image, Formatter *f)
+{
+  int r;
+  TextTable tbl;
+
+  size_t count = 0;
+  std::string last_key;
+  bool more_results = true;
+  while (more_results) {
+    std::map<std::string, bufferlist> pairs;
+    r = image.metadata_list(last_key, MAX_KEYS, &pairs);
+    if (r < 0) {
+      std::cerr << "failed to list metadata of image : " << cpp_strerror(r)
+                << std::endl;
+      return r;
+    }
+
+    more_results = (pairs.size() == MAX_KEYS);
+    if (!pairs.empty()) {
+      if (count == 0) {
+        if (f) {
+          f->open_object_section("metadatas");
+        } else {
+          tbl.define_column("Key", TextTable::LEFT, TextTable::LEFT);
+          tbl.define_column("Value", TextTable::LEFT, TextTable::LEFT);
+        }
+      }
+
+      last_key = pairs.rbegin()->first;
+      count += pairs.size();
+
+      for (auto kv : pairs) {
+        std::string val(kv.second.c_str(), kv.second.length());
+        if (f) {
+          f->dump_string(kv.first.c_str(), val.c_str());
+        } else {
+          tbl << kv.first << val << TextTable::endrow;
+        }
+      }
+    }
+  }
+
+  if (f == nullptr) {
+    bool single = (count == 1);
+    std::cout << "There " << (single ? "is" : "are") << " " << count << " "
+              << (single ? "metadatum" : "metadata") << " on this image"
+              << (count == 0 ? "." : ":") << std::endl;
+  }
+
+  if (count > 0) {
+    if (f) {
+      f->close_section();
+      f->flush(std::cout);
+    } else {
+      std::cout << std::endl << tbl;
+    }
+  }
+  return 0;
+}
+
+static int do_metadata_set(librbd::Image& image, std::string &key,
+                          std::string &value)
+{
+  int r = image.metadata_set(key, value);
+  if (r < 0) {
+    std::cerr << "failed to set metadata " << key << " of image : "
+              << cpp_strerror(r) << std::endl;
+  }
+  return r;
+}
+
+static int do_metadata_remove(librbd::Image& image, std::string &key)
+{
+  int r = image.metadata_remove(key);
+  if (r == -ENOENT) {
+      std::cerr << "rbd: no existing metadata key " << key << " of image : "
+                << cpp_strerror(r) << std::endl;
+  } else if(r < 0) {
+      std::cerr << "failed to remove metadata " << key << " of image : "
+                << cpp_strerror(r) << std::endl;
+  }
+  return r;
+}
+
+static int do_metadata_get(librbd::Image& image, std::string &key)
+{
+  std::string s;
+  int r = image.metadata_get(key, &s);
+  if (r < 0) {
+    std::cerr << "failed to get metadata " << key << " of image : "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  std::cout << s << std::endl;
+  return r;
+}
+
+void get_list_arguments(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_format_options(options);
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_metadata_list(image, formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: listing metadata failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_get_arguments(po::options_description *positional,
+                       po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  add_key_option(positional);
+}
+
+int execute_get(const po::variables_map &vm,
+                const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_metadata_get(image, key);
+  if (r < 0) {
+    std::cerr << "rbd: getting metadata failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_set_arguments(po::options_description *positional,
+                       po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  add_key_option(positional);
+  positional->add_options()
+    ("value", "image meta value");
+}
+
+int execute_set(const po::variables_map &vm,
+                const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string value = utils::get_positional_argument(vm, arg_index);
+  if (value.empty()) {
+    std::cerr << "rbd: metadata value was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_metadata_set(image, key, value);
+  if (r < 0) {
+    std::cerr << "rbd: setting metadata failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_remove_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  add_key_option(positional);
+}
+
+int execute_remove(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string key;
+  r = get_key(vm, &arg_index, &key);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_metadata_remove(image, key);
+  if (r < 0) {
+    std::cerr << "rbd: removing metadata failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action_list(
+  {"image-meta", "list"}, {"image-meta", "ls"}, "Image metadata list keys with values.", "",
+  &get_list_arguments, &execute_list);
+Shell::Action action_get(
+  {"image-meta", "get"}, {},
+  "Image metadata get the value associated with the key.", "",
+  &get_get_arguments, &execute_get);
+Shell::Action action_set(
+  {"image-meta", "set"}, {}, "Image metadata set key with value.", "",
+  &get_set_arguments, &execute_set);
+Shell::Action action_remove(
+  {"image-meta", "remove"}, {"image-meta", "rm"},
+  "Image metadata remove the key and value associated.", "",
+  &get_remove_arguments, &execute_remove);
+
+} // namespace image_meta
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Import.cc b/src/tools/rbd/action/Import.cc
new file mode 100644
index 00000000..7397d926
--- /dev/null
+++ b/src/tools/rbd/action/Import.cc
@@ -0,0 +1,1037 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/Context.h"
+#include "common/blkdev.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/Throttle.h"
+#include "include/compat.h"
+#include "include/encoding.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+#include <boost/scoped_ptr.hpp>
+#include "include/ceph_assert.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd
+
+namespace rbd {
+namespace action {
+namespace import {
+
+struct ImportDiffContext {
+  librbd::Image *image;
+  int fd;
+  size_t size;
+  utils::ProgressContext pc;
+  OrderedThrottle throttle;
+  uint64_t last_offset;
+
+  ImportDiffContext(librbd::Image *image, int fd, size_t size, bool no_progress)
+    : image(image), fd(fd), size(size), pc("Importing image diff", no_progress),
+      throttle((fd == STDIN_FILENO) ? 1 :
+                  g_conf().get_val<uint64_t>("rbd_concurrent_management_ops"),
+               false),
+      last_offset(0) {
+  }
+
+  void update_size(size_t new_size)
+  {
+    if (fd == STDIN_FILENO) {
+      size = new_size;
+    }
+  }
+
+  void update_progress(uint64_t off)
+  {
+    if (size) {
+      pc.update_progress(off, size);
+      last_offset = off;
+    }
+  }
+
+  void update_progress()
+  {
+    uint64_t off = last_offset;
+    if (fd != STDIN_FILENO) {
+      off = lseek(fd, 0, SEEK_CUR);
+    }
+
+    update_progress(off);
+  }
+
+  void finish(int r)
+  {
+    if (r < 0) {
+      pc.fail();
+    } else {
+      pc.finish();
+    }
+  }
+};
+
+class C_ImportDiff : public Context {
+public:
+  C_ImportDiff(ImportDiffContext *idiffctx, bufferlist data, uint64_t offset,
+               uint64_t length, bool write_zeroes)
+    : m_idiffctx(idiffctx), m_data(data), m_offset(offset), m_length(length),
+      m_write_zeroes(write_zeroes) {
+    // use block offset (stdin) or import file position to report
+    // progress.
+    if (m_idiffctx->fd == STDIN_FILENO) {
+      m_prog_offset = offset;
+    } else {
+      m_prog_offset  = lseek(m_idiffctx->fd, 0, SEEK_CUR);
+    }
+  }
+
+  int send()
+  {
+    if (m_idiffctx->throttle.pending_error()) {
+      return m_idiffctx->throttle.wait_for_ret();
+    }
+
+    C_OrderedThrottle *ctx = m_idiffctx->throttle.start_op(this);
+    librbd::RBD::AioCompletion *aio_completion =
+      new librbd::RBD::AioCompletion(ctx, &utils::aio_context_callback);
+
+    int r;
+    if (m_write_zeroes) {
+      r = m_idiffctx->image->aio_write_zeroes(m_offset, m_length,
+                                              aio_completion, 0U,
+                                              LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
+    } else {
+      r = m_idiffctx->image->aio_write2(m_offset, m_length, m_data,
+                                        aio_completion,
+                                        LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
+    }
+
+    if (r < 0) {
+      aio_completion->release();
+      ctx->complete(r);
+    }
+
+    return r;
+  }
+
+  void finish(int r) override
+  {
+    m_idiffctx->update_progress(m_prog_offset);
+    m_idiffctx->throttle.end_op(r);
+  }
+
+private:
+  ImportDiffContext *m_idiffctx;
+  bufferlist m_data;
+  uint64_t m_offset;
+  uint64_t m_length;
+  bool m_write_zeroes;
+  uint64_t m_prog_offset;
+};
+
+static int do_image_snap_from(ImportDiffContext *idiffctx)
+{
+  int r;
+  string from;
+  r = utils::read_string(idiffctx->fd, 4096, &from);   // 4k limit to make sure we don't get a garbage string
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode start snap name" << std::endl;
+    return r;
+  }
+
+  bool exists;
+  r = idiffctx->image->snap_exists2(from.c_str(), &exists);
+  if (r < 0) {
+    std::cerr << "rbd: failed to query start snap state" << std::endl;
+    return r;
+  }
+
+  if (!exists) {
+    std::cerr << "start snapshot '" << from
+              << "' does not exist in the image, aborting" << std::endl;
+    return -EINVAL;
+  }
+
+  idiffctx->update_progress();
+  return 0;
+}
+
+static int do_image_snap_to(ImportDiffContext *idiffctx, std::string *tosnap)
+{
+  int r;
+  string to;
+  r = utils::read_string(idiffctx->fd, 4096, &to);   // 4k limit to make sure we don't get a garbage string
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode end snap name" << std::endl;
+    return r;
+  }
+
+  bool exists;
+  r = idiffctx->image->snap_exists2(to.c_str(), &exists);
+  if (r < 0) {
+    std::cerr << "rbd: failed to query end snap state" << std::endl;
+    return r;
+  }
+
+  if (exists) {
+    std::cerr << "end snapshot '" << to << "' already exists, aborting"
+              << std::endl;
+    return -EEXIST;
+  }
+
+  *tosnap = to;
+  idiffctx->update_progress();
+
+  return 0;
+}
+
+static int get_snap_protection_status(ImportDiffContext *idiffctx,
+                                      bool *is_protected)
+{
+  int r;
+  char buf[sizeof(__u8)];
+  r = safe_read_exact(idiffctx->fd, buf, sizeof(buf));
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode snap protection status" << std::endl;
+    return r;
+  }
+
+  *is_protected = (buf[0] != 0);
+  idiffctx->update_progress();
+
+  return 0;
+}
+
+static int do_image_resize(ImportDiffContext *idiffctx)
+{
+  int r;
+  char buf[sizeof(uint64_t)];
+  uint64_t end_size;
+  r = safe_read_exact(idiffctx->fd, buf, sizeof(buf));
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode image size" << std::endl;
+    return r;
+  }
+
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  auto p = bl.cbegin();
+  decode(end_size, p);
+
+  uint64_t cur_size;
+  idiffctx->image->size(&cur_size);
+  if (cur_size != end_size) {
+    idiffctx->image->resize(end_size);
+  }
+
+  idiffctx->update_size(end_size);
+  idiffctx->update_progress();
+  return 0;
+}
+
+static int do_image_io(ImportDiffContext *idiffctx, bool write_zeroes,
+                       size_t sparse_size)
+{
+  int r;
+  char buf[16];
+  r = safe_read_exact(idiffctx->fd, buf, sizeof(buf));
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode IO length" << std::endl;
+    return r;
+  }
+
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  auto p = bl.cbegin();
+
+  uint64_t image_offset, buffer_length;
+  decode(image_offset, p);
+  decode(buffer_length, p);
+
+  if (!write_zeroes) {
+    bufferptr bp = buffer::create(buffer_length);
+    r = safe_read_exact(idiffctx->fd, bp.c_str(), buffer_length);
+    if (r < 0) {
+      std::cerr << "rbd: failed to decode write data" << std::endl;
+      return r;
+    }
+
+    size_t buffer_offset = 0;
+    while (buffer_offset < buffer_length) {
+      size_t write_length = 0;
+      bool zeroed = false;
+      utils::calc_sparse_extent(bp, sparse_size, buffer_offset, buffer_length,
+				&write_length, &zeroed);
+      ceph_assert(write_length > 0);
+
+      bufferlist write_bl;
+      if (!zeroed) {
+	bufferptr write_ptr(bp, buffer_offset, write_length);
+	write_bl.push_back(write_ptr);
+	ceph_assert(write_bl.length() == write_length);
+      }
+
+      C_ImportDiff *ctx = new C_ImportDiff(idiffctx, write_bl,
+					   image_offset + buffer_offset,
+					   write_length, zeroed);
+      r = ctx->send();
+      if (r < 0) {
+	return r;
+      }
+
+      buffer_offset += write_length;
+    }
+  } else {
+    bufferlist data;
+    C_ImportDiff *ctx = new C_ImportDiff(idiffctx, data, image_offset,
+					 buffer_length, true);
+    return ctx->send();
+  }
+  return r;
+}
+
+static int validate_banner(int fd, std::string banner)
+{
+  int r;
+  char buf[banner.size() + 1];
+  memset(buf, 0, sizeof(buf));
+  r = safe_read_exact(fd, buf, banner.size());
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode diff banner" << std::endl;
+    return r;
+  }
+
+  buf[banner.size()] = '\0';
+  if (strcmp(buf, banner.c_str())) {
+    std::cerr << "rbd: invalid or unexpected diff banner" << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+static int skip_tag(int fd, uint64_t length)
+{
+  int r;
+
+  if (fd == STDIN_FILENO) {
+    // read the appending data out to skip this tag.
+    char buf[4096];
+    uint64_t len = min<uint64_t>(length, sizeof(buf));
+    while (len > 0) {
+      r = safe_read_exact(fd, buf, len);
+      if (r < 0) {
+        std::cerr << "rbd: failed to decode skipped tag data" << std::endl;
+        return r;
+      }
+      length -= len;
+      len = min<uint64_t>(length, sizeof(buf));
+    }
+  } else {
+    // lseek to skip this tag
+    off64_t offs = lseek64(fd, length, SEEK_CUR);
+    if (offs < 0) {
+      return -errno;
+    }
+  }
+
+  return 0;
+}
+
+static int read_tag(int fd, __u8 end_tag, int format, __u8 *tag, uint64_t *readlen)
+{
+  int r;
+  __u8 read_tag;
+
+  r = safe_read_exact(fd, &read_tag, sizeof(read_tag));
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode tag" << std::endl;
+    return r;
+  }
+
+  *tag = read_tag;
+  if (read_tag != end_tag && format == 2) {
+    char buf[sizeof(uint64_t)];
+    r = safe_read_exact(fd, buf, sizeof(buf));
+    if (r < 0) {
+      std::cerr << "rbd: failed to decode tag length" << std::endl;
+      return r;
+    }
+
+    bufferlist bl;
+    bl.append(buf, sizeof(buf));
+    auto p = bl.cbegin();
+    decode(*readlen, p);
+  }
+
+  return 0;
+}
+
+int do_import_diff_fd(librados::Rados &rados, librbd::Image &image, int fd,
+		      bool no_progress, int format, size_t sparse_size)
+{
+  int r;
+
+  uint64_t size = 0;
+  bool from_stdin = (fd == STDIN_FILENO);
+  if (!from_stdin) {
+    struct stat stat_buf;
+    r = ::fstat(fd, &stat_buf);
+    if (r < 0) {
+      std::cerr << "rbd: failed to stat specified diff file" << std::endl;
+      return r;
+    }
+    size = (uint64_t)stat_buf.st_size;
+  }
+
+  r = validate_banner(fd, (format == 1 ? utils::RBD_DIFF_BANNER :
+                           utils::RBD_DIFF_BANNER_V2));
+  if (r < 0) {
+    return r;
+  }
+
+  // begin image import
+  std::string tosnap;
+  bool is_protected = false;
+  ImportDiffContext idiffctx(&image, fd, size, no_progress);
+  while (r == 0) {
+    __u8 tag;
+    uint64_t length = 0;
+
+    r = read_tag(fd, RBD_DIFF_END, format, &tag, &length);
+    if (r < 0 || tag == RBD_DIFF_END) {
+      break;
+    }
+
+    if (tag == RBD_DIFF_FROM_SNAP) {
+      r = do_image_snap_from(&idiffctx);
+    } else if (tag == RBD_DIFF_TO_SNAP) {
+      r = do_image_snap_to(&idiffctx, &tosnap);
+    } else if (tag == RBD_SNAP_PROTECTION_STATUS) {
+      r = get_snap_protection_status(&idiffctx, &is_protected);
+    } else if (tag == RBD_DIFF_IMAGE_SIZE) {
+      r = do_image_resize(&idiffctx);
+    } else if (tag == RBD_DIFF_WRITE || tag == RBD_DIFF_ZERO) {
+      r = do_image_io(&idiffctx, (tag == RBD_DIFF_ZERO), sparse_size);
+    } else {
+      std::cerr << "unrecognized tag byte " << (int)tag << " in stream; skipping"
+                << std::endl;
+      r = skip_tag(fd, length);
+    }
+  }
+
+  int temp_r = idiffctx.throttle.wait_for_ret();
+  r = (r < 0) ? r : temp_r; // preserve original error
+  if (r == 0 && tosnap.length()) {
+    r = idiffctx.image->snap_create(tosnap.c_str());
+    if (r == 0 && is_protected) {
+      r = idiffctx.image->snap_protect(tosnap.c_str());
+    }
+  }
+
+  idiffctx.finish(r);
+  return r;
+}
+
+int do_import_diff(librados::Rados &rados, librbd::Image &image,
+		   const char *path, bool no_progress, size_t sparse_size)
+{
+  int r;
+  int fd;
+
+  if (strcmp(path, "-") == 0) {
+    fd = STDIN_FILENO;
+  } else {
+    fd = open(path, O_RDONLY);
+    if (fd < 0) {
+      r = -errno;
+      std::cerr << "rbd: error opening " << path << std::endl;
+      return r;
+    }
+  }
+  r = do_import_diff_fd(rados, image, fd, no_progress, 1, sparse_size);
+
+  if (fd != 0)
+    close(fd);
+  return r;
+}
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+void get_arguments_diff(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_path_options(positional, options,
+                       "import file (or '-' for stdin)");
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_sparse_size_option(options);
+  at::add_no_progress_option(options);
+}
+
+int execute_diff(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  std::string path;
+  size_t arg_index = 0;
+  int r = utils::get_path(vm, &arg_index, &path);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  size_t sparse_size = utils::RBD_DEFAULT_SPARSE_SIZE;
+  if (vm.count(at::IMAGE_SPARSE_SIZE)) {
+    sparse_size = vm[at::IMAGE_SPARSE_SIZE].as<size_t>();
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_import_diff(rados, image, path.c_str(),
+		     vm[at::NO_PROGRESS].as<bool>(), sparse_size);
+  if (r == -EDOM) {
+    r = -EBADMSG;
+  }
+  if (r < 0) {
+    cerr << "rbd: import-diff failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action_diff(
+  {"import-diff"}, {}, "Import an incremental diff.", "", &get_arguments_diff,
+  &execute_diff);
+
+class C_Import : public Context {
+public:
+  C_Import(SimpleThrottle &simple_throttle, librbd::Image &image,
+           bufferlist &bl, uint64_t offset)
+    : m_throttle(simple_throttle), m_image(image),
+      m_aio_completion(
+        new librbd::RBD::AioCompletion(this, &utils::aio_context_callback)),
+      m_bufferlist(bl), m_offset(offset)
+  {
+  }
+
+  void send()
+  {
+    m_throttle.start_op();
+
+    int op_flags = LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL |
+                   LIBRADOS_OP_FLAG_FADVISE_NOCACHE;
+    int r = m_image.aio_write2(m_offset, m_bufferlist.length(), m_bufferlist,
+                               m_aio_completion, op_flags);
+    if (r < 0) {
+      std::cerr << "rbd: error requesting write to destination image"
+                << std::endl;
+      m_aio_completion->release();
+      m_throttle.end_op(r);
+    }
+  }
+
+  void finish(int r) override
+  {
+    if (r < 0) {
+      std::cerr << "rbd: error writing to destination image at offset "
+                << m_offset << ": " << cpp_strerror(r) << std::endl;
+    }
+    m_throttle.end_op(r);
+  }
+
+private:
+  SimpleThrottle &m_throttle;
+  librbd::Image &m_image;
+  librbd::RBD::AioCompletion *m_aio_completion;
+  bufferlist m_bufferlist;
+  uint64_t m_offset;
+};
+
+static int decode_and_set_image_option(int fd, uint64_t imageopt, librbd::ImageOptions& opts)
+{
+  int r;
+  char buf[sizeof(uint64_t)];
+
+  r = safe_read_exact(fd, buf, sizeof(buf));
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode image option" << std::endl;
+    return r;
+  }
+
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  auto it = bl.cbegin();
+
+  uint64_t val;
+  decode(val, it);
+
+  if (opts.get(imageopt, &val) != 0) {
+    opts.set(imageopt, val);
+  }
+
+  return 0;
+}
+
+static int do_import_metadata(int import_format, librbd::Image& image,
+                              const std::map<std::string, std::string> &imagemetas)
+{
+  int r = 0;
+
+  //v1 format
+  if (import_format == 1) {
+    return 0;
+  }
+
+  for (std::map<std::string, std::string>::const_iterator it = imagemetas.begin();
+       it != imagemetas.end(); ++it) {
+    r = image.metadata_set(it->first, it->second);
+    if (r < 0)
+      return r;
+  }
+
+  return 0;
+}
+
+static int decode_imagemeta(int fd, uint64_t length, std::map<std::string, std::string>* imagemetas)
+{
+  int r;
+  string key;
+  string value;
+
+  r = utils::read_string(fd, length, &key);
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode metadata key" << std::endl;
+    return r;
+  }
+
+  r = utils::read_string(fd, length, &value);
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode metadata value" << std::endl;
+    return r;
+  }
+
+  (*imagemetas)[key] = value;
+  return 0;
+}
+
+static int do_import_header(int fd, int import_format, librbd::ImageOptions& opts,
+                            std::map<std::string, std::string>* imagemetas)
+{
+  // There is no header in v1 image.
+  if (import_format == 1) {
+    return 0;
+  }
+
+  int r;
+  r = validate_banner(fd, utils::RBD_IMAGE_BANNER_V2);
+  if (r < 0) {
+    return r;
+  }
+
+  // As V1 format for image is already deprecated, import image in V2 by default.
+  uint64_t image_format = 2;
+  if (opts.get(RBD_IMAGE_OPTION_FORMAT, &image_format) != 0) {
+    opts.set(RBD_IMAGE_OPTION_FORMAT, image_format);
+  }
+
+  while (r == 0) {
+    __u8 tag;
+    uint64_t length = 0;
+    r = read_tag(fd, RBD_EXPORT_IMAGE_END, image_format, &tag, &length);
+    if (r < 0 || tag == RBD_EXPORT_IMAGE_END) {
+      break;
+    }
+
+    if (tag == RBD_EXPORT_IMAGE_ORDER) {
+      r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_ORDER, opts);
+    } else if (tag == RBD_EXPORT_IMAGE_FEATURES) {
+      r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_FEATURES, opts);
+    } else if (tag == RBD_EXPORT_IMAGE_STRIPE_UNIT) {
+      r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_STRIPE_UNIT, opts);
+    } else if (tag == RBD_EXPORT_IMAGE_STRIPE_COUNT) {
+      r = decode_and_set_image_option(fd, RBD_IMAGE_OPTION_STRIPE_COUNT, opts);
+    } else if (tag == RBD_EXPORT_IMAGE_META) {
+      r = decode_imagemeta(fd, length, imagemetas);
+    } else {
+      std::cerr << "rbd: invalid tag in image properties zone: " << tag << "Skip it."
+                << std::endl;
+      r = skip_tag(fd, length);
+    }
+  }
+
+  return r;
+}
+
+static int do_import_v2(librados::Rados &rados, int fd, librbd::Image &image,
+			uint64_t size, size_t imgblklen,
+			utils::ProgressContext &pc, size_t sparse_size)
+{
+  int r = 0;
+  r = validate_banner(fd, utils::RBD_IMAGE_DIFFS_BANNER_V2);
+  if (r < 0) {
+    return r;
+  }
+
+  char buf[sizeof(uint64_t)];
+  r = safe_read_exact(fd, buf, sizeof(buf));
+  if (r < 0) {
+    std::cerr << "rbd: failed to decode diff count" << std::endl;
+    return r;
+  }
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  auto p = bl.cbegin();
+  uint64_t diff_num;
+  decode(diff_num, p);
+  for (size_t i = 0; i < diff_num; i++) {
+    r = do_import_diff_fd(rados, image, fd, true, 2, sparse_size);
+    if (r < 0) {
+      pc.fail();
+      std::cerr << "rbd: import-diff failed: " << cpp_strerror(r) << std::endl;
+      return r;
+    }
+    pc.update_progress(i + 1, diff_num);
+  }
+
+  return r;
+}
+
+static int do_import_v1(int fd, librbd::Image &image, uint64_t size,
+                        size_t imgblklen, utils::ProgressContext &pc,
+			size_t sparse_size)
+{
+  int r = 0;
+  size_t reqlen = imgblklen;    // amount requested from read
+  ssize_t readlen;              // amount received from one read
+  size_t blklen = 0;            // amount accumulated from reads to fill blk
+  char *p = new char[imgblklen];
+  uint64_t image_pos = 0;
+  bool from_stdin = (fd == STDIN_FILENO);
+  boost::scoped_ptr<SimpleThrottle> throttle;
+
+  if (from_stdin) {
+    throttle.reset(new SimpleThrottle(1, false));
+  } else {
+    throttle.reset(new SimpleThrottle(
+      g_conf().get_val<uint64_t>("rbd_concurrent_management_ops"), false));
+  }
+
+  reqlen = min<uint64_t>(reqlen, size);
+  // loop body handles 0 return, as we may have a block to flush
+  while ((readlen = ::read(fd, p + blklen, reqlen)) >= 0) {
+    if (throttle->pending_error()) {
+      break;
+    }
+
+    blklen += readlen;
+    // if read was short, try again to fill the block before writing
+    if (readlen && ((size_t)readlen < reqlen)) {
+      reqlen -= readlen;
+      continue;
+    }
+    if (!from_stdin)
+      pc.update_progress(image_pos, size);
+
+    bufferptr blkptr(p, blklen); 
+    // resize output image by binary expansion as we go for stdin
+    if (from_stdin && (image_pos + (size_t)blklen) > size) {
+      size *= 2;
+      r = image.resize(size);
+      if (r < 0) {
+	std::cerr << "rbd: can't resize image during import" << std::endl;
+	goto out;
+      }
+    }
+
+    // write as much as we got; perhaps less than imgblklen
+    // but skip writing zeros to create sparse images
+    size_t buffer_offset = 0;
+    while (buffer_offset < blklen) {
+      size_t write_length = 0;
+      bool zeroed = false;
+      utils::calc_sparse_extent(blkptr, sparse_size, buffer_offset, blklen,
+				&write_length, &zeroed);
+
+      if (!zeroed) {
+	bufferlist write_bl;
+	bufferptr write_ptr(blkptr, buffer_offset, write_length);
+	write_bl.push_back(write_ptr);
+	ceph_assert(write_bl.length() == write_length);
+
+	C_Import *ctx = new C_Import(*throttle, image, write_bl,
+				     image_pos + buffer_offset);
+	ctx->send();
+      }
+
+      buffer_offset += write_length;
+    }
+
+    // done with whole block, whether written or not
+    image_pos += blklen;
+    if (!from_stdin && image_pos >= size)
+      break;
+    // if read had returned 0, we're at EOF and should quit
+    if (readlen == 0)
+      break;
+    blklen = 0;
+    reqlen = imgblklen;
+  }
+  r = throttle->wait_for_ret();
+  if (r < 0) {
+    goto out;
+  }
+
+  if (fd == STDIN_FILENO) {
+    r = image.resize(image_pos);
+    if (r < 0) {
+      std::cerr << "rbd: final image resize failed" << std::endl;
+      goto out;
+    }
+  }
+out:
+  delete[] p;
+  return r;
+}
+
+static int do_import(librados::Rados &rados, librbd::RBD &rbd,
+		     librados::IoCtx& io_ctx, const char *imgname,
+		     const char *path, librbd::ImageOptions& opts,
+		     bool no_progress, int import_format, size_t sparse_size)
+{
+  int fd, r;
+  struct stat stat_buf;
+  utils::ProgressContext pc("Importing image", no_progress);
+  std::map<std::string, std::string> imagemetas;
+
+  ceph_assert(imgname);
+
+  uint64_t order;
+  if (opts.get(RBD_IMAGE_OPTION_ORDER, &order) != 0) {
+    order = g_conf().get_val<uint64_t>("rbd_default_order");
+  }
+
+  // try to fill whole imgblklen blocks for sparsification
+  size_t imgblklen = 1 << order;
+  librbd::Image image;
+  uint64_t size = 0;
+
+  bool from_stdin = !strcmp(path, "-");
+  if (from_stdin) {
+    fd = STDIN_FILENO;
+    size = 1ULL << order;
+  } else {
+    if ((fd = open(path, O_RDONLY)) < 0) {
+      r = -errno;
+      std::cerr << "rbd: error opening " << path << std::endl;
+      goto done2;
+    }
+
+    if ((fstat(fd, &stat_buf)) < 0) {
+      r = -errno;
+      std::cerr << "rbd: stat error " << path << std::endl;
+      goto done;
+    }
+    if (S_ISDIR(stat_buf.st_mode)) {
+      r = -EISDIR;
+      std::cerr << "rbd: cannot import a directory" << std::endl;
+      goto done;
+    }
+    if (stat_buf.st_size)
+      size = (uint64_t)stat_buf.st_size;
+
+    if (!size) {
+      int64_t bdev_size = 0;
+      BlkDev blkdev(fd);
+      r = blkdev.get_size(&bdev_size);
+      if (r < 0) {
+        std::cerr << "rbd: unable to get size of file/block device"
+                  << std::endl;
+        goto done;
+      }
+      ceph_assert(bdev_size >= 0);
+      size = (uint64_t) bdev_size;
+    }
+#ifdef HAVE_POSIX_FADVISE
+    posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+#endif
+  }
+
+  r = do_import_header(fd, import_format, opts, &imagemetas);
+  if (r < 0) {
+    std::cerr << "rbd: import header failed." << std::endl;
+    goto done;
+  }
+
+  r = rbd.create4(io_ctx, imgname, size, opts);
+  if (r < 0) {
+    std::cerr << "rbd: image creation failed" << std::endl;
+    goto done;
+  }
+
+  r = rbd.open(io_ctx, image, imgname);
+  if (r < 0) {
+    std::cerr << "rbd: failed to open image" << std::endl;
+    goto err;
+  }
+
+  r = do_import_metadata(import_format, image, imagemetas);
+  if (r < 0) {
+    std::cerr << "rbd: failed to import image-meta" << std::endl;
+    goto err;
+  }
+
+  if (import_format == 1) {
+    r = do_import_v1(fd, image, size, imgblklen, pc, sparse_size);
+  } else {
+    r = do_import_v2(rados, fd, image, size, imgblklen, pc, sparse_size);
+  }
+  if (r < 0) {
+    std::cerr << "rbd: failed to import image" << std::endl;
+    image.close();
+    goto err;
+  }
+
+  r = image.close();
+err:
+  if (r < 0)
+    rbd.remove(io_ctx, imgname);
+done:
+  if (r < 0)
+    pc.fail();
+  else
+    pc.finish();
+  if (!from_stdin)
+    close(fd);
+done2:
+  return r;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_path_options(positional, options,
+                       "import file (or '-' for stdin)");
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
+  at::add_create_image_options(options, true);
+  at::add_sparse_size_option(options);
+  at::add_no_progress_option(options);
+  at::add_export_format_option(options);
+
+  // TODO legacy rbd allowed import to accept both 'image'/'dest' and
+  //      'pool'/'dest-pool'
+  at::add_pool_option(options, at::ARGUMENT_MODIFIER_NONE, " (deprecated)");
+  at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE, " (deprecated)");
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  std::string path;
+  size_t arg_index = 0;
+  int r = utils::get_path(vm, &arg_index, &path);
+  if (r < 0) {
+    return r;
+  }
+
+  // odd check to support legacy / deprecated behavior of import
+  std::string deprecated_pool_name;
+  if (vm.count(at::POOL_NAME)) {
+    deprecated_pool_name = vm[at::POOL_NAME].as<std::string>();
+    std::cerr << "rbd: --pool is deprecated for import, use --dest-pool"
+              << std::endl;
+  }
+
+  std::string deprecated_image_name;
+  if (vm.count(at::IMAGE_NAME)) {
+    deprecated_image_name = vm[at::IMAGE_NAME].as<std::string>();
+    std::cerr << "rbd: --image is deprecated for import, use --dest"
+              << std::endl;
+  } else {
+    deprecated_image_name = path.substr(path.find_last_of("/") + 1);
+  }
+
+  std::string deprecated_snap_name;
+  r = utils::extract_spec(deprecated_image_name, &deprecated_pool_name,
+                          nullptr, &deprecated_image_name,
+                          &deprecated_snap_name, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  size_t sparse_size = utils::RBD_DEFAULT_SPARSE_SIZE;
+  if (vm.count(at::IMAGE_SPARSE_SIZE)) {
+    sparse_size = vm[at::IMAGE_SPARSE_SIZE].as<size_t>();
+  }
+
+  std::string pool_name = deprecated_pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name = deprecated_snap_name;
+  r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, false, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  if (image_name.empty()) {
+    image_name = deprecated_image_name;
+  }
+
+  librbd::ImageOptions opts;
+  r = utils::get_image_options(vm, true, &opts);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  int format = 1;
+  if (vm.count("export-format"))
+    format = vm["export-format"].as<uint64_t>();
+
+  librbd::RBD rbd;
+  r = do_import(rados, rbd, io_ctx, image_name.c_str(), path.c_str(),
+                opts, vm[at::NO_PROGRESS].as<bool>(), format, sparse_size);
+  if (r < 0) {
+    std::cerr << "rbd: import failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+Shell::Action action(
+  {"import"}, {}, "Import image from file.", at::get_long_features_help(),
+  &get_arguments, &execute);
+
+} // namespace import
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Info.cc b/src/tools/rbd/action/Info.cc
new file mode 100644
index 00000000..5adacb92
--- /dev/null
+++ b/src/tools/rbd/action/Info.cc
@@ -0,0 +1,459 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/types.h"
+#include "include/stringify.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+#include "common/Clock.h"
+
+namespace rbd {
+namespace action {
+namespace info {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static void format_bitmask(Formatter *f, const std::string &name,
+                           const std::map<uint64_t, std::string>& mapping,
+                           uint64_t bitmask)
+{
+  int count = 0;
+  std::string group_name(name + "s");
+  if (f == NULL) {
+    std::cout << "\t" << group_name << ": ";
+  } else {
+    f->open_array_section(group_name.c_str());
+  }
+  for (std::map<uint64_t, std::string>::const_iterator it = mapping.begin();
+       it != mapping.end(); ++it) {
+    if ((it->first & bitmask) == 0) {
+      continue;
+    }
+
+    if (f == NULL) {
+      if (count++ > 0) {
+        std::cout << ", ";
+      }
+      std::cout << it->second;
+    } else {
+      f->dump_string(name.c_str(), it->second);
+    }
+  }
+  if (f == NULL) {
+    std::cout << std::endl;
+  } else {
+    f->close_section();
+  }
+}
+
+static void format_features(Formatter *f, uint64_t features)
+{
+  format_bitmask(f, "feature", at::ImageFeatures::FEATURE_MAPPING, features);
+}
+
+static void format_op_features(Formatter *f, uint64_t op_features)
+{
+  static std::map<uint64_t, std::string> mapping = {
+    {RBD_OPERATION_FEATURE_CLONE_PARENT, RBD_OPERATION_FEATURE_NAME_CLONE_PARENT},
+    {RBD_OPERATION_FEATURE_CLONE_CHILD, RBD_OPERATION_FEATURE_NAME_CLONE_CHILD},
+    {RBD_OPERATION_FEATURE_GROUP, RBD_OPERATION_FEATURE_NAME_GROUP},
+    {RBD_OPERATION_FEATURE_SNAP_TRASH, RBD_OPERATION_FEATURE_NAME_SNAP_TRASH}};
+  format_bitmask(f, "op_feature", mapping, op_features);
+}
+
+static void format_flags(Formatter *f, uint64_t flags)
+{
+  std::map<uint64_t, std::string> mapping = {
+    {RBD_FLAG_OBJECT_MAP_INVALID, "object map invalid"},
+    {RBD_FLAG_FAST_DIFF_INVALID, "fast diff invalid"}};
+  format_bitmask(f, "flag", mapping, flags);
+}
+
+void format_timestamp(struct timespec timestamp, std::string &timestamp_str) {
+  if(timestamp.tv_sec != 0) {
+    time_t ts = timestamp.tv_sec;
+    timestamp_str = ctime(&ts);
+    timestamp_str = timestamp_str.substr(0, timestamp_str.length() - 1);
+  }
+}
+
+static int do_show_info(librados::IoCtx &io_ctx, librbd::Image& image,
+                        const std::string &snapname, Formatter *f)
+{
+  librbd::image_info_t info;
+  uint8_t old_format;
+  uint64_t overlap, features, flags, snap_limit;
+  bool snap_protected = false;
+  librbd::mirror_image_info_t mirror_image;
+  std::vector<librbd::snap_info_t> snaps;
+  int r;
+
+  std::string imgname;
+  r = image.get_name(&imgname);
+  if (r < 0)
+    return r;
+
+  r = image.snap_list(snaps);
+  if (r < 0)
+    return r;
+
+  r = image.stat(info, sizeof(info));
+  if (r < 0)
+    return r;
+
+  r = image.old_format(&old_format);
+  if (r < 0)
+    return r;
+
+  std::string imgid;
+  if (!old_format) {
+    r = image.get_id(&imgid);
+    if (r < 0)
+      return r;
+  }
+
+  std::string data_pool;
+  if (!old_format) {
+    int64_t data_pool_id = image.get_data_pool_id();
+    if (data_pool_id != io_ctx.get_id()) {
+      librados::Rados rados(io_ctx);
+      librados::IoCtx data_io_ctx;
+      r = rados.ioctx_create2(data_pool_id, data_io_ctx);
+      if (r < 0) {
+        data_pool = "<missing data pool " + stringify(data_pool_id) + ">";
+      } else {
+        data_pool = data_io_ctx.get_pool_name();
+      }
+    }
+  }
+
+  r = image.overlap(&overlap);
+  if (r < 0)
+    return r;
+
+  r = image.features(&features);
+  if (r < 0)
+    return r;
+
+  uint64_t op_features;
+  r = image.get_op_features(&op_features);
+  if (r < 0) {
+    return r;
+  }
+
+  r = image.get_flags(&flags);
+  if (r < 0) {
+    return r;
+  }
+
+  if (!snapname.empty()) {
+    r = image.snap_is_protected(snapname.c_str(), &snap_protected);
+    if (r < 0)
+      return r;
+  }
+
+  if (features & RBD_FEATURE_JOURNALING) {
+    r = image.mirror_image_get_info(&mirror_image, sizeof(mirror_image));
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  r = image.snap_get_limit(&snap_limit);
+  if (r < 0)
+    return r;
+
+  std::string prefix = image.get_block_name_prefix();
+
+  librbd::group_info_t group_info;
+  r = image.get_group(&group_info, sizeof(group_info));
+  if (r < 0) {
+    return r;
+  }
+
+  std::string group_string = "";
+  if (RBD_GROUP_INVALID_POOL != group_info.pool) {
+    std::string group_pool;
+    librados::Rados rados(io_ctx);
+    librados::IoCtx group_io_ctx;
+    r = rados.ioctx_create2(group_info.pool, group_io_ctx);
+    if (r < 0) {
+      group_pool = "<missing group pool " + stringify(group_info.pool) + ">";
+    } else {
+      group_pool = group_io_ctx.get_pool_name();
+    }
+
+    group_string = group_pool + "/";
+    if (!io_ctx.get_namespace().empty()) {
+      group_string += io_ctx.get_namespace() + "/";
+    }
+    group_string += group_info.name;
+  }
+
+  struct timespec create_timestamp;
+  image.get_create_timestamp(&create_timestamp);
+
+  std::string create_timestamp_str = "";
+  format_timestamp(create_timestamp, create_timestamp_str);
+
+  struct timespec access_timestamp;
+  image.get_access_timestamp(&access_timestamp);
+
+  std::string access_timestamp_str = "";
+  format_timestamp(access_timestamp, access_timestamp_str);
+
+  struct timespec modify_timestamp;
+  image.get_modify_timestamp(&modify_timestamp);
+
+  std::string modify_timestamp_str = "";
+  format_timestamp(modify_timestamp, modify_timestamp_str);
+
+  if (f) {
+    f->open_object_section("image");
+    f->dump_string("name", imgname);
+    f->dump_string("id", imgid);
+    f->dump_unsigned("size", info.size);
+    f->dump_unsigned("objects", info.num_objs);
+    f->dump_int("order", info.order);
+    f->dump_unsigned("object_size", info.obj_size);
+    f->dump_int("snapshot_count", snaps.size());
+    if (!data_pool.empty()) {
+      f->dump_string("data_pool", data_pool);
+    }
+    f->dump_string("block_name_prefix", prefix);
+    f->dump_int("format", (old_format ? 1 : 2));
+  } else {
+    std::cout << "rbd image '" << imgname << "':\n"
+              << "\tsize " << byte_u_t(info.size) << " in "
+              << info.num_objs << " objects"
+              << std::endl
+              << "\torder " << info.order
+              << " (" << byte_u_t(info.obj_size) << " objects)"
+              << std::endl
+              << "\tsnapshot_count: " << snaps.size()
+              << std::endl;
+    if (!imgid.empty()) {
+      std::cout << "\tid: " << imgid << std::endl;
+    }
+    if (!data_pool.empty()) {
+      std::cout << "\tdata_pool: " << data_pool << std::endl;
+    }
+    std::cout << "\tblock_name_prefix: " << prefix
+              << std::endl
+              << "\tformat: " << (old_format ? "1" : "2")
+	      << std::endl;
+  }
+
+  if (!old_format) {
+    format_features(f, features);
+    format_op_features(f, op_features);
+    format_flags(f, flags);
+  }
+
+  if (!group_string.empty()) {
+    if (f) {
+      f->dump_string("group", group_string);
+    } else {
+      std::cout << "\tgroup: " << group_string
+		<< std::endl;
+    }
+  }
+
+  if (!create_timestamp_str.empty()) {
+    if (f) {
+      f->dump_string("create_timestamp", create_timestamp_str);
+    } else {
+      std::cout << "\tcreate_timestamp: " << create_timestamp_str
+                << std::endl;
+    }
+  }
+
+  if (!access_timestamp_str.empty()) {
+    if (f) {
+      f->dump_string("access_timestamp", access_timestamp_str);
+    } else {
+      std::cout << "\taccess_timestamp: " << access_timestamp_str
+                << std::endl;
+    }
+  }
+
+  if (!modify_timestamp_str.empty()) {
+    if (f) {
+      f->dump_string("modify_timestamp", modify_timestamp_str);
+    } else {
+      std::cout << "\tmodify_timestamp: " << modify_timestamp_str
+                << std::endl;
+    }
+  }
+
+  // snapshot info, if present
+  if (!snapname.empty()) {
+    if (f) {
+      f->dump_string("protected", snap_protected ? "true" : "false");
+    } else {
+      std::cout << "\tprotected: " << (snap_protected ? "True" : "False")
+                << std::endl;
+    }
+  }
+
+  if (snap_limit < UINT64_MAX) {
+    if (f) {
+      f->dump_unsigned("snapshot_limit", snap_limit);
+    } else {
+      std::cout << "\tsnapshot_limit: " << snap_limit << std::endl;
+    }
+  }
+
+  // parent info, if present
+  librbd::linked_image_spec_t parent_image_spec;
+  librbd::snap_spec_t parent_snap_spec;
+  if ((image.get_parent(&parent_image_spec, &parent_snap_spec) == 0) &&
+      (parent_image_spec.image_name.length() > 0)) {
+    if (f) {
+      f->open_object_section("parent");
+      f->dump_string("pool", parent_image_spec.pool_name);
+      f->dump_string("pool_namespace", parent_image_spec.pool_namespace);
+      f->dump_string("image", parent_image_spec.image_name);
+      f->dump_string("id", parent_image_spec.image_id);
+      f->dump_string("snapshot", parent_snap_spec.name);
+      f->dump_bool("trash", parent_image_spec.trash);
+      f->dump_unsigned("overlap", overlap);
+      f->close_section();
+    } else {
+      std::cout << "\tparent: " << parent_image_spec.pool_name << "/";
+      if (!parent_image_spec.pool_namespace.empty()) {
+        std::cout << parent_image_spec.pool_namespace << "/";
+      }
+      std::cout << parent_image_spec.image_name << "@"
+                << parent_snap_spec.name;
+      if (parent_image_spec.trash) {
+        std::cout << " (trash " << parent_image_spec.image_id << ")";
+      }
+      std::cout << std::endl;
+      std::cout << "\toverlap: " << byte_u_t(overlap) << std::endl;
+    }
+  }
+
+  // striping info, if feature is set
+  if (features & RBD_FEATURE_STRIPINGV2) {
+    if (f) {
+      f->dump_unsigned("stripe_unit", image.get_stripe_unit());
+      f->dump_unsigned("stripe_count", image.get_stripe_count());
+    } else {
+      std::cout << "\tstripe unit: " << byte_u_t(image.get_stripe_unit())
+                << std::endl
+                << "\tstripe count: " << image.get_stripe_count() << std::endl;
+    }
+  }
+
+  if (features & RBD_FEATURE_JOURNALING) {
+    if (f) {
+      f->dump_string("journal", utils::image_id(image));
+    } else {
+      std::cout << "\tjournal: " << utils::image_id(image) << std::endl;
+    }
+  }
+
+  if (features & RBD_FEATURE_JOURNALING) {
+    if (f) {
+      f->open_object_section("mirroring");
+      f->dump_string("state",
+          utils::mirror_image_state(mirror_image.state));
+      if (mirror_image.state != RBD_MIRROR_IMAGE_DISABLED) {
+        f->dump_string("global_id", mirror_image.global_id);
+        f->dump_bool("primary", mirror_image.primary);
+      }
+      f->close_section();
+    } else {
+      std::cout << "\tmirroring state: "
+                << utils::mirror_image_state(mirror_image.state) << std::endl;
+      if (mirror_image.state != RBD_MIRROR_IMAGE_DISABLED) {
+        std::cout << "\tmirroring global id: " << mirror_image.global_id
+                  << std::endl
+                  << "\tmirroring primary: "
+                  << (mirror_image.primary ? "true" : "false") <<std::endl;
+      }
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  }
+
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_id_option(options);
+  at::add_format_options(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  std::string image_id;
+
+  if (vm.count(at::IMAGE_ID)) {
+    image_id = vm[at::IMAGE_ID].as<std::string>();
+  }
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, image_id.empty(),
+    utils::SNAPSHOT_PRESENCE_PERMITTED, utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  if (!image_id.empty() && !image_name.empty()) {
+    std::cerr << "rbd: trying to access image using both name and id. "
+              << std::endl;
+    return -EINVAL;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name,
+                                 image_id, snap_name, true, &rados, &io_ctx,
+                                 &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_show_info(io_ctx, image, snap_name, formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: info: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"info"}, {}, "Show information about image size, striping, etc.", "",
+  &get_arguments, &execute);
+
+} // namespace info
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Journal.cc b/src/tools/rbd/action/Journal.cc
new file mode 100644
index 00000000..d3a54f94
--- /dev/null
+++ b/src/tools/rbd/action/Journal.cc
@@ -0,0 +1,1254 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/Cond.h"
+#include "common/Formatter.h"
+#include "common/ceph_json.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "include/stringify.h"
+#include <fstream>
+#include <sstream>
+#include <boost/program_options.hpp>
+#include "cls/rbd/cls_rbd_client.h"
+#include "cls/journal/cls_journal_types.h"
+#include "cls/journal/cls_journal_client.h"
+
+#include "journal/Journaler.h"
+#include "journal/ReplayEntry.h"
+#include "journal/ReplayHandler.h"
+#include "journal/Settings.h"
+#include "librbd/journal/Types.h"
+
+namespace rbd {
+namespace action {
+namespace journal {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static const std::string JOURNAL_SPEC("journal-spec");
+static const std::string JOURNAL_NAME("journal");
+static const std::string DEST_JOURNAL_NAME("dest-journal");
+
+void add_journal_option(po::options_description *opt,
+                        at::ArgumentModifier modifier) {
+  std::string name = JOURNAL_NAME;
+  std::string description = at::get_description_prefix(modifier) +
+                            "journal name";
+  switch (modifier) {
+  case at::ARGUMENT_MODIFIER_NONE:
+  case at::ARGUMENT_MODIFIER_SOURCE:
+    break;
+  case at::ARGUMENT_MODIFIER_DEST:
+    name = DEST_JOURNAL_NAME;
+    break;
+  }
+
+  // TODO add validator
+  opt->add_options()
+    (name.c_str(), po::value<std::string>(), description.c_str());
+}
+
+void add_journal_spec_options(po::options_description *pos,
+			      po::options_description *opt,
+			      at::ArgumentModifier modifier) {
+
+  pos->add_options()
+    ((get_name_prefix(modifier) + JOURNAL_SPEC).c_str(),
+     (get_description_prefix(modifier) + "journal specification\n" +
+      "(example: [<pool-name>/[<namespace>/]]<journal-name>)").c_str());
+  add_pool_option(opt, modifier);
+  add_namespace_option(opt, modifier);
+  add_image_option(opt, modifier);
+  add_journal_option(opt, modifier);
+}
+
+int get_pool_journal_names(const po::variables_map &vm,
+			   at::ArgumentModifier mod,
+			   size_t *spec_arg_index,
+			   std::string *pool_name,
+			   std::string *namespace_name,
+			   std::string *journal_name) {
+  std::string pool_key = (mod == at::ARGUMENT_MODIFIER_DEST ?
+    at::DEST_POOL_NAME : at::POOL_NAME);
+  std::string namespace_key = (mod == at::ARGUMENT_MODIFIER_DEST ?
+    at::DEST_NAMESPACE_NAME : at::NAMESPACE_NAME);
+  std::string image_key = (mod == at::ARGUMENT_MODIFIER_DEST ?
+    at::DEST_IMAGE_NAME : at::IMAGE_NAME);
+  std::string journal_key = (mod == at::ARGUMENT_MODIFIER_DEST ?
+    DEST_JOURNAL_NAME : JOURNAL_NAME);
+
+  if (vm.count(pool_key) && pool_name != nullptr) {
+    *pool_name = vm[pool_key].as<std::string>();
+  }
+  if (vm.count(namespace_key) && namespace_name != nullptr) {
+    *namespace_name = vm[namespace_key].as<std::string>();
+  }
+  if (vm.count(journal_key) && journal_name != nullptr) {
+    *journal_name = vm[journal_key].as<std::string>();
+  }
+
+  std::string image_name;
+  if (vm.count(image_key)) {
+    image_name = vm[image_key].as<std::string>();
+  }
+
+  int r;
+  if (journal_name != nullptr && !journal_name->empty()) {
+    // despite the separate pool option,
+    // we can also specify them via the journal option
+    std::string journal_name_copy(*journal_name);
+    r = extract_spec(journal_name_copy, pool_name, namespace_name, journal_name,
+                     nullptr, utils::SPEC_VALIDATION_FULL);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  if (!image_name.empty()) {
+    // despite the separate pool option,
+    // we can also specify them via the image option
+    std::string image_name_copy(image_name);
+    r = extract_spec(image_name_copy, pool_name, namespace_name, &image_name,
+                     nullptr, utils::SPEC_VALIDATION_NONE);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  if (journal_name != nullptr && spec_arg_index != nullptr &&
+      journal_name->empty()) {
+    std::string spec = utils::get_positional_argument(vm, (*spec_arg_index)++);
+    if (!spec.empty()) {
+      r = extract_spec(spec, pool_name, namespace_name, journal_name, nullptr,
+                       utils::SPEC_VALIDATION_FULL);
+      if (r < 0) {
+        return r;
+      }
+    }
+  }
+
+  if (pool_name != nullptr && pool_name->empty()) {
+    *pool_name = utils::get_default_pool_name();
+  }
+
+  if (pool_name != nullptr && namespace_name != nullptr &&
+      journal_name != nullptr && journal_name->empty() && !image_name.empty()) {
+    // Try to get journal name from image info.
+    librados::Rados rados;
+    librados::IoCtx io_ctx;
+    librbd::Image image;
+    int r = utils::init_and_open_image(*pool_name, *namespace_name, image_name,
+                                       "", "", true, &rados, &io_ctx, &image);
+    if (r < 0) {
+      std::cerr << "rbd: failed to open image " << image_name
+		<< " to get journal name: " << cpp_strerror(r) << std::endl;
+      return r;
+    }
+
+    uint64_t features;
+    r = image.features(&features);
+    if (r < 0) {
+      return r;
+    }
+    if ((features & RBD_FEATURE_JOURNALING) == 0) {
+      std::cerr << "rbd: journaling is not enabled for image " << image_name
+		<< std::endl;
+      return -EINVAL;
+    }
+    *journal_name = utils::image_id(image);
+  }
+
+  if (journal_name != nullptr && journal_name->empty()) {
+    std::string prefix = at::get_description_prefix(mod);
+    std::cerr << "rbd: "
+              << (mod == at::ARGUMENT_MODIFIER_DEST ? prefix : std::string())
+              << "journal was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+static int do_show_journal_info(librados::Rados& rados, librados::IoCtx& io_ctx,
+				const std::string& journal_id, Formatter *f)
+{
+  int r;
+  C_SaferCond cond;
+
+  std::string header_oid = ::journal::Journaler::header_oid(journal_id);
+  std::string object_oid_prefix = ::journal::Journaler::object_oid_prefix(
+    io_ctx.get_id(), journal_id);
+  uint8_t order;
+  uint8_t splay_width;
+  int64_t pool_id;
+
+  cls::journal::client::get_immutable_metadata(io_ctx, header_oid, &order,
+					       &splay_width, &pool_id, &cond);
+  r = cond.wait();
+  if (r < 0) {
+    std::cerr << "failed to get journal metadata: "  << cpp_strerror(r)
+	      << std::endl;
+    return r;
+  }
+
+  std::string object_pool_name;
+  if (pool_id >= 0) {
+    r = rados.pool_reverse_lookup(pool_id, &object_pool_name);
+    if (r < 0) {
+      std::cerr << "error looking up pool name for pool_id=" << pool_id << ": "
+		<< cpp_strerror(r) << std::endl;
+    }
+  }
+
+  if (f) {
+    f->open_object_section("journal");
+    f->dump_string("journal_id", journal_id);
+    f->dump_string("header_oid", header_oid);
+    f->dump_string("object_oid_prefix", object_oid_prefix);
+    f->dump_int("order", order);
+    f->dump_int("splay_width", splay_width);
+    if (!object_pool_name.empty()) {
+      f->dump_string("object_pool", object_pool_name);
+    }
+    f->close_section();
+    f->flush(std::cout);
+  } else {
+    std::cout << "rbd journal '" << journal_id << "':" << std::endl;
+    std::cout << "\theader_oid: " << header_oid << std::endl;
+    std::cout << "\tobject_oid_prefix: " << object_oid_prefix << std::endl;
+    std::cout << "\torder: " << static_cast<int>(order) << " ("
+	      << byte_u_t(1ull << order) << " objects)"<< std::endl;
+    std::cout << "\tsplay_width: " << static_cast<int>(splay_width) << std::endl;
+    if (!object_pool_name.empty()) {
+      std::cout << "\tobject_pool: " << object_pool_name << std::endl;
+    }
+  }
+  return 0;
+}
+
+static int do_show_journal_status(librados::IoCtx& io_ctx,
+				  const std::string& journal_id, Formatter *f)
+{
+  int r;
+
+  C_SaferCond cond;
+  uint64_t minimum_set;
+  uint64_t active_set;
+  std::set<cls::journal::Client> registered_clients;
+  std::string oid = ::journal::Journaler::header_oid(journal_id);
+
+  cls::journal::client::get_mutable_metadata(io_ctx, oid, &minimum_set,
+                                            &active_set, &registered_clients,
+                                            &cond);
+  r = cond.wait();
+  if (r < 0) {
+    std::cerr << "warning: failed to get journal metadata" << std::endl;
+    return r;
+  }
+
+  if (f) {
+    f->open_object_section("status");
+    f->dump_unsigned("minimum_set", minimum_set);
+    f->dump_unsigned("active_set", active_set);
+    f->open_array_section("registered_clients");
+    for (std::set<cls::journal::Client>::iterator c =
+          registered_clients.begin(); c != registered_clients.end(); ++c) {
+      f->open_object_section("client");
+      c->dump(f);
+      f->close_section();
+    }
+    f->close_section();
+    f->close_section();
+    f->flush(std::cout);
+  } else {
+    std::cout << "minimum_set: " << minimum_set << std::endl;
+    std::cout << "active_set: " << active_set << std::endl;
+    std::cout << "registered clients: " << std::endl;
+    for (std::set<cls::journal::Client>::iterator c =
+          registered_clients.begin(); c != registered_clients.end(); ++c) {
+      std::cout << "\t" << *c << std::endl;
+    }
+  }
+  return 0;
+}
+
+static int do_reset_journal(librados::IoCtx& io_ctx,
+			    const std::string& journal_id)
+{
+  // disable/re-enable journaling to delete/re-create the journal
+  // to properly handle mirroring constraints
+  std::string image_name;
+  int r = librbd::cls_client::dir_get_name(&io_ctx, RBD_DIRECTORY, journal_id,
+                                           &image_name);
+  if (r < 0) {
+    std::cerr << "failed to locate journal's image: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  librbd::Image image;
+  r = utils::open_image(io_ctx, image_name, false, &image);
+  if (r < 0) {
+    std::cerr << "failed to open image: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  r = image.update_features(RBD_FEATURE_JOURNALING, false);
+  if (r < 0) {
+    std::cerr << "failed to disable image journaling: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  r = image.update_features(RBD_FEATURE_JOURNALING, true);
+  if (r < 0) {
+    std::cerr << "failed to re-enable image journaling: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+static int do_disconnect_journal_client(librados::IoCtx& io_ctx,
+					const std::string& journal_id,
+					const std::string& client_id)
+{
+  int r;
+
+  C_SaferCond cond;
+  uint64_t minimum_set;
+  uint64_t active_set;
+  std::set<cls::journal::Client> registered_clients;
+  std::string oid = ::journal::Journaler::header_oid(journal_id);
+
+  cls::journal::client::get_mutable_metadata(io_ctx, oid, &minimum_set,
+                                            &active_set, &registered_clients,
+                                            &cond);
+  r = cond.wait();
+  if (r < 0) {
+    std::cerr << "warning: failed to get journal metadata" << std::endl;
+    return r;
+  }
+
+  static const std::string IMAGE_CLIENT_ID("");
+
+  bool found = false;
+  for (auto &c : registered_clients) {
+    if (c.id == IMAGE_CLIENT_ID || (!client_id.empty() && client_id != c.id)) {
+      continue;
+    }
+    r = cls::journal::client::client_update_state(io_ctx, oid, c.id,
+				  cls::journal::CLIENT_STATE_DISCONNECTED);
+    if (r < 0) {
+      std::cerr << "warning: failed to disconnect client " << c.id << ": "
+		<< cpp_strerror(r) << std::endl;
+      return r;
+    }
+    std::cout << "client " << c.id << " disconnected" << std::endl;
+    found = true;
+  }
+
+  if (!found) {
+    if (!client_id.empty()) {
+      std::cerr << "warning: client " << client_id << " is not registered"
+		<< std::endl;
+    } else {
+      std::cerr << "no registered clients to disconnect" << std::endl;
+    }
+    return -ENOENT;
+  }
+
+  bufferlist bl;
+  r = io_ctx.notify2(oid, bl, 5000, NULL);
+  if (r < 0) {
+    std::cerr << "warning: failed to notify state change:" << ": "
+	      << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+class Journaler : public ::journal::Journaler {
+public:
+  Journaler(librados::IoCtx& io_ctx, const std::string& journal_id,
+	    const std::string &client_id) :
+    ::journal::Journaler(io_ctx, journal_id, client_id, {}) {
+  }
+
+  int init() {
+    int r;
+
+    // TODO register with librbd payload
+    r = register_client(bufferlist());
+    if (r < 0) {
+      std::cerr << "failed to register client: " << cpp_strerror(r)
+		<< std::endl;
+      return r;
+    }
+
+    C_SaferCond cond;
+
+    ::journal::Journaler::init(&cond);
+    r = cond.wait();
+    if (r < 0) {
+      std::cerr << "failed to initialize journal: " << cpp_strerror(r)
+		<< std::endl;
+      (void) unregister_client();
+      return r;
+    }
+
+    return 0;
+  }
+
+  int shut_down() {
+    int r = unregister_client();
+    if (r < 0) {
+      std::cerr << "rbd: failed to unregister journal client: "
+		<< cpp_strerror(r) << std::endl;
+    }
+    ::journal::Journaler::shut_down();
+
+    return r;
+  }
+};
+
+class JournalPlayer {
+public:
+  JournalPlayer(librados::IoCtx& io_ctx, const std::string& journal_id,
+		const std::string &client_id) :
+    m_journaler(io_ctx, journal_id, client_id),
+    m_cond(),
+    m_r(0) {
+  }
+
+  virtual ~JournalPlayer() {}
+
+  virtual int exec() {
+    int r;
+
+    r = m_journaler.init();
+    if (r < 0) {
+      return r;
+    }
+
+    ReplayHandler replay_handler(this);
+
+    m_journaler.start_replay(&replay_handler);
+
+    r = m_cond.wait();
+    if (r < 0) {
+      std::cerr << "rbd: failed to process journal: " << cpp_strerror(r)
+		<< std::endl;
+      if (m_r == 0) {
+       m_r = r;
+      }
+    }
+    return m_r;
+  }
+
+  int shut_down() {
+    return m_journaler.shut_down();
+  }
+
+protected:
+  struct ReplayHandler : public ::journal::ReplayHandler {
+    JournalPlayer *journal;
+    explicit ReplayHandler(JournalPlayer *_journal) : journal(_journal) {}
+
+    void get() override {}
+    void put() override {}
+
+    void handle_entries_available() override {
+      journal->handle_replay_ready();
+    }
+    void handle_complete(int r) override {
+      journal->handle_replay_complete(r);
+    }
+  };
+
+  void handle_replay_ready() {
+    int r = 0;
+    while (true) {
+      ::journal::ReplayEntry replay_entry;
+      uint64_t tag_id;
+      if (!m_journaler.try_pop_front(&replay_entry, &tag_id)) {
+	break;
+      }
+
+      r = process_entry(replay_entry, tag_id);
+      if (r < 0) {
+	break;
+      }
+    }
+  }
+
+  virtual int process_entry(::journal::ReplayEntry replay_entry,
+			    uint64_t tag_id) = 0;
+
+  void handle_replay_complete(int r) {
+    if (m_r == 0 && r < 0) {
+      m_r = r;
+    }
+    m_journaler.stop_replay(&m_cond);
+  }
+
+  Journaler m_journaler;
+  C_SaferCond m_cond;
+  int m_r;
+};
+
+static int inspect_entry(bufferlist& data,
+			 librbd::journal::EventEntry& event_entry,
+			 bool verbose) {
+  try {
+    auto it = data.cbegin();
+    decode(event_entry, it);
+  } catch (const buffer::error &err) {
+    std::cerr << "failed to decode event entry: " << err.what() << std::endl;
+    return -EINVAL;
+  }
+  if (verbose) {
+    JSONFormatter f(true);
+    f.open_object_section("event_entry");
+    event_entry.dump(&f);
+    f.close_section();
+    f.flush(std::cout);
+  }
+  return 0;
+}
+
+class JournalInspector : public JournalPlayer {
+public:
+  JournalInspector(librados::IoCtx& io_ctx, const std::string& journal_id,
+		   bool verbose) :
+    JournalPlayer(io_ctx, journal_id, "INSPECT"),
+    m_verbose(verbose),
+    m_s() {
+  }
+
+  int exec() override {
+    int r = JournalPlayer::exec();
+    m_s.print();
+    return r;
+  }
+
+private:
+  struct Stats {
+    Stats() : total(0), error(0) {}
+
+    void print() {
+      std::cout << "Summary:" << std::endl
+		<< "  " << total << " entries inspected, " << error << " errors"
+		<< std::endl;
+    }
+
+    int total;
+    int error;
+  };
+
+  int process_entry(::journal::ReplayEntry replay_entry,
+		    uint64_t tag_id) override {
+    m_s.total++;
+    if (m_verbose) {
+      std::cout << "Entry: tag_id=" << tag_id << ", commit_tid="
+		<< replay_entry.get_commit_tid() << std::endl;
+    }
+    bufferlist data = replay_entry.get_data();
+    librbd::journal::EventEntry event_entry;
+    int r = inspect_entry(data, event_entry, m_verbose);
+    if (r < 0) {
+      m_r = r;
+      m_s.error++;
+    }
+    return 0;
+  }
+
+  bool m_verbose;
+  Stats m_s;
+};
+
+static int do_inspect_journal(librados::IoCtx& io_ctx,
+			      const std::string& journal_id,
+			      bool verbose) {
+  JournalInspector inspector(io_ctx, journal_id, verbose);
+  int r = inspector.exec();
+  if (r < 0) {
+    inspector.shut_down();
+    return r;
+  }
+
+  r = inspector.shut_down();
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
+struct ExportEntry {
+  uint64_t tag_id;
+  uint64_t commit_tid;
+  int type;
+  bufferlist entry;
+
+  ExportEntry() : tag_id(0), commit_tid(0), type(0), entry() {}
+
+  ExportEntry(uint64_t tag_id, uint64_t commit_tid, int type,
+	      const bufferlist& entry)
+    : tag_id(tag_id), commit_tid(commit_tid), type(type), entry(entry) {
+  }
+
+  void dump(Formatter *f) const {
+    ::encode_json("tag_id", tag_id, f);
+    ::encode_json("commit_tid", commit_tid, f);
+    ::encode_json("type", type, f);
+    ::encode_json("entry", entry, f);
+  }
+
+  void decode_json(JSONObj *obj) {
+    JSONDecoder::decode_json("tag_id", tag_id, obj);
+    JSONDecoder::decode_json("commit_tid", commit_tid, obj);
+    JSONDecoder::decode_json("type", type, obj);
+    JSONDecoder::decode_json("entry", entry, obj);
+  }
+};
+
+class JournalExporter : public JournalPlayer {
+public:
+  JournalExporter(librados::IoCtx& io_ctx, const std::string& journal_id,
+		  int fd, bool no_error, bool verbose) :
+    JournalPlayer(io_ctx, journal_id, "EXPORT"),
+    m_journal_id(journal_id),
+    m_fd(fd),
+    m_no_error(no_error),
+    m_verbose(verbose),
+    m_s() {
+  }
+
+  int exec() override {
+    std::string header("# journal_id: " + m_journal_id + "\n");
+    int r;
+    r = safe_write(m_fd, header.c_str(), header.size());
+    if (r < 0) {
+      std::cerr << "rbd: failed to write to export file: " << cpp_strerror(r)
+		<< std::endl;
+      return r;
+    }
+    r = JournalPlayer::exec();
+    m_s.print();
+    return r;
+  }
+
+private:
+  struct Stats {
+    Stats() : total(0), error(0) {}
+
+    void print() {
+      std::cout << total << " entries processed, " << error << " errors"
+		<< std::endl;
+    }
+
+    int total;
+    int error;
+  };
+
+  int process_entry(::journal::ReplayEntry replay_entry,
+		    uint64_t tag_id) override {
+    m_s.total++;
+    int type = -1;
+    bufferlist entry = replay_entry.get_data();
+    librbd::journal::EventEntry event_entry;
+    int r = inspect_entry(entry, event_entry, m_verbose);
+    if (r < 0) {
+      m_s.error++;
+      m_r = r;
+      return m_no_error ? 0 : r;
+    } else {
+      type = event_entry.get_event_type();
+    }
+    ExportEntry export_entry(tag_id, replay_entry.get_commit_tid(), type,
+                             entry);
+    JSONFormatter f;
+    ::encode_json("event_entry", export_entry, &f);
+    std::ostringstream oss;
+    f.flush(oss);
+    std::string objstr = oss.str();
+    std::string header = stringify(objstr.size()) + " ";
+    r = safe_write(m_fd, header.c_str(), header.size());
+    if (r == 0) {
+      r = safe_write(m_fd, objstr.c_str(), objstr.size());
+    }
+    if (r == 0) {
+      r = safe_write(m_fd, "\n", 1);
+    }
+    if (r < 0) {
+      std::cerr << "rbd: failed to write to export file: " << cpp_strerror(r)
+		<< std::endl;
+      m_s.error++;
+      return r;
+    }
+    return 0;
+  }
+
+  std::string m_journal_id;
+  int m_fd;
+  bool m_no_error;
+  bool m_verbose;
+  Stats m_s;
+};
+
+static int do_export_journal(librados::IoCtx& io_ctx,
+			     const std::string& journal_id,
+			     const std::string& path,
+			     bool no_error, bool verbose) {
+  int r;
+  int fd;
+  bool to_stdout = path == "-";
+  if (to_stdout) {
+    fd = STDOUT_FILENO;
+  } else {
+    fd = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0644);
+    if (fd < 0) {
+      r = -errno;
+      std::cerr << "rbd: error creating " << path << std::endl;
+      return r;
+    }
+#ifdef HAVE_POSIX_FADVISE
+    posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+#endif
+  }
+
+  JournalExporter exporter(io_ctx, journal_id, fd, no_error, verbose);
+  r = exporter.exec();
+
+  if (!to_stdout) {
+    close(fd);
+  }
+
+  int shut_down_r = exporter.shut_down();
+  if (r == 0 && shut_down_r < 0) {
+    r = shut_down_r;
+  }
+
+  return r;
+}
+
+class JournalImporter {
+public:
+  JournalImporter(librados::IoCtx& io_ctx, const std::string& journal_id,
+		  int fd, bool no_error, bool verbose) :
+    m_journaler(io_ctx, journal_id, "IMPORT"),
+    m_fd(fd),
+    m_no_error(no_error),
+    m_verbose(verbose) {
+  }
+
+  bool read_entry(bufferlist& bl, int& r) {
+    // Entries are stored in the file using the following format:
+    //
+    //   # Optional comments
+    //   NNN {json encoded entry}
+    //   ...
+    //
+    // Where NNN is the encoded entry size.
+    bl.clear();
+    char buf[80];
+    // Skip line feed and comments (lines started with #).
+    while ((r = safe_read_exact(m_fd, buf, 1)) == 0) {
+      if (buf[0] == '\n') {
+	continue;
+      } else if (buf[0] == '#') {
+	while ((r = safe_read_exact(m_fd, buf, 1)) == 0) {
+	  if (buf[0] == '\n') {
+	    break;
+	  }
+	}
+      } else {
+	break;
+      }
+    }
+    if (r < 0) {
+      if (r == -EDOM) {
+	r = 0;
+      }
+      return false;
+    }
+    // Read entry size to buf.
+    if (!isdigit(buf[0])) {
+      r = -EINVAL;
+      std::cerr << "rbd: import data invalid format (digit expected)"
+		<< std::endl;
+      return false;
+    }
+    for (size_t i = 1; i < sizeof(buf); i++) {
+      r = safe_read_exact(m_fd, buf + i, 1);
+      if (r < 0) {
+	std::cerr << "rbd: error reading import data" << std::endl;
+	return false;
+      }
+      if (!isdigit(buf[i])) {
+	if (buf[i] != ' ') {
+	  r = -EINVAL;
+	  std::cerr << "rbd: import data invalid format (space expected)"
+		    << std::endl;
+	  return false;
+	}
+	buf[i] = '\0';
+	break;
+      }
+    }
+    int entry_size = atoi(buf);
+    if (entry_size == 0) {
+      r = -EINVAL;
+      std::cerr << "rbd: import data invalid format (zero entry size)"
+		<< std::endl;
+      return false;
+    }
+    ceph_assert(entry_size > 0);
+    // Read entry.
+    r = bl.read_fd(m_fd, entry_size);
+    if (r < 0) {
+      std::cerr << "rbd: error reading from stdin: " << cpp_strerror(r)
+		<< std::endl;
+      return false;
+    }
+    if (r != entry_size) {
+      std::cerr << "rbd: error reading from stdin: truncated"
+		<< std::endl;
+      r = -EINVAL;
+      return false;
+    }
+    r = 0;
+    return true;
+  }
+
+  int exec() {
+    int r = m_journaler.init();
+    if (r < 0) {
+      return r;
+    }
+    m_journaler.start_append(0);
+
+    int r1 = 0;
+    bufferlist bl;
+    int n = 0;
+    int error_count = 0;
+    while (read_entry(bl, r)) {
+      n++;
+      error_count++;
+      JSONParser p;
+      if (!p.parse(bl.c_str(), bl.length())) {
+	std::cerr << "rbd: error parsing input (entry " << n << ")"
+		  << std::endl;
+	r = -EINVAL;
+	if (m_no_error) {
+	  r1 = r;
+	  continue;
+	} else {
+	  break;
+	}
+      }
+      ExportEntry e;
+      try {
+	decode_json_obj(e, &p);
+      } catch (JSONDecoder::err& err) {
+	std::cerr << "rbd: error json decoding import data (entry " << n << "):"
+		  << err.message << std::endl;
+	r = -EINVAL;
+	if (m_no_error) {
+	  r1 = r;
+	  continue;
+	} else {
+	  break;
+	}
+      }
+      librbd::journal::EventEntry event_entry;
+      r = inspect_entry(e.entry, event_entry, m_verbose);
+      if (r < 0) {
+	std::cerr << "rbd: corrupted entry " << n << ": tag_tid=" << e.tag_id
+		  << ", commit_tid=" << e.commit_tid << std::endl;
+	if (m_no_error) {
+	  r1 = r;
+	  continue;
+	} else {
+	  break;
+	}
+      }
+      m_journaler.append(e.tag_id, e.entry);
+      error_count--;
+    }
+
+    std::cout << n << " entries processed, " << error_count << " errors"  << std::endl;
+
+    std::cout << "Waiting for journal append to complete..."  << std::endl;
+
+    C_SaferCond cond;
+    m_journaler.stop_append(&cond);
+    r = cond.wait();
+
+    if (r < 0) {
+      std::cerr << "failed to append journal: " << cpp_strerror(r) << std::endl;
+    }
+
+    if (r1 < 0 && r == 0) {
+      r = r1;
+    }
+    return r;
+  }
+
+  int shut_down() {
+    return m_journaler.shut_down();
+  }
+
+private:
+  Journaler m_journaler;
+  int m_fd;
+  bool m_no_error;
+  bool m_verbose;
+};
+
+static int do_import_journal(librados::IoCtx& io_ctx,
+			     const std::string& journal_id,
+			     const std::string& path,
+			     bool no_error, bool verbose) {
+  int r;
+
+  int fd;
+  bool from_stdin = path == "-";
+  if (from_stdin) {
+    fd = STDIN_FILENO;
+  } else {
+    if ((fd = open(path.c_str(), O_RDONLY)) < 0) {
+      r = -errno;
+      std::cerr << "rbd: error opening " << path << std::endl;
+      return r;
+    }
+#ifdef HAVE_POSIX_FADVISE
+    posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+#endif
+  }
+
+  JournalImporter importer(io_ctx, journal_id, fd, no_error, verbose);
+  r = importer.exec();
+
+  if (!from_stdin) {
+    close(fd);
+  }
+
+  int shut_down_r = importer.shut_down();
+  if (r == 0 && shut_down_r < 0) {
+    r = shut_down_r;
+  }
+
+  return r;
+}
+
+void get_info_arguments(po::options_description *positional,
+			po::options_description *options) {
+  add_journal_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_format_options(options);
+}
+
+int execute_info(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string journal_name;
+  int r = get_pool_journal_names(vm, at::ARGUMENT_MODIFIER_NONE, &arg_index,
+                                 &pool_name, &namespace_name, &journal_name);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_show_journal_info(rados, io_ctx, journal_name, formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: journal info: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+
+}
+
+void get_status_arguments(po::options_description *positional,
+			  po::options_description *options) {
+  add_journal_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_format_options(options);
+}
+
+int execute_status(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string journal_name;
+  int r = get_pool_journal_names(vm, at::ARGUMENT_MODIFIER_NONE, &arg_index,
+                                 &pool_name, &namespace_name, &journal_name);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_show_journal_status(io_ctx, journal_name, formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: journal status: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_reset_arguments(po::options_description *positional,
+			 po::options_description *options) {
+  add_journal_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+}
+
+int execute_reset(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string journal_name;
+  int r = get_pool_journal_names(vm, at::ARGUMENT_MODIFIER_NONE, &arg_index,
+                                 &pool_name, &namespace_name, &journal_name);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_reset_journal(io_ctx, journal_name);
+  if (r < 0) {
+    std::cerr << "rbd: journal reset: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_client_disconnect_arguments(po::options_description *positional,
+				     po::options_description *options) {
+  add_journal_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  options->add_options()
+    ("client-id", po::value<std::string>(),
+     "client ID (or leave unspecified to disconnect all)");
+}
+
+int execute_client_disconnect(const po::variables_map &vm,
+                              const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string journal_name;
+  int r = get_pool_journal_names(vm, at::ARGUMENT_MODIFIER_NONE, &arg_index,
+                                 &pool_name, &namespace_name, &journal_name);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string client_id;
+  if (vm.count("client-id")) {
+    client_id = vm["client-id"].as<std::string>();
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_disconnect_journal_client(io_ctx, journal_name, client_id);
+  if (r < 0) {
+    std::cerr << "rbd: journal client disconnect: " << cpp_strerror(r)
+	      << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_inspect_arguments(po::options_description *positional,
+			   po::options_description *options) {
+  add_journal_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_verbose_option(options);
+}
+
+int execute_inspect(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string journal_name;
+  int r = get_pool_journal_names(vm, at::ARGUMENT_MODIFIER_NONE, &arg_index,
+                                 &pool_name, &namespace_name, &journal_name);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_inspect_journal(io_ctx, journal_name, vm[at::VERBOSE].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: journal inspect: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_export_arguments(po::options_description *positional,
+			  po::options_description *options) {
+  add_journal_spec_options(positional, options,
+			       at::ARGUMENT_MODIFIER_SOURCE);
+  at::add_path_options(positional, options,
+                       "export file (or '-' for stdout)");
+  at::add_verbose_option(options);
+  at::add_no_error_option(options);
+}
+
+int execute_export(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string journal_name;
+  int r = get_pool_journal_names(vm, at::ARGUMENT_MODIFIER_SOURCE, &arg_index,
+                                 &pool_name, &namespace_name, &journal_name);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string path;
+  r = utils::get_path(vm, &arg_index, &path);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_export_journal(io_ctx, journal_name, path, vm[at::NO_ERROR].as<bool>(),
+			vm[at::VERBOSE].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: journal export: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_import_arguments(po::options_description *positional,
+			  po::options_description *options) {
+  at::add_path_options(positional, options,
+                       "import file (or '-' for stdin)");
+  add_journal_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
+  at::add_verbose_option(options);
+  at::add_no_error_option(options);
+}
+
+int execute_import(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  std::string path;
+  size_t arg_index = 0;
+  int r = utils::get_path(vm, &arg_index, &path);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string pool_name;
+  std::string namespace_name;
+  std::string journal_name;
+  r = get_pool_journal_names(vm, at::ARGUMENT_MODIFIER_DEST, &arg_index,
+                             &pool_name, &namespace_name, &journal_name);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_import_journal(io_ctx, journal_name, path, vm[at::NO_ERROR].as<bool>(),
+			vm[at::VERBOSE].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: journal import: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action_info(
+  {"journal", "info"}, {}, "Show information about image journal.", "",
+  &get_info_arguments, &execute_info);
+
+Shell::Action action_status(
+  {"journal", "status"}, {}, "Show status of image journal.", "",
+  &get_status_arguments, &execute_status);
+
+Shell::Action action_reset(
+  {"journal", "reset"}, {}, "Reset image journal.", "",
+  &get_reset_arguments, &execute_reset);
+
+Shell::Action action_inspect(
+  {"journal", "inspect"}, {}, "Inspect image journal for structural errors.", "",
+  &get_inspect_arguments, &execute_inspect);
+
+Shell::Action action_export(
+  {"journal", "export"}, {}, "Export image journal.", "",
+  &get_export_arguments, &execute_export);
+
+Shell::Action action_import(
+  {"journal", "import"}, {}, "Import image journal.", "",
+  &get_import_arguments, &execute_import);
+
+Shell::Action action_disconnect(
+  {"journal", "client", "disconnect"}, {},
+  "Flag image journal client as disconnected.", "",
+  &get_client_disconnect_arguments, &execute_client_disconnect);
+
+} // namespace journal
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Kernel.cc b/src/tools/rbd/action/Kernel.cc
new file mode 100644
index 00000000..dc0938eb
--- /dev/null
+++ b/src/tools/rbd/action/Kernel.cc
@@ -0,0 +1,561 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "acconfig.h"
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/krbd.h"
+#include "include/stringify.h"
+#include "include/uuid.h"
+#include "common/config_proxy.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "common/strtol.h"
+#include "common/Formatter.h"
+#include "msg/msg_types.h"
+#include "global/global_context.h"
+#include <iostream>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/scope_exit.hpp>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace kernel {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+namespace {
+
+std::map<std::string, std::string> map_options; // used for both map and unmap
+
+} // anonymous namespace
+
+static std::string map_option_uuid_cb(const char *value_char)
+{
+  uuid_d u;
+  if (!u.parse(value_char))
+    return "";
+
+  return stringify(u);
+}
+
+static std::string map_option_ip_cb(const char *value_char)
+{
+  entity_addr_t a;
+  const char *endptr;
+  if (!a.parse(value_char, &endptr) ||
+      endptr != value_char + strlen(value_char)) {
+    return "";
+  }
+
+  return stringify(a.get_sockaddr());
+}
+
+static std::string map_option_int_cb(const char *value_char)
+{
+  std::string err;
+  int d = strict_strtol(value_char, 10, &err);
+  if (!err.empty() || d < 0)
+    return "";
+
+  return stringify(d);
+}
+
+static std::string map_option_ms_mode_cb(const char *value_char)
+{
+  if (!strcmp(value_char, "legacy") || !strcmp(value_char, "crc") ||
+      !strcmp(value_char, "secure") || !strcmp(value_char, "prefer-crc") ||
+      !strcmp(value_char, "prefer-secure")) {
+    return value_char;
+  }
+  return "";
+}
+
+static void put_map_option(const std::string &key, const std::string &val)
+{
+  map_options[key] = val;
+}
+
+static int put_map_option_value(const std::string &opt, const char *value_char,
+                                std::string (*parse_cb)(const char *))
+{
+  if (!value_char || *value_char == '\0') {
+    std::cerr << "rbd: " << opt << " option requires a value" << std::endl;
+    return -EINVAL;
+  }
+
+  std::string value = parse_cb(value_char);
+  if (value.empty()) {
+    std::cerr << "rbd: invalid " << opt << " value '" << value_char << "'"
+              << std::endl;
+    return -EINVAL;
+  }
+
+  put_map_option(opt, opt + "=" + value);
+  return 0;
+}
+
+static int parse_map_options(const std::string &options_string)
+{
+  char *options = strdup(options_string.c_str());
+  BOOST_SCOPE_EXIT(options) {
+    free(options);
+  } BOOST_SCOPE_EXIT_END;
+
+  for (char *this_char = strtok(options, ", ");
+       this_char != NULL;
+       this_char = strtok(NULL, ",")) {
+    char *value_char;
+
+    if ((value_char = strchr(this_char, '=')) != NULL)
+      *value_char++ = '\0';
+
+    if (!strcmp(this_char, "fsid")) {
+      if (put_map_option_value("fsid", value_char, map_option_uuid_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "ip")) {
+      if (put_map_option_value("ip", value_char, map_option_ip_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "share") || !strcmp(this_char, "noshare")) {
+      put_map_option("share", this_char);
+    } else if (!strcmp(this_char, "crc") || !strcmp(this_char, "nocrc")) {
+      put_map_option("crc", this_char);
+    } else if (!strcmp(this_char, "cephx_require_signatures") ||
+               !strcmp(this_char, "nocephx_require_signatures")) {
+      put_map_option("cephx_require_signatures", this_char);
+    } else if (!strcmp(this_char, "tcp_nodelay") ||
+               !strcmp(this_char, "notcp_nodelay")) {
+      put_map_option("tcp_nodelay", this_char);
+    } else if (!strcmp(this_char, "cephx_sign_messages") ||
+               !strcmp(this_char, "nocephx_sign_messages")) {
+      put_map_option("cephx_sign_messages", this_char);
+    } else if (!strcmp(this_char, "mount_timeout")) {
+      if (put_map_option_value("mount_timeout", value_char, map_option_int_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "osd_request_timeout")) {
+      if (put_map_option_value("osd_request_timeout", value_char, map_option_int_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "lock_timeout")) {
+      if (put_map_option_value("lock_timeout", value_char, map_option_int_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "osdkeepalive")) {
+      if (put_map_option_value("osdkeepalive", value_char, map_option_int_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "osd_idle_ttl")) {
+      if (put_map_option_value("osd_idle_ttl", value_char, map_option_int_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "rw") || !strcmp(this_char, "ro")) {
+      put_map_option("rw", this_char);
+    } else if (!strcmp(this_char, "queue_depth")) {
+      if (put_map_option_value("queue_depth", value_char, map_option_int_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "lock_on_read")) {
+      put_map_option("lock_on_read", this_char);
+    } else if (!strcmp(this_char, "exclusive")) {
+      put_map_option("exclusive", this_char);
+    } else if (!strcmp(this_char, "notrim")) {
+      put_map_option("notrim", this_char);
+    } else if (!strcmp(this_char, "abort_on_full")) {
+      put_map_option("abort_on_full", this_char);
+    } else if (!strcmp(this_char, "alloc_size")) {
+      if (put_map_option_value("alloc_size", value_char, map_option_int_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "ms_mode")) {
+      if (put_map_option_value("ms_mode", value_char, map_option_ms_mode_cb))
+        return -EINVAL;
+    } else if (!strcmp(this_char, "udev") || !strcmp(this_char, "noudev")) {
+      put_map_option("udev", this_char);
+    } else {
+      std::cerr << "rbd: unknown map option '" << this_char << "'" << std::endl;
+      return -EINVAL;
+    }
+  }
+
+  return 0;
+}
+
+static int parse_unmap_options(const std::string &options_string)
+{
+  char *options = strdup(options_string.c_str());
+  BOOST_SCOPE_EXIT(options) {
+    free(options);
+  } BOOST_SCOPE_EXIT_END;
+
+  for (char *this_char = strtok(options, ", ");
+       this_char != NULL;
+       this_char = strtok(NULL, ",")) {
+    char *value_char;
+
+    if ((value_char = strchr(this_char, '=')) != NULL)
+      *value_char++ = '\0';
+
+    if (!strcmp(this_char, "force")) {
+      put_map_option("force", this_char);
+    } else if (!strcmp(this_char, "udev") || !strcmp(this_char, "noudev")) {
+      put_map_option("udev", this_char);
+    } else {
+      std::cerr << "rbd: unknown unmap option '" << this_char << "'" << std::endl;
+      return -EINVAL;
+    }
+  }
+
+  return 0;
+}
+
+static int do_kernel_list(Formatter *f) {
+#if defined(WITH_KRBD)
+  struct krbd_ctx *krbd;
+  int r;
+
+  r = krbd_create_from_context(g_ceph_context, 0, &krbd);
+  if (r < 0)
+    return r;
+
+  r = krbd_showmapped(krbd, f);
+
+  krbd_destroy(krbd);
+  return r;
+#else
+  std::cerr << "rbd: kernel device is not supported" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+}
+
+static int get_unsupported_features(librbd::Image &image,
+                                    uint64_t *unsupported_features)
+{
+  char buf[20];
+  uint64_t features, supported_features;
+  int r;
+
+  r = safe_read_file("/sys/bus/rbd/", "supported_features", buf,
+                     sizeof(buf) - 1);
+  if (r < 0)
+    return r;
+
+  buf[r] = '\0';
+  try {
+    supported_features = std::stoull(buf, nullptr, 16);
+  } catch (...) {
+    return -EINVAL;
+  }
+
+  r = image.features(&features);
+  if (r < 0)
+    return r;
+
+  *unsupported_features = features & ~supported_features;
+  return 0;
+}
+
+/*
+ * hint user to check syslog for krbd related messages and provide suggestions
+ * based on errno return by krbd_map(). also note that even if some librbd calls
+ * fail, we at least dump the "try dmesg..." message to aid debugging.
+ */
+static void print_error_description(const char *poolname,
+                                    const char *nspace_name,
+                                    const char *imgname,
+                                    const char *snapname,
+                                    int maperrno)
+{
+  int r;
+  uint8_t oldformat;
+  librados::Rados rados;
+  librados::IoCtx ioctx;
+  librbd::Image image;
+
+  if (maperrno == -ENOENT)
+    goto done;
+
+  r = utils::init_and_open_image(poolname, nspace_name, imgname, "", snapname,
+				 true, &rados, &ioctx, &image);
+  if (r < 0)
+    goto done;
+
+  r = image.old_format(&oldformat);
+  if (r < 0)
+    goto done;
+
+  /*
+   * kernel returns -ENXIO when mapping a V2 image due to unsupported feature
+   * set - so, hint about that too...
+   */
+  if (!oldformat && (maperrno == -ENXIO)) {
+    uint64_t unsupported_features;
+    bool need_terminate = true;
+
+    std::cout << "RBD image feature set mismatch. ";
+    r = get_unsupported_features(image, &unsupported_features);
+    if (r == 0 && (unsupported_features & ~RBD_FEATURES_ALL) == 0) {
+      uint64_t immutable = RBD_FEATURES_ALL & ~(RBD_FEATURES_MUTABLE |
+                                                RBD_FEATURES_DISABLE_ONLY);
+      if (unsupported_features & immutable) {
+        std::cout << "This image cannot be mapped because the following "
+                  << "immutable features are unsupported by the kernel:";
+        unsupported_features &= immutable;
+        need_terminate = false;
+      } else {
+        std::cout << "You can disable features unsupported by the kernel "
+                  << "with \"rbd feature disable ";
+        if (poolname != utils::get_default_pool_name() || *nspace_name) {
+          std::cout << poolname << "/";
+        }
+        if (*nspace_name) {
+          std::cout << nspace_name << "/";
+        }
+        std::cout << imgname;
+      }
+    } else {
+      std::cout << "Try disabling features unsupported by the kernel "
+                << "with \"rbd feature disable";
+      unsupported_features = 0;
+    }
+    for (auto it : at::ImageFeatures::FEATURE_MAPPING) {
+      if (it.first & unsupported_features) {
+        std::cout << " " << it.second;
+      }
+    }
+    if (need_terminate)
+      std::cout << "\"";
+    std::cout << "." << std::endl;
+  }
+
+ done:
+  std::cout << "In some cases useful info is found in syslog - try \"dmesg | tail\"." << std::endl;
+}
+
+static int do_kernel_map(const char *poolname, const char *nspace_name,
+                         const char *imgname, const char *snapname)
+{
+#if defined(WITH_KRBD)
+  struct krbd_ctx *krbd;
+  std::ostringstream oss;
+  uint32_t flags = 0;
+  char *devnode;
+  int r;
+
+  for (auto it = map_options.begin(); it != map_options.end(); ) {
+    // for compatibility with < 3.7 kernels, assume that rw is on by
+    // default and omit it even if it was specified by the user
+    // (see ceph.git commit fb0f1986449b)
+    if (it->first == "rw" && it->second == "rw") {
+      it = map_options.erase(it);
+    } else if (it->first == "udev") {
+      if (it->second == "noudev") {
+        flags |= KRBD_CTX_F_NOUDEV;
+      }
+      it = map_options.erase(it);
+    } else {
+      if (it != map_options.begin())
+        oss << ",";
+      oss << it->second;
+      ++it;
+    }
+  }
+
+  r = krbd_create_from_context(g_ceph_context, flags, &krbd);
+  if (r < 0)
+    return r;
+
+  r = krbd_is_mapped(krbd, poolname, nspace_name, imgname, snapname, &devnode);
+  if (r < 0) {
+    std::cerr << "rbd: warning: can't get image map information: "
+	      << cpp_strerror(r) << std::endl;
+  } else if (r > 0) {
+    std::cerr << "rbd: warning: image already mapped as " << devnode
+              << std::endl;
+    free(devnode);
+  }
+
+  r = krbd_map(krbd, poolname, nspace_name, imgname, snapname,
+               oss.str().c_str(), &devnode);
+  if (r < 0) {
+    print_error_description(poolname, nspace_name, imgname, snapname, r);
+    goto out;
+  }
+
+  std::cout << devnode << std::endl;
+
+  free(devnode);
+out:
+  krbd_destroy(krbd);
+  return r;
+#else
+  std::cerr << "rbd: kernel device is not supported" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+}
+
+static int do_kernel_unmap(const char *dev, const char *poolname,
+                           const char *nspace_name, const char *imgname,
+                           const char *snapname)
+{
+#if defined(WITH_KRBD)
+  struct krbd_ctx *krbd;
+  std::ostringstream oss;
+  uint32_t flags = 0;
+  int r;
+
+  for (auto it = map_options.begin(); it != map_options.end(); ) {
+    if (it->first == "udev") {
+      if (it->second == "noudev") {
+        flags |= KRBD_CTX_F_NOUDEV;
+      }
+      it = map_options.erase(it);
+    } else {
+      if (it != map_options.begin())
+        oss << ",";
+      oss << it->second;
+      ++it;
+    }
+  }
+
+  r = krbd_create_from_context(g_ceph_context, flags, &krbd);
+  if (r < 0)
+    return r;
+
+  if (dev)
+    r = krbd_unmap(krbd, dev, oss.str().c_str());
+  else
+    r = krbd_unmap_by_spec(krbd, poolname, nspace_name, imgname, snapname,
+                           oss.str().c_str());
+
+  krbd_destroy(krbd);
+  return r;
+#else
+  std::cerr << "rbd: kernel device is not supported" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  at::Format::Formatter formatter;
+  int r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  utils::init_context();
+
+  r = do_kernel_list(formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: device list failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+int execute_map(const po::variables_map &vm,
+                const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string nspace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &nspace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  // parse default options first so they can be overwritten by cli options
+  r = parse_map_options(
+      g_conf().get_val<std::string>("rbd_default_map_options"));
+  if (r < 0) {
+    std::cerr << "rbd: couldn't parse default map options" << std::endl;
+    return r;
+  }
+
+  if (vm.count("options")) {
+    for (auto &options : vm["options"].as<std::vector<std::string>>()) {
+      r = parse_map_options(options);
+      if (r < 0) {
+        std::cerr << "rbd: couldn't parse map options" << std::endl;
+        return r;
+      }
+    }
+  }
+
+  // parse options common to all device types after parsing krbd-specific
+  // options so that common options win (in particular "-o rw --read-only"
+  // should result in read-only mapping)
+  if (vm["read-only"].as<bool>()) {
+    put_map_option("rw", "ro");
+  }
+  if (vm["exclusive"].as<bool>()) {
+    put_map_option("exclusive", "exclusive");
+  }
+
+  utils::init_context();
+
+  r = do_kernel_map(pool_name.c_str(), nspace_name.c_str(), image_name.c_str(),
+                    snap_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: map failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+int execute_unmap(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+  std::string device_name = utils::get_positional_argument(vm, 0);
+  if (!boost::starts_with(device_name, "/dev/")) {
+    device_name.clear();
+  }
+
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string nspace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r;
+  if (device_name.empty()) {
+    r = utils::get_pool_image_snapshot_names(
+      vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &nspace_name,
+      &image_name, &snap_name, false, utils::SNAPSHOT_PRESENCE_PERMITTED,
+      utils::SPEC_VALIDATION_NONE);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  if (device_name.empty() && image_name.empty()) {
+    std::cerr << "rbd: unmap requires either image name or device path"
+              << std::endl;
+    return -EINVAL;
+  }
+
+  if (vm.count("options")) {
+    for (auto &options : vm["options"].as<std::vector<std::string>>()) {
+      r = parse_unmap_options(options);
+      if (r < 0) {
+        std::cerr << "rbd: couldn't parse unmap options" << std::endl;
+        return r;
+      }
+    }
+  }
+
+  utils::init_context();
+
+  r = do_kernel_unmap(device_name.empty() ? nullptr : device_name.c_str(),
+                      pool_name.c_str(), nspace_name.c_str(),
+                      image_name.c_str(), snap_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: unmap failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+} // namespace kernel
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/List.cc b/src/tools/rbd/action/List.cc
new file mode 100644
index 00000000..e6025418
--- /dev/null
+++ b/src/tools/rbd/action/List.cc
@@ -0,0 +1,340 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/Context.h"
+#include "include/stringify.h"
+#include "include/types.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include <iostream>
+#include <boost/bind.hpp>
+#include <boost/program_options.hpp>
+#include "global/global_context.h"
+
+namespace rbd {
+
+namespace action {
+namespace list {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+enum WorkerState {
+  STATE_IDLE = 0,
+  STATE_OPENED,
+  STATE_DONE
+} ;
+
+struct WorkerEntry {
+  librbd::Image img;
+  librbd::RBD::AioCompletion* completion;
+  WorkerState state;
+  string name;
+
+  WorkerEntry() {
+    state = STATE_IDLE;
+    completion = nullptr;
+  }
+};
+
+
+int list_process_image(librados::Rados* rados, WorkerEntry* w, bool lflag, Formatter *f, TextTable &tbl)
+{
+  int r = 0;
+  librbd::image_info_t info;
+  std::string parent;
+
+  // handle second-nth trips through loop
+  librbd::linked_image_spec_t parent_image_spec;
+  librbd::snap_spec_t parent_snap_spec;
+  r = w->img.get_parent(&parent_image_spec, &parent_snap_spec);
+  if (r < 0 && r != -ENOENT) {
+    return r;
+  }
+
+  bool has_parent = false;
+  if (r != -ENOENT) {
+    parent = parent_image_spec.pool_name + "/";
+    if (!parent_image_spec.pool_namespace.empty()) {
+      parent += parent_image_spec.pool_namespace + "/";
+    }
+    parent +=  parent_image_spec.image_name + "@" + parent_snap_spec.name;
+    has_parent = true;
+  }
+
+  if (w->img.stat(info, sizeof(info)) < 0) {
+    return -EINVAL;
+  }
+
+  uint8_t old_format;
+  w->img.old_format(&old_format);
+
+  std::list<librbd::locker_t> lockers;
+  bool exclusive;
+  r = w->img.list_lockers(&lockers, &exclusive, NULL);
+  if (r < 0)
+    return r;
+  std::string lockstr;
+  if (!lockers.empty()) {
+    lockstr = (exclusive) ? "excl" : "shr";
+  }
+
+  if (f) {
+    f->open_object_section("image");
+    f->dump_string("image", w->name);
+    f->dump_unsigned("size", info.size);
+    if (has_parent) {
+      f->open_object_section("parent");
+      f->dump_string("pool", parent_image_spec.pool_name);
+      f->dump_string("pool_namespace", parent_image_spec.pool_namespace);
+      f->dump_string("image", parent_image_spec.image_name);
+      f->dump_string("snapshot", parent_snap_spec.name);
+      f->close_section();
+    }
+    f->dump_int("format", old_format ? 1 : 2);
+    if (!lockers.empty())
+      f->dump_string("lock_type", exclusive ? "exclusive" : "shared");
+    f->close_section();
+  } else {
+    tbl << w->name
+        << stringify(byte_u_t(info.size))
+        << parent
+        << ((old_format) ? '1' : '2')
+        << ""                         // protect doesn't apply to images
+        << lockstr
+        << TextTable::endrow;
+  }
+
+  std::vector<librbd::snap_info_t> snaplist;
+  if (w->img.snap_list(snaplist) >= 0 && !snaplist.empty()) {
+    snaplist.erase(remove_if(snaplist.begin(),
+                             snaplist.end(),
+                             boost::bind(utils::is_not_user_snap_namespace, &w->img, _1)),
+                   snaplist.end());
+    for (std::vector<librbd::snap_info_t>::iterator s = snaplist.begin();
+         s != snaplist.end(); ++s) {
+      bool is_protected;
+      bool has_parent = false;
+      parent.clear();
+      w->img.snap_set(s->name.c_str());
+      r = w->img.snap_is_protected(s->name.c_str(), &is_protected);
+      if (r < 0)
+        return r;
+      if (w->img.get_parent(&parent_image_spec, &parent_snap_spec) >= 0) {
+        parent = parent_image_spec.pool_name + "/";
+        if (!parent_image_spec.pool_namespace.empty()) {
+          parent += parent_image_spec.pool_namespace + "/";
+        }
+        parent +=  parent_image_spec.image_name + "@" + parent_snap_spec.name;
+        has_parent = true;
+      }
+      if (f) {
+        f->open_object_section("snapshot");
+        f->dump_string("image", w->name);
+        f->dump_string("snapshot", s->name);
+        f->dump_unsigned("size", s->size);
+        if (has_parent) {
+          f->open_object_section("parent");
+          f->dump_string("pool", parent_image_spec.pool_name);
+          f->dump_string("pool_namespace", parent_image_spec.pool_namespace);
+          f->dump_string("image", parent_image_spec.image_name);
+          f->dump_string("snapshot", parent_snap_spec.name);
+          f->close_section();
+        }
+        f->dump_int("format", old_format ? 1 : 2);
+        f->dump_string("protected", is_protected ? "true" : "false");
+        f->close_section();
+      } else {
+        tbl << w->name + "@" + s->name
+            << stringify(byte_u_t(s->size))
+            << parent
+            << ((old_format) ? '1' : '2')
+            << (is_protected ? "yes" : "")
+            << ""                     // locks don't apply to snaps
+            << TextTable::endrow;
+      }
+    }
+  }
+
+  return 0;
+}
+
+int do_list(const std::string &pool_name, const std::string& namespace_name,
+            bool lflag, int threads, Formatter *f) {
+  std::vector<WorkerEntry*> workers;
+  std::vector<librbd::image_spec_t> images;
+  librados::Rados rados;
+  librbd::RBD rbd;
+  librados::IoCtx ioctx;
+
+  if (threads < 1) {
+    threads = 1;
+  }
+  if (threads > 32) {
+    threads = 32;
+  }
+
+  int r = utils::init(pool_name, namespace_name, &rados, &ioctx);
+  if (r < 0) {
+    return r;
+  }
+
+  utils::disable_cache();
+
+  r = rbd.list2(ioctx, &images);
+  if (r < 0)
+    return r;
+
+  if (!lflag) {
+    if (f)
+      f->open_array_section("images");
+    for (auto& image : images) {
+       if (f)
+	 f->dump_string("name", image.name);
+       else
+	 std::cout << image.name << std::endl;
+    }
+    if (f) {
+      f->close_section();
+      f->flush(std::cout);
+    }
+    return 0;
+  }
+
+  TextTable tbl;
+
+  if (f) {
+    f->open_array_section("images");
+  } else {
+    tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("SIZE", TextTable::LEFT, TextTable::RIGHT);
+    tbl.define_column("PARENT", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("FMT", TextTable::LEFT, TextTable::RIGHT);
+    tbl.define_column("PROT", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("LOCK", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  for (size_t left = 0; left < std::min<size_t>(threads, images.size());
+       left++) {
+    workers.push_back(new WorkerEntry());
+  }
+
+  auto i = images.begin();
+  while (true) {
+    size_t workers_idle = 0;
+    for (auto comp : workers) {
+      switch (comp->state) {
+	case STATE_DONE:
+	  comp->completion->wait_for_complete();
+	  comp->state = STATE_IDLE;
+	  comp->completion->release();
+	  comp->completion = nullptr;
+	  // we want it to fall through in this case
+	case STATE_IDLE:
+	  if (i == images.end()) {
+	    workers_idle++;
+	    continue;
+	  }
+	  comp->name = i->name;
+	  comp->completion = new librbd::RBD::AioCompletion(nullptr, nullptr);
+	  r = rbd.aio_open_read_only(ioctx, comp->img, i->name.c_str(), nullptr,
+                                     comp->completion);
+	  i++;
+	  comp->state = STATE_OPENED;
+	  break;
+	case STATE_OPENED:
+	  comp->completion->wait_for_complete();
+	  // image might disappear between rbd.list() and rbd.open(); ignore
+	  // that, warn about other possible errors (EPERM, say, for opening
+	  // an old-format image, because you need execute permission for the
+	  // class method)
+	  r = comp->completion->get_return_value();
+	  comp->completion->release();
+	  if (r < 0) {
+	    std::cerr << "rbd: error opening " << comp->name << ": "
+                      << cpp_strerror(r) << std::endl;
+
+	    // in any event, continue to next image
+	    comp->state = STATE_IDLE;
+	    continue;
+	  }
+	  r = list_process_image(&rados, comp, lflag, f, tbl);
+	  if (r < 0) {
+	      std::cerr << "rbd: error processing image " << comp->name << ": "
+                        << cpp_strerror(r) << std::endl;
+	  }
+	  comp->completion = new librbd::RBD::AioCompletion(nullptr, nullptr);
+	  r = comp->img.aio_close(comp->completion);
+	  comp->state = STATE_DONE;
+	  break;
+      }
+    }
+    if (workers_idle == workers.size()) {
+	break;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  } else if (!images.empty()) {
+    std::cout << tbl;
+  }
+
+  rados.shutdown();
+
+  for (auto comp : workers) {
+    delete comp;
+  }
+
+  return r < 0 ? r : 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  options->add_options()
+    ("long,l", po::bool_switch(), "long listing format");
+  at::add_pool_options(positional, options, true);
+  at::add_format_options(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  std::string namespace_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, false, &pool_name,
+                                              &namespace_name, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_list(pool_name, namespace_name, vm["long"].as<bool>(),
+              g_conf().get_val<uint64_t>("rbd_concurrent_management_ops"),
+              formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: listing images failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+Shell::SwitchArguments switched_arguments({"long", "l"});
+Shell::Action action(
+  {"list"}, {"ls"}, "List rbd images.", "", &get_arguments, &execute);
+
+} // namespace list
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Lock.cc b/src/tools/rbd/action/Lock.cc
new file mode 100644
index 00000000..754cb384
--- /dev/null
+++ b/src/tools/rbd/action/Lock.cc
@@ -0,0 +1,279 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace lock {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+namespace {
+
+void add_id_option(po::options_description *positional) {
+  positional->add_options()
+    ("lock-id", "unique lock id");
+}
+
+int get_id(const po::variables_map &vm, size_t *arg_index,
+           std::string *id) {
+  *id = utils::get_positional_argument(vm, *arg_index);
+  if (id->empty()) {
+    std::cerr << "rbd: lock id was not specified" << std::endl;
+    return -EINVAL;
+  } else {
+    ++(*arg_index);
+  }
+  return 0;
+}
+
+} // anonymous namespace
+
+static int do_lock_list(librbd::Image& image, Formatter *f)
+{
+  std::list<librbd::locker_t> lockers;
+  bool exclusive;
+  std::string tag;
+  TextTable tbl;
+  int r;
+
+  r = image.list_lockers(&lockers, &exclusive, &tag);
+  if (r < 0)
+    return r;
+
+  if (f) {
+    f->open_array_section("locks");
+  } else {
+    tbl.define_column("Locker", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("ID", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("Address", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  if (lockers.size()) {
+    bool one = (lockers.size() == 1);
+
+    if (!f) {
+      std::cout << "There " << (one ? "is " : "are ") << lockers.size()
+           << (exclusive ? " exclusive" : " shared")
+           << " lock" << (one ? "" : "s") << " on this image.\n";
+      if (!exclusive)
+        std::cout << "Lock tag: " << tag << "\n";
+    }
+
+    for (std::list<librbd::locker_t>::const_iterator it = lockers.begin();
+         it != lockers.end(); ++it) {
+      if (f) {
+        f->open_object_section("lock");
+        f->dump_string("id", it->cookie);
+        f->dump_string("locker", it->client);
+        f->dump_string("address", it->address);
+        f->close_section();
+      } else {
+        tbl << it->client << it->cookie << it->address << TextTable::endrow;
+      }
+    }
+    if (!f)
+      std::cout << tbl;
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  }
+  return 0;
+}
+
+static int do_lock_add(librbd::Image& image, const char *cookie,
+                       const char *tag)
+{
+  if (tag)
+    return image.lock_shared(cookie, tag);
+  else
+    return image.lock_exclusive(cookie);
+}
+
+static int do_lock_remove(librbd::Image& image, const char *client,
+                          const char *cookie)
+{
+  return image.break_lock(client, cookie);
+}
+
+void get_list_arguments(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_format_options(options);
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_lock_list(image, formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: listing locks failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_add_arguments(po::options_description *positional,
+                       po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  add_id_option(positional);
+  options->add_options()
+    ("shared", po::value<std::string>(), "shared lock tag");
+}
+
+int execute_add(const po::variables_map &vm,
+                const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string lock_cookie;
+  r = get_id(vm, &arg_index, &lock_cookie);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string lock_tag;
+  if (vm.count("shared")) {
+    lock_tag = vm["shared"].as<std::string>();
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_lock_add(image, lock_cookie.c_str(),
+                  lock_tag.empty() ? nullptr : lock_tag.c_str());
+  if (r < 0) {
+    if (r == -EBUSY || r == -EEXIST) {
+      if (!lock_tag.empty()) {
+        std::cerr << "rbd: lock is already held by someone else"
+                  << " with a different tag" << std::endl;
+      } else {
+        std::cerr << "rbd: lock is already held by someone else" << std::endl;
+      }
+    } else {
+      std::cerr << "rbd: taking lock failed: " << cpp_strerror(r) << std::endl;
+    }
+    return r;
+  }
+  return 0;
+}
+
+void get_remove_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  add_id_option(positional);
+  positional->add_options()
+    ("locker", "locker client");
+}
+
+int execute_remove(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string lock_cookie;
+  r = get_id(vm, &arg_index, &lock_cookie);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string lock_client = utils::get_positional_argument(vm, arg_index);
+  if (lock_client.empty()) {
+    std::cerr << "rbd: locker was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_lock_remove(image, lock_client.c_str(), lock_cookie.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: releasing lock failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action_list(
+  {"lock", "list"}, {"lock", "ls"}, "Show locks held on an image.", "",
+  &get_list_arguments, &execute_list);
+Shell::Action action_add(
+  {"lock", "add"}, {}, "Take a lock on an image.", "",
+  &get_add_arguments, &execute_add);
+Shell::Action action_remove(
+  {"lock", "remove"}, {"lock", "rm"}, "Release a lock on an image.", "",
+  &get_remove_arguments, &execute_remove);
+
+} // namespace lock
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/MergeDiff.cc b/src/tools/rbd/action/MergeDiff.cc
new file mode 100644
index 00000000..406b23b4
--- /dev/null
+++ b/src/tools/rbd/action/MergeDiff.cc
@@ -0,0 +1,454 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#define _LARGEFILE64_SOURCE
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "include/compat.h"
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/safe_io.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd
+
+namespace rbd {
+namespace action {
+namespace merge_diff {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int parse_diff_header(int fd, __u8 *tag, string *from, string *to, uint64_t *size)
+{
+  int r;
+
+  {//header
+    char buf[utils::RBD_DIFF_BANNER.size() + 1];
+    r = safe_read_exact(fd, buf, utils::RBD_DIFF_BANNER.size());
+    if (r < 0)
+      return r;
+
+    buf[utils::RBD_DIFF_BANNER.size()] = '\0';
+    if (strcmp(buf, utils::RBD_DIFF_BANNER.c_str())) {
+      std::cerr << "invalid banner '" << buf << "', expected '"
+                << utils::RBD_DIFF_BANNER << "'" << std::endl;
+      return -EINVAL;
+    }
+  }
+
+  while (true) {
+    r = safe_read_exact(fd, tag, 1);
+    if (r < 0)
+      return r;
+
+    if (*tag == RBD_DIFF_FROM_SNAP) {
+      r = utils::read_string(fd, 4096, from);   // 4k limit to make sure we don't get a garbage string
+      if (r < 0)
+        return r;
+      dout(2) << " from snap " << *from << dendl;
+    } else if (*tag == RBD_DIFF_TO_SNAP) {
+      r = utils::read_string(fd, 4096, to);   // 4k limit to make sure we don't get a garbage string
+      if (r < 0)
+        return r;
+      dout(2) << " to snap " << *to << dendl;
+    } else if (*tag == RBD_DIFF_IMAGE_SIZE) {
+      char buf[8];
+      r = safe_read_exact(fd, buf, 8);
+      if (r < 0)
+        return r;
+
+      bufferlist bl;
+      bl.append(buf, 8);
+      auto p = bl.cbegin();
+      decode(*size, p);
+    } else {
+      break;
+    }
+  }
+
+  return 0;
+}
+
+static int parse_diff_body(int fd, __u8 *tag, uint64_t *offset, uint64_t *length)
+{
+  int r;
+
+  if (!(*tag)) {
+    r = safe_read_exact(fd, tag, 1);
+    if (r < 0)
+      return r;
+  }
+
+  if (*tag == RBD_DIFF_END) {
+    offset = 0;
+    length = 0;
+    return 0;
+  }
+
+  if (*tag != RBD_DIFF_WRITE && *tag != RBD_DIFF_ZERO)
+    return -ENOTSUP;
+
+  char buf[16];
+  r = safe_read_exact(fd, buf, 16);
+  if (r < 0)
+    return r;
+
+  bufferlist bl;
+  bl.append(buf, 16);
+  auto p = bl.cbegin();
+  decode(*offset, p);
+  decode(*length, p);
+
+  if (!(*length))
+    return -ENOTSUP;
+
+  return 0;
+}
+
+/*
+ * fd: the diff file to read from
+ * pd: the diff file to be written into
+ */
+static int accept_diff_body(int fd, int pd, __u8 tag, uint64_t offset, uint64_t length)
+{
+  if (tag == RBD_DIFF_END)
+    return 0;
+
+  bufferlist bl;
+  encode(tag, bl);
+  encode(offset, bl);
+  encode(length, bl);
+  int r;
+  r = bl.write_fd(pd);
+  if (r < 0)
+    return r;
+
+  if (tag == RBD_DIFF_WRITE) {
+    bufferptr bp = buffer::create(length);
+    r = safe_read_exact(fd, bp.c_str(), length);
+    if (r < 0)
+      return r;
+    bufferlist data;
+    data.append(bp);
+    r = data.write_fd(pd);
+    if (r < 0)
+      return r;
+  }
+
+  return 0;
+}
+
+/*
+ * Merge two diff files into one single file
+ * Note: It does not do the merging work if
+ * either of the source diff files is stripped,
+ * since which complicates the process and is
+ * rarely used
+ */
+static int do_merge_diff(const char *first, const char *second,
+                         const char *path, bool no_progress)
+{
+  utils::ProgressContext pc("Merging image diff", no_progress);
+  int fd = -1, sd = -1, pd = -1, r;
+
+  string f_from, f_to;
+  string s_from, s_to;
+  uint64_t f_size = 0;
+  uint64_t s_size = 0;
+  uint64_t pc_size;
+
+  __u8 f_tag = 0, s_tag = 0;
+  uint64_t f_off = 0, f_len = 0;
+  uint64_t s_off = 0, s_len = 0;
+  bool f_end = false, s_end = false;
+
+  bool first_stdin = !strcmp(first, "-");
+  if (first_stdin) {
+    fd = STDIN_FILENO;
+  } else {
+    fd = open(first, O_RDONLY);
+    if (fd < 0) {
+      r = -errno;
+      std::cerr << "rbd: error opening " << first << std::endl;
+      goto done;
+    }
+  }
+
+  sd = open(second, O_RDONLY);
+  if (sd < 0) {
+    r = -errno;
+    std::cerr << "rbd: error opening " << second << std::endl;
+    goto done;
+  }
+
+  if (strcmp(path, "-") == 0) {
+    pd = 1;
+  } else {
+    pd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0644);
+    if (pd < 0) {
+      r = -errno;
+      std::cerr << "rbd: error create " << path << std::endl;
+      goto done;
+    }
+  }
+
+  //We just handle the case like 'banner, [ftag], [ttag], stag, [wztag]*,etag',
+  // and the (offset,length) in wztag must be ascending order.
+  r = parse_diff_header(fd, &f_tag, &f_from, &f_to, &f_size);
+  if (r < 0) {
+    std::cerr << "rbd: failed to parse first diff header" << std::endl;
+    goto done;
+  }
+
+  r = parse_diff_header(sd, &s_tag, &s_from, &s_to, &s_size);
+  if (r < 0) {
+    std::cerr << "rbd: failed to parse second diff header" << std::endl;
+    goto done;
+  }
+
+  if (f_to != s_from) {
+    r = -EINVAL;
+    std::cerr << "The first TO snapshot must be equal with the second FROM "
+              << "snapshot, aborting" << std::endl;
+    goto done;
+  }
+
+  {
+    // header
+    bufferlist bl;
+    bl.append(utils::RBD_DIFF_BANNER);
+
+    __u8 tag;
+    if (f_from.size()) {
+      tag = RBD_DIFF_FROM_SNAP;
+      encode(tag, bl);
+      encode(f_from, bl);
+    }
+
+    if (s_to.size()) {
+      tag = RBD_DIFF_TO_SNAP;
+      encode(tag, bl);
+      encode(s_to, bl);
+    }
+
+    tag = RBD_DIFF_IMAGE_SIZE;
+    encode(tag, bl);
+    encode(s_size, bl);
+
+    r = bl.write_fd(pd);
+    if (r < 0) {
+      std::cerr << "rbd: failed to write merged diff header" << std::endl;
+      goto done;
+    }
+  }
+  if (f_size > s_size)
+    pc_size = f_size << 1;
+  else
+    pc_size = s_size << 1;
+
+  //data block
+  while (!f_end || !s_end) {
+    // progress through input
+    pc.update_progress(f_off + s_off, pc_size);
+
+    if (!f_end && !f_len) {
+      uint64_t last_off = f_off;
+
+      r = parse_diff_body(fd, &f_tag, &f_off, &f_len);
+      dout(2) << "first diff data chunk: tag=" << f_tag << ", "
+              << "off=" << f_off << ", "
+              << "len=" << f_len << dendl;
+      if (r < 0) {
+        std::cerr << "rbd: failed to read first diff data chunk header"
+                  << std::endl;
+        goto done;
+      }
+
+      if (f_tag == RBD_DIFF_END) {
+        f_end = true;
+        f_tag = RBD_DIFF_ZERO;
+        f_off = f_size;
+        if (f_size < s_size)
+          f_len = s_size - f_size;
+        else
+          f_len = 0;
+      }
+
+      if (last_off > f_off) {
+        r = -ENOTSUP;
+        std::cerr << "rbd: out-of-order offset from first diff ("
+             << last_off << " > " << f_off << ")" << std::endl;
+        goto done;
+      }
+    }
+
+    if (!s_end && !s_len) {
+      uint64_t last_off = s_off;
+
+      r = parse_diff_body(sd, &s_tag, &s_off, &s_len);
+      dout(2) << "second diff data chunk: tag=" << s_tag << ", "
+              << "off=" << s_off << ", "
+              << "len=" << s_len << dendl;
+      if (r < 0) {
+        std::cerr << "rbd: failed to read second diff data chunk header"
+                  << std::endl;
+        goto done;
+      }
+
+      if (s_tag == RBD_DIFF_END) {
+        s_end = true;
+        s_off = s_size;
+        if (s_size < f_size)
+          s_len = f_size - s_size;
+        else
+          s_len = 0;
+      }
+
+      if (last_off > s_off) {
+        r = -ENOTSUP;
+        std::cerr << "rbd: out-of-order offset from second diff ("
+                  << last_off << " > " << s_off << ")" << std::endl;
+        goto done;
+      }
+    }
+
+    if (f_off < s_off && f_len) {
+      uint64_t delta = s_off - f_off;
+      if (delta > f_len)
+        delta = f_len;
+      r = accept_diff_body(fd, pd, f_tag, f_off, delta);
+      if (r < 0) {
+        std::cerr << "rbd: failed to merge diff chunk" << std::endl;
+        goto done;
+      }
+      f_off += delta;
+      f_len -= delta;
+
+      if (!f_len) {
+        f_tag = 0;
+        continue;
+      }
+    }
+    ceph_assert(f_off >= s_off);
+
+    if (f_off < s_off + s_len && f_len) {
+      uint64_t delta = s_off + s_len - f_off;
+      if (delta > f_len)
+        delta = f_len;
+      if (f_tag == RBD_DIFF_WRITE) {
+        if (first_stdin) {
+          bufferptr bp = buffer::create(delta);
+          r = safe_read_exact(fd, bp.c_str(), delta);
+        } else {
+          off64_t l = lseek64(fd, delta, SEEK_CUR);
+          r = l < 0 ? -errno : 0;
+        }
+        if (r < 0) {
+          std::cerr << "rbd: failed to skip first diff data" << std::endl;
+          goto done;
+        }
+      }
+      f_off += delta;
+      f_len -= delta;
+
+      if (!f_len) {
+        f_tag = 0;
+        continue;
+      }
+    }
+    ceph_assert(f_off >= s_off + s_len);
+    if (s_len) {
+      r = accept_diff_body(sd, pd, s_tag, s_off, s_len);
+      if (r < 0) {
+        std::cerr << "rbd: failed to merge diff chunk" << std::endl;
+        goto done;
+      }
+      s_off += s_len;
+      s_len = 0;
+      s_tag = 0;
+    } else {
+      ceph_assert(f_end && s_end);
+    }
+    continue;
+  }
+
+  {//tail
+    __u8 tag = RBD_DIFF_END;
+    bufferlist bl;
+    encode(tag, bl);
+    r = bl.write_fd(pd);
+  }
+
+done:
+  if (pd > 2)
+    close(pd);
+  if (sd > 2)
+    close(sd);
+  if (fd > 2)
+    close(fd);
+
+  if(r < 0) {
+    pc.fail();
+    if (pd > 2)
+      unlink(path);
+  } else
+    pc.finish();
+
+  return r;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  positional->add_options()
+    ("diff1-path", "path to first diff (or '-' for stdin)")
+    ("diff2-path", "path to second diff");
+  at::add_path_options(positional, options,
+                       "path to merged diff (or '-' for stdout)");
+  at::add_no_progress_option(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  std::string first_diff = utils::get_positional_argument(vm, 0);
+  if (first_diff.empty()) {
+    std::cerr << "rbd: first diff was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  std::string second_diff = utils::get_positional_argument(vm, 1);
+  if (second_diff.empty()) {
+    std::cerr << "rbd: second diff was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  std::string path;
+  size_t arg_index = 2;
+  int r = utils::get_path(vm, &arg_index, &path);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_merge_diff(first_diff.c_str(), second_diff.c_str(), path.c_str(),
+                    vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    cerr << "rbd: merge-diff error" << std::endl;
+    return -r;
+  }
+
+  return 0;
+}
+
+Shell::Action action(
+  {"merge-diff"}, {}, "Merge two diff exports together.", "",
+  &get_arguments, &execute);
+
+} // namespace merge_diff
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Migration.cc b/src/tools/rbd/action/Migration.cc
new file mode 100644
index 00000000..bb05e376
--- /dev/null
+++ b/src/tools/rbd/action/Migration.cc
@@ -0,0 +1,338 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/errno.h"
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace migration {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_prepare(librados::IoCtx& io_ctx, const std::string &image_name,
+                      librados::IoCtx& dest_io_ctx,
+                      const std::string &dest_image_name,
+                      librbd::ImageOptions& opts) {
+  int r = librbd::RBD().migration_prepare(io_ctx, image_name.c_str(),
+                                          dest_io_ctx, dest_image_name.c_str(),
+                                          opts);
+  if (r < 0) {
+    std::cerr << "rbd: preparing migration failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+static int do_execute(librados::IoCtx& io_ctx, const std::string &image_name,
+                      bool no_progress) {
+  utils::ProgressContext pc("Image migration", no_progress);
+  int r = librbd::RBD().migration_execute_with_progress(io_ctx,
+                                                        image_name.c_str(), pc);
+  if (r < 0) {
+    pc.fail();
+    std::cerr << "rbd: migration failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+static int do_abort(librados::IoCtx& io_ctx, const std::string &image_name,
+                    bool no_progress) {
+  utils::ProgressContext pc("Abort image migration", no_progress);
+  int r = librbd::RBD().migration_abort_with_progress(io_ctx,
+                                                      image_name.c_str(), pc);
+  if (r < 0) {
+    pc.fail();
+    std::cerr << "rbd: aborting migration failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+static int do_commit(librados::IoCtx& io_ctx, const std::string &image_name,
+                     bool force, bool no_progress) {
+  librbd::image_migration_status_t migration_status;
+  int r = librbd::RBD().migration_status(io_ctx, image_name.c_str(),
+                                         &migration_status,
+                                         sizeof(migration_status));
+  if (r < 0) {
+    std::cerr << "rbd: getting migration status failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  librados::IoCtx dst_io_ctx;
+  r = librados::Rados(io_ctx).ioctx_create2(migration_status.dest_pool_id, dst_io_ctx);
+  if (r < 0) {
+    std::cerr << "rbd: accessing source pool id="
+              << migration_status.dest_pool_id << " failed: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  r = utils::set_namespace(migration_status.dest_pool_namespace, &dst_io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::Image image;
+  r = utils::open_image_by_id(dst_io_ctx, migration_status.dest_image_id,
+                              true, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  std::vector<librbd::linked_image_spec_t> children;
+  r = image.list_descendants(&children);
+  if (r < 0) {
+    std::cerr << "rbd: listing descendants failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  if (children.size() > 0) {
+    std::cerr << "rbd: the image has "
+              << (children.size() == 1 ? "a descendant" : "descendants") << ": "
+              << std::endl;
+    for (auto& child : children) {
+      std::cerr << "  " << child.pool_name << "/";
+      if (!child.pool_namespace.empty()) {
+        std::cerr << child.pool_namespace << "/";
+      }
+      std::cerr << child.image_name;
+      if (child.trash) {
+        std::cerr << " (trash " << child.image_id << ")";
+      }
+      std::cerr << std::endl;
+    }
+    std::cerr << "Warning: in-use, read-only descendant images"
+              << " will not detect the parent update." << std::endl;
+    if (force) {
+      std::cerr << "Proceeding anyway due to force flag set." << std::endl;
+    } else {
+      std::cerr << "Ensure no descendant images are opened read-only"
+                << " and run again with force flag." << std::endl;
+      return -EBUSY;
+    }
+  }
+
+  utils::ProgressContext pc("Commit image migration", no_progress);
+  r = librbd::RBD().migration_commit_with_progress(io_ctx, image_name.c_str(),
+                                                   pc);
+  if (r < 0) {
+    pc.fail();
+    std::cerr << "rbd: committing migration failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+void get_prepare_arguments(po::options_description *positional,
+                           po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_SOURCE);
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
+  at::add_create_image_options(options, true);
+  at::add_flatten_option(options);
+}
+
+int execute_prepare(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_SOURCE, &arg_index, &pool_name, &namespace_name,
+    &image_name, nullptr, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+  io_ctx.set_osdmap_full_try();
+
+  std::string dest_pool_name;
+  std::string dest_namespace_name;
+  std::string dest_image_name;
+  r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, &dest_pool_name,
+    &dest_namespace_name, &dest_image_name, nullptr, false,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::ImageOptions opts;
+  r = utils::get_image_options(vm, true, &opts);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::IoCtx dest_io_ctx;
+  if (!dest_pool_name.empty()) {
+    r = utils::init_io_ctx(rados, dest_pool_name, dest_namespace_name,
+                           &dest_io_ctx);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  r = do_prepare(io_ctx, image_name, dest_pool_name.empty() ? io_ctx :
+                 dest_io_ctx, dest_image_name, opts);
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+void get_execute_arguments(po::options_description *positional,
+                           po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_no_progress_option(options);
+}
+
+int execute_execute(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, nullptr, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+  io_ctx.set_osdmap_full_try();
+
+  r = do_execute(io_ctx, image_name, vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+void get_abort_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_no_progress_option(options);
+}
+
+int execute_abort(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, nullptr, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+  io_ctx.set_osdmap_full_try();
+
+  r = do_abort(io_ctx, image_name, vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+void get_commit_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_no_progress_option(options);
+  options->add_options()
+      ("force", po::bool_switch(), "proceed even if the image has children");
+}
+
+int execute_commit(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, nullptr, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+  io_ctx.set_osdmap_full_try();
+
+  r = do_commit(io_ctx, image_name, vm["force"].as<bool>(),
+                vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+Shell::Action action_prepare(
+  {"migration", "prepare"}, {}, "Prepare image migration.",
+  at::get_long_features_help(), &get_prepare_arguments, &execute_prepare);
+
+Shell::Action action_execute(
+  {"migration", "execute"}, {}, "Execute image migration.", "",
+  &get_execute_arguments, &execute_execute);
+
+Shell::Action action_abort(
+  {"migration", "abort"}, {}, "Cancel interrupted image migration.", "",
+  &get_abort_arguments, &execute_abort);
+
+Shell::Action action_commit(
+  {"migration", "commit"}, {}, "Commit image migration.", "",
+  &get_commit_arguments, &execute_commit);
+
+} // namespace migration
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/MirrorImage.cc b/src/tools/rbd/action/MirrorImage.cc
new file mode 100644
index 00000000..a250b694
--- /dev/null
+++ b/src/tools/rbd/action/MirrorImage.cc
@@ -0,0 +1,360 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2016 SUSE LINUX GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/MirrorDaemonServiceInfo.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/stringify.h"
+#include "common/config.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include "global/global_context.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace mirror_image {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+namespace {
+
+int validate_mirroring_enabled(librbd::Image& image) {
+  librbd::mirror_image_info_t mirror_image;
+  int r = image.mirror_image_get_info(&mirror_image, sizeof(mirror_image));
+  if (r < 0) {
+    std::cerr << "rbd: failed to retrieve mirror mode: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  if (mirror_image.state != RBD_MIRROR_IMAGE_ENABLED) {
+    std::cerr << "rbd: mirroring not enabled on the image" << std::endl;
+    return -EINVAL;
+  }
+  return 0;
+}
+
+} // anonymous namespace
+
+void get_arguments(po::options_description *positional,
+                           po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+}
+
+void get_arguments_disable(po::options_description *positional,
+                           po::options_description *options) {
+  options->add_options()
+    ("force", po::bool_switch(), "disable even if not primary");
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+}
+
+int execute_enable_disable(const po::variables_map &vm, bool enable,
+                           bool force) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+      vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, nullptr,
+      &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+      utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, "", image_name, "", "", false,
+                                 &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = enable ? image.mirror_image_enable() : image.mirror_image_disable(force);
+  if (r < 0) {
+    return r;
+  }
+
+  std::cout << (enable ? "Mirroring enabled" : "Mirroring disabled")
+    << std::endl;
+
+  return 0;
+}
+
+int execute_disable(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  return execute_enable_disable(vm, false, vm["force"].as<bool>());
+}
+
+int execute_enable(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  return execute_enable_disable(vm, true, false);
+}
+
+void get_arguments_promote(po::options_description *positional,
+                           po::options_description *options) {
+  options->add_options()
+    ("force", po::bool_switch(), "promote even if not cleanly demoted by remote cluster");
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+}
+
+int execute_promote(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+      vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, nullptr,
+      &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+      utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  bool force = vm["force"].as<bool>();
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, "", image_name, "", "", false,
+                                 &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = image.mirror_image_promote(force);
+  if (r < 0) {
+    std::cerr << "rbd: error promoting image to primary" << std::endl;
+    return r;
+  }
+
+  std::cout << "Image promoted to primary" << std::endl;
+  return 0;
+}
+
+int execute_demote(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+      vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, nullptr,
+      &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+      utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, "", image_name, "", "", false,
+                                 &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = image.mirror_image_demote();
+  if (r < 0) {
+    std::cerr << "rbd: error demoting image to non-primary" << std::endl;
+    return r;
+  }
+
+  std::cout << "Image demoted to non-primary" << std::endl;
+  return 0;
+}
+
+int execute_resync(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+      vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, nullptr,
+      &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+      utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, "", image_name, "", "", false,
+                                 &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = image.mirror_image_resync();
+  if (r < 0) {
+    std::cerr << "rbd: error flagging image resync" << std::endl;
+    return r;
+  }
+
+  std::cout << "Flagged image for resync from primary" << std::endl;
+  return 0;
+}
+
+void get_status_arguments(po::options_description *positional,
+			  po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_format_options(options);
+}
+
+int execute_status(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  at::Format::Formatter formatter;
+  int r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string image_name;
+  std::string snap_name;
+  r = utils::get_pool_image_snapshot_names(
+      vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, nullptr,
+      &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+      utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, "", image_name, "", "", false,
+                                 &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(image);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::mirror_image_status_t status;
+  r = image.mirror_image_get_status(&status, sizeof(status));
+  if (r < 0) {
+    std::cerr << "rbd: failed to get status for image " << image_name << ": "
+	      << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  std::string instance_id;
+  MirrorDaemonServiceInfo daemon_service_info(io_ctx);
+
+  if (status.up) {
+    r = image.mirror_image_get_instance_id(&instance_id);
+    if (r == -EOPNOTSUPP) {
+      std::cerr << "rbd: newer release of Ceph OSDs required to map image "
+                << "to rbd-mirror daemon instance" << std::endl;
+      // not fatal
+    } else if (r < 0 && r != -ENOENT) {
+      std::cerr << "rbd: failed to get service id for image "
+                << image_name << ": " << cpp_strerror(r) << std::endl;
+      // not fatal
+    } else if (!instance_id.empty()) {
+      daemon_service_info.init();
+    }
+  }
+
+  std::string state = utils::mirror_image_status_state(status);
+  std::string last_update = (
+    status.last_update == 0 ? "" : utils::timestr(status.last_update));
+
+  if (formatter != nullptr) {
+    formatter->open_object_section("image");
+    formatter->dump_string("name", image_name);
+    formatter->dump_string("global_id", status.info.global_id);
+    formatter->dump_string("state", state);
+    formatter->dump_string("description", status.description);
+    daemon_service_info.dump(instance_id, formatter);
+    formatter->dump_string("last_update", last_update);
+    formatter->close_section(); // image
+    formatter->flush(std::cout);
+  } else {
+    std::cout << image_name << ":\n"
+	      << "  global_id:   " << status.info.global_id << "\n"
+	      << "  state:       " << state << "\n"
+              << "  description: " << status.description << "\n";
+    if (!instance_id.empty()) {
+      std::cout << "  service:     " <<
+        daemon_service_info.get_description(instance_id) << "\n";
+    }
+    std::cout << "  last_update: " << last_update << std::endl;
+  }
+
+  return 0;
+}
+
+Shell::Action action_enable(
+  {"mirror", "image", "enable"}, {},
+  "Enable RBD mirroring for an image.", "",
+  &get_arguments, &execute_enable);
+Shell::Action action_disable(
+  {"mirror", "image", "disable"}, {},
+  "Disable RBD mirroring for an image.", "",
+  &get_arguments_disable, &execute_disable);
+Shell::Action action_promote(
+  {"mirror", "image", "promote"}, {},
+  "Promote an image to primary for RBD mirroring.", "",
+  &get_arguments_promote, &execute_promote);
+Shell::Action action_demote(
+  {"mirror", "image", "demote"}, {},
+  "Demote an image to non-primary for RBD mirroring.", "",
+  &get_arguments, &execute_demote);
+Shell::Action action_resync(
+  {"mirror", "image", "resync"}, {},
+  "Force resync to primary image for RBD mirroring.", "",
+  &get_arguments, &execute_resync);
+Shell::Action action_status(
+  {"mirror", "image", "status"}, {},
+  "Show RBD mirroring status for an image.", "",
+  &get_status_arguments, &execute_status);
+
+} // namespace mirror_image
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/MirrorPool.cc b/src/tools/rbd/action/MirrorPool.cc
new file mode 100644
index 00000000..ff7c3031
--- /dev/null
+++ b/src/tools/rbd/action/MirrorPool.cc
@@ -0,0 +1,1537 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/MirrorDaemonServiceInfo.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/Context.h"
+#include "include/stringify.h"
+#include "include/rbd/librbd.hpp"
+#include "common/ceph_json.h"
+#include "common/config.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include "common/Throttle.h"
+#include "global/global_context.h"
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <regex>
+#include <set>
+#include <boost/program_options.hpp>
+#include "include/ceph_assert.h"
+
+#include <atomic>
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::action::MirrorPool: "
+
+namespace rbd {
+namespace action {
+namespace mirror_pool {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static const std::string ALL_NAME("all");
+static const std::string SITE_NAME("site-name");
+
+namespace {
+
+void add_site_name_optional(po::options_description *options) {
+  options->add_options()
+    (SITE_NAME.c_str(), po::value<std::string>(), "local site name");
+}
+
+int set_site_name(librados::Rados& rados, const std::string& site_name) {
+  librbd::RBD rbd;
+  int r = rbd.mirror_site_name_set(rados, site_name);
+  if (r == -EOPNOTSUPP) {
+    std::cerr << "rbd: cluster does not support site names" << std::endl;
+    return r;
+  } else if (r < 0) {
+    std::cerr << "rbd: failed to set site name" << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+struct MirrorPeerDirection {};
+
+void validate(boost::any& v, const std::vector<std::string>& values,
+              MirrorPeerDirection *target_type, int) {
+  po::validators::check_first_occurrence(v);
+  const std::string &s = po::validators::get_single_string(values);
+
+  if (s == "rx-only") {
+    v = boost::any(RBD_MIRROR_PEER_DIRECTION_RX);
+  } else if (s == "rx-tx") {
+    v = boost::any(RBD_MIRROR_PEER_DIRECTION_RX_TX);
+  } else {
+    throw po::validation_error(po::validation_error::invalid_option_value);
+  }
+}
+
+int validate_mirroring_enabled(librados::IoCtx& io_ctx) {
+  librbd::RBD rbd;
+  rbd_mirror_mode_t mirror_mode;
+  int r = rbd.mirror_mode_get(io_ctx, &mirror_mode);
+  if (r < 0) {
+    std::cerr << "rbd: failed to retrieve mirror mode: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  if (mirror_mode == RBD_MIRROR_MODE_DISABLED) {
+    std::cerr << "rbd: mirroring not enabled on the pool" << std::endl;
+    return -EINVAL;
+  }
+  return 0;
+}
+
+int validate_uuid(const std::string &uuid) {
+  std::regex pattern("^[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12}$",
+                       std::regex::icase);
+  std::smatch match;
+  if (!std::regex_match(uuid, match, pattern)) {
+    std::cerr << "rbd: invalid uuid '" << uuid << "'" << std::endl;
+    return -EINVAL;
+  }
+  return 0;
+}
+
+int read_key_file(std::string path, std::string* key) {
+  std::ifstream key_file;
+  key_file.open(path);
+  if (key_file.fail()) {
+    std::cerr << "rbd: failed to open " << path << std::endl;
+    return -EINVAL;
+  }
+
+  std::getline(key_file, *key);
+  if (key_file.bad()) {
+    std::cerr << "rbd: failed to read key from " << path << std::endl;
+    return -EINVAL;
+  }
+
+  key_file.close();
+  return 0;
+}
+
+void add_uuid_option(po::options_description *positional) {
+  positional->add_options()
+    ("uuid", po::value<std::string>(), "peer uuid");
+}
+
+int get_uuid(const po::variables_map &vm, size_t arg_index,
+             std::string *uuid) {
+  *uuid = utils::get_positional_argument(vm, arg_index);
+  if (uuid->empty()) {
+    std::cerr << "rbd: must specify peer uuid" << std::endl;
+    return -EINVAL;
+  }
+  return validate_uuid(*uuid);
+}
+
+int get_remote_cluster_spec(const po::variables_map &vm,
+                            const std::string &spec,
+                            std::string *remote_client_name,
+                            std::string *remote_cluster,
+                            std::map<std::string, std::string>* attributes) {
+  if (vm.count("remote-client-name")) {
+    *remote_client_name = vm["remote-client-name"].as<std::string>();
+  }
+  if (vm.count("remote-cluster")) {
+    *remote_cluster = vm["remote-cluster"].as<std::string>();
+  }
+  if (vm.count("remote-mon-host")) {
+    (*attributes)["mon_host"] = vm["remote-mon-host"].as<std::string>();
+  }
+  if (vm.count("remote-key-file")) {
+    std::string key;
+    int r = read_key_file(vm["remote-key-file"].as<std::string>(), &key);
+    if (r < 0) {
+      return r;
+    }
+    (*attributes)["key"] = key;
+  }
+
+  if (!spec.empty()) {
+    std::regex pattern("^(?:(client\\.[^@]+)@)?([^/@]+)$");
+    std::smatch match;
+    if (!std::regex_match(spec, match, pattern)) {
+      std::cerr << "rbd: invalid spec '" << spec << "'" << std::endl;
+      return -EINVAL;
+    }
+    if (match[1].matched) {
+      *remote_client_name = match[1];
+    }
+    *remote_cluster = match[2];
+  }
+
+  if (remote_cluster->empty()) {
+    std::cerr << "rbd: remote cluster was not specified" << std::endl;
+    return -EINVAL;
+  }
+  return 0;
+}
+
+int set_peer_config_key(librados::IoCtx& io_ctx, const std::string& peer_uuid,
+                        std::map<std::string, std::string>&& attributes) {
+  librbd::RBD rbd;
+  int r = rbd.mirror_peer_set_attributes(io_ctx, peer_uuid, attributes);
+  if (r == -EPERM) {
+    std::cerr << "rbd: permission denied attempting to set peer "
+              << "config-key secrets in the monitor" << std::endl;
+    return r;
+  } else if (r < 0) {
+    std::cerr << "rbd: failed to update mirroring peer config: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+int get_peer_config_key(librados::IoCtx& io_ctx, const std::string& peer_uuid,
+                        std::map<std::string, std::string>* attributes) {
+  librbd::RBD rbd;
+  int r = rbd.mirror_peer_get_attributes(io_ctx, peer_uuid, attributes);
+  if (r == -ENOENT) {
+    return r;
+  } else if (r == -EPERM) {
+    std::cerr << "rbd: permission denied attempting to access peer "
+              << "config-key secrets from the monitor" << std::endl;
+    return r;
+  } else if (r == -EINVAL) {
+    std::cerr << "rbd: corrupt mirroring peer config" << std::endl;
+    return r;
+  } else if (r < 0) {
+    std::cerr << "rbd: error reading mirroring peer config: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+int update_peer_config_key(librados::IoCtx& io_ctx,
+                           const std::string& peer_uuid,
+                           const std::string& key,
+                           const std::string& value) {
+  std::map<std::string, std::string> attributes;
+  int r = get_peer_config_key(io_ctx, peer_uuid, &attributes);
+  if (r == -ENOENT) {
+    return set_peer_config_key(io_ctx, peer_uuid, {{key, value}});
+  } else if (r < 0) {
+    return r;
+  }
+
+  if (value.empty()) {
+    attributes.erase(key);
+  } else {
+    attributes[key] = value;
+  }
+  return set_peer_config_key(io_ctx, peer_uuid, std::move(attributes));
+}
+
+int format_mirror_peers(librados::IoCtx& io_ctx,
+                        at::Format::Formatter formatter,
+                        const std::vector<librbd::mirror_peer_t> &peers,
+                        bool config_key) {
+  TextTable tbl;
+  if (formatter != nullptr) {
+    formatter->open_array_section("peers");
+  } else {
+    std::cout << "Peers: ";
+    if (peers.empty()) {
+      std::cout << "none" << std::endl;
+    } else {
+      tbl.define_column("", TextTable::LEFT, TextTable::LEFT);
+      tbl.define_column("UUID", TextTable::LEFT, TextTable::LEFT);
+      tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+      tbl.define_column("CLIENT", TextTable::LEFT, TextTable::LEFT);
+      if (config_key) {
+        tbl.define_column("MON_HOST", TextTable::LEFT, TextTable::LEFT);
+        tbl.define_column("KEY", TextTable::LEFT, TextTable::LEFT);
+      }
+    }
+  }
+
+  for (auto &peer : peers) {
+    std::map<std::string, std::string> attributes;
+    if (config_key) {
+      int r = get_peer_config_key(io_ctx, peer.uuid, &attributes);
+      if (r < 0 && r != -ENOENT) {
+        return r;
+      }
+    }
+
+    if (formatter != nullptr) {
+      formatter->open_object_section("peer");
+      formatter->dump_string("uuid", peer.uuid);
+      formatter->dump_string("cluster_name", peer.cluster_name);
+      formatter->dump_string("client_name", peer.client_name);
+      for (auto& pair : attributes) {
+        formatter->dump_string(pair.first.c_str(), pair.second);
+      }
+      formatter->close_section();
+    } else {
+      tbl << " "
+          << peer.uuid
+          << peer.cluster_name
+          << peer.client_name;
+      if (config_key) {
+        tbl << attributes["mon_host"]
+            << attributes["key"];
+      }
+      tbl << TextTable::endrow;
+    }
+  }
+
+  if (formatter != nullptr) {
+    formatter->close_section();
+  } else {
+    std::cout << std::endl << tbl;
+  }
+  return 0;
+}
+
+class ImageRequestBase {
+public:
+  void send() {
+    dout(20) << this << " " << __func__ << ": image_name=" << m_image_name
+             << dendl;
+
+    auto ctx = new FunctionContext([this](int r) {
+        handle_finalize(r);
+      });
+
+    // will pause here until slots are available
+    m_finalize_ctx = m_throttle.start_op(ctx);
+
+    open_image();
+  }
+
+protected:
+  ImageRequestBase(librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+                   const std::string &image_name)
+    : m_io_ctx(io_ctx), m_throttle(throttle), m_image_name(image_name) {
+  }
+  virtual ~ImageRequestBase() {
+  }
+
+  virtual bool skip_get_info() const {
+    return false;
+  }
+  virtual void get_info(librbd::Image &image, librbd::mirror_image_info_t *info,
+                        librbd::RBD::AioCompletion *aio_comp) {
+    image.aio_mirror_image_get_info(info, sizeof(librbd::mirror_image_info_t),
+                                    aio_comp);
+  }
+
+  virtual bool skip_action(const librbd::mirror_image_info_t &info) const {
+    return false;
+  }
+  virtual void execute_action(librbd::Image &image,
+                              librbd::RBD::AioCompletion *aio_comp) = 0;
+  virtual void handle_execute_action(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r < 0 && r != -ENOENT) {
+      std::cerr << "rbd: failed to " << get_action_type() << " image "
+                << m_image_name << ": " << cpp_strerror(r) << std::endl;
+      m_ret_val = r;
+    }
+
+    close_image();
+  }
+
+  virtual void finalize_action() {
+  }
+  virtual std::string get_action_type() const = 0;
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * OPEN_IMAGE
+   *    |
+   *    v
+   * GET_INFO
+   *    |
+   *    v
+   * EXECUTE_ACTION
+   *    |
+   *    v
+   * CLOSE_IMAGE
+   *    |
+   *    v
+   * FINALIZE_ACTION
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  librados::IoCtx &m_io_ctx;
+  OrderedThrottle &m_throttle;
+  const std::string m_image_name;
+
+  librbd::Image m_image;
+  Context *m_finalize_ctx = nullptr;
+
+  librbd::mirror_image_info_t m_mirror_image_info;
+
+  int m_ret_val = 0;
+
+  void open_image() {
+    dout(20) << this << " " << __func__ << dendl;
+
+    librbd::RBD rbd;
+    auto aio_completion = utils::create_aio_completion<
+      ImageRequestBase, &ImageRequestBase::handle_open_image>(this);
+    rbd.aio_open(m_io_ctx, m_image, m_image_name.c_str(), nullptr,
+                 aio_completion);
+  }
+
+  void handle_open_image(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r < 0) {
+      std::cerr << "rbd: failed to open image "
+                << m_image_name << ": " << cpp_strerror(r) << std::endl;
+      m_finalize_ctx->complete(r);
+      return;
+    }
+
+    get_info();
+  }
+
+  void get_info() {
+    if (skip_get_info()) {
+      execute_action();
+      return;
+    }
+    dout(20) << this << " " << __func__ << dendl;
+
+    auto aio_completion = utils::create_aio_completion<
+      ImageRequestBase, &ImageRequestBase::handle_get_info>(this);
+    get_info(m_image, &m_mirror_image_info, aio_completion);
+  }
+
+  void handle_get_info(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r == -ENOENT) {
+      close_image();
+      return;
+    } else if (r < 0) {
+      std::cerr << "rbd: failed to retrieve mirror image info for "
+                << m_image_name << ": " << cpp_strerror(r) << std::endl;
+      m_ret_val = r;
+      close_image();
+      return;
+    }
+
+    execute_action();
+  }
+
+  void execute_action() {
+    if (skip_action(m_mirror_image_info)) {
+      close_image();
+      return;
+    }
+    dout(20) << this << " " << __func__ << dendl;
+
+    auto aio_completion = utils::create_aio_completion<
+      ImageRequestBase, &ImageRequestBase::handle_execute_action>(this);
+    execute_action(m_image, aio_completion);
+  }
+
+  void close_image() {
+    dout(20) << this << " " << __func__ << dendl;
+
+    auto aio_completion = utils::create_aio_completion<
+      ImageRequestBase, &ImageRequestBase::handle_close_image>(this);
+    m_image.aio_close(aio_completion);
+  }
+
+  void handle_close_image(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r < 0) {
+      std::cerr << "rbd: failed to close image "
+                << m_image_name << ": " << cpp_strerror(r) << std::endl;
+    }
+
+    m_finalize_ctx->complete(r);
+  }
+
+  void handle_finalize(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r == 0 && m_ret_val < 0) {
+      r = m_ret_val;
+    }
+    if (r >= 0) {
+      finalize_action();
+    }
+    m_throttle.end_op(r);
+    delete this;
+  }
+
+};
+
+class PromoteImageRequest : public ImageRequestBase {
+public:
+  PromoteImageRequest(librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+                      const std::string &image_name, std::atomic<unsigned> *counter,
+                      bool force)
+    : ImageRequestBase(io_ctx, throttle, image_name), m_counter(counter),
+      m_force(force) {
+  }
+
+protected:
+  bool skip_action(const librbd::mirror_image_info_t &info) const override {
+    return (info.state != RBD_MIRROR_IMAGE_ENABLED || info.primary);
+  }
+
+  void execute_action(librbd::Image &image,
+                      librbd::RBD::AioCompletion *aio_comp) override {
+    image.aio_mirror_image_promote(m_force, aio_comp);
+  }
+
+  void handle_execute_action(int r) override {
+    if (r >= 0) {
+      (*m_counter)++;
+    }
+    ImageRequestBase::handle_execute_action(r);
+  }
+
+  std::string get_action_type() const override {
+    return "promote";
+  }
+
+private:
+  std::atomic<unsigned> *m_counter = nullptr;
+  bool m_force;
+};
+
+class DemoteImageRequest : public ImageRequestBase {
+public:
+  DemoteImageRequest(librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+                     const std::string &image_name, std::atomic<unsigned> *counter)
+    : ImageRequestBase(io_ctx, throttle, image_name), m_counter(counter) {
+  }
+
+protected:
+  bool skip_action(const librbd::mirror_image_info_t &info) const override {
+    return (info.state != RBD_MIRROR_IMAGE_ENABLED || !info.primary);
+  }
+
+  void execute_action(librbd::Image &image,
+                      librbd::RBD::AioCompletion *aio_comp) override {
+    image.aio_mirror_image_demote(aio_comp);
+  }
+  void handle_execute_action(int r) override {
+    if (r >= 0) {
+      (*m_counter)++;
+    }
+    ImageRequestBase::handle_execute_action(r);
+  }
+
+  std::string get_action_type() const override {
+    return "demote";
+  }
+
+private:
+  std::atomic<unsigned> *m_counter = nullptr;
+};
+
+class StatusImageRequest : public ImageRequestBase {
+public:
+  StatusImageRequest(
+      librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+      const std::string &image_name,
+      const std::map<std::string, std::string> &instance_ids,
+      const MirrorDaemonServiceInfo &daemon_service_info,
+      at::Format::Formatter formatter)
+    : ImageRequestBase(io_ctx, throttle, image_name),
+      m_instance_ids(instance_ids), m_daemon_service_info(daemon_service_info),
+      m_formatter(formatter) {
+  }
+
+protected:
+  bool skip_get_info() const override {
+    return true;
+  }
+
+  void execute_action(librbd::Image &image,
+                      librbd::RBD::AioCompletion *aio_comp) override {
+    image.get_id(&m_image_id);
+    image.aio_mirror_image_get_status(&m_mirror_image_status,
+                                      sizeof(m_mirror_image_status), aio_comp);
+  }
+
+  void finalize_action() override {
+    if (m_mirror_image_status.info.global_id.empty()) {
+      return;
+    }
+
+    std::string state = utils::mirror_image_status_state(m_mirror_image_status);
+    std::string instance_id = (m_mirror_image_status.up &&
+                               m_instance_ids.count(m_image_id)) ?
+        m_instance_ids.find(m_image_id)->second : "";
+    std::string last_update = (
+      m_mirror_image_status.last_update == 0 ?
+        "" : utils::timestr(m_mirror_image_status.last_update));
+
+    if (m_formatter != nullptr) {
+      m_formatter->open_object_section("image");
+      m_formatter->dump_string("name", m_mirror_image_status.name);
+      m_formatter->dump_string("global_id",
+                               m_mirror_image_status.info.global_id);
+      m_formatter->dump_string("state", state);
+      m_formatter->dump_string("description",
+                               m_mirror_image_status.description);
+      m_daemon_service_info.dump(instance_id, m_formatter);
+      m_formatter->dump_string("last_update", last_update);
+      m_formatter->close_section(); // image
+    } else {
+      std::cout << "\n" << m_mirror_image_status.name << ":\n"
+	        << "  global_id:   "
+                << m_mirror_image_status.info.global_id << "\n"
+	        << "  state:       " << state << "\n"
+	        << "  description: "
+                << m_mirror_image_status.description << "\n";
+      if (!instance_id.empty()) {
+        std::cout << "  service:     "
+                  << m_daemon_service_info.get_description(instance_id) << "\n";
+      }
+      std::cout << "  last_update: " << last_update << std::endl;
+    }
+  }
+
+  std::string get_action_type() const override {
+    return "status";
+  }
+
+private:
+  const std::map<std::string, std::string> &m_instance_ids;
+  const MirrorDaemonServiceInfo &m_daemon_service_info;
+  at::Format::Formatter m_formatter;
+  std::string m_image_id;
+  librbd::mirror_image_status_t m_mirror_image_status;
+};
+
+template <typename RequestT>
+class ImageRequestAllocator {
+public:
+  template <class... Args>
+  RequestT *operator()(librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+                       const std::string &image_name, Args&&... args) {
+    return new RequestT(io_ctx, throttle, image_name,
+                        std::forward<Args>(args)...);
+  }
+};
+
+template <typename RequestT>
+class ImageRequestGenerator {
+public:
+  template <class... Args>
+  ImageRequestGenerator(librados::IoCtx &io_ctx, Args&&... args)
+    : m_io_ctx(io_ctx),
+      m_factory(std::bind(ImageRequestAllocator<RequestT>(),
+                          std::ref(m_io_ctx), std::ref(m_throttle),
+                          std::placeholders::_1, std::forward<Args>(args)...)),
+      m_throttle(g_conf().get_val<uint64_t>("rbd_concurrent_management_ops"),
+                 true) {
+  }
+
+  int execute() {
+    // use the alphabetical list of image names for pool-level
+    // mirror image operations
+    librbd::RBD rbd;
+    int r = rbd.list2(m_io_ctx, &m_images);
+    if (r < 0 && r != -ENOENT) {
+      std::cerr << "rbd: failed to list images within pool" << std::endl;
+      return r;
+    }
+
+    for (auto &image : m_images) {
+      auto request = m_factory(image.name);
+      request->send();
+    }
+
+    return m_throttle.wait_for_ret();
+  }
+private:
+  typedef std::function<RequestT*(const std::string&)>  Factory;
+
+  librados::IoCtx &m_io_ctx;
+  Factory m_factory;
+
+  OrderedThrottle m_throttle;
+
+  std::vector<librbd::image_spec_t> m_images;
+
+};
+
+} // anonymous namespace
+
+void get_peer_bootstrap_create_arguments(po::options_description *positional,
+                                         po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  options->add_options()
+    (SITE_NAME.c_str(), po::value<std::string>(), "local site name");
+}
+
+int execute_peer_bootstrap_create(
+    const po::variables_map &vm,
+    const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  if (vm.count(SITE_NAME)) {
+    r = set_site_name(rados, vm[SITE_NAME].as<std::string>());
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  librbd::RBD rbd;
+  std::string token;
+  r = rbd.mirror_peer_bootstrap_create(io_ctx, &token);
+  if (r == -EEXIST) {
+    std::cerr << "rbd: mismatch with pre-existing RBD mirroring peer user caps"
+              << std::endl;
+  } else if (r < 0) {
+    std::cerr << "rbd: failed to create mirroring bootstrap token: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  std::cout << token << std::endl;
+  return 0;
+}
+
+void get_peer_bootstrap_import_arguments(po::options_description *positional,
+                                         po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  options->add_options()
+    (SITE_NAME.c_str(), po::value<std::string>(), "local site name");
+  positional->add_options()
+    ("token-path", po::value<std::string>(),
+     "bootstrap token file (or '-' for stdin)");
+  options->add_options()
+    ("token-path", po::value<std::string>(),
+     "bootstrap token file (or '-' for stdin)")
+    ("direction", po::value<MirrorPeerDirection>(),
+     "mirroring direction (rx-only, rx-tx)\n"
+     "[default: rx-tx]");
+}
+
+int execute_peer_bootstrap_import(
+    const po::variables_map &vm,
+    const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string token_path;
+  if (vm.count("token-path")) {
+    token_path = vm["token-path"].as<std::string>();
+  } else {
+    token_path = utils::get_positional_argument(vm, arg_index++);
+  }
+
+  if (token_path.empty()) {
+    std::cerr << "rbd: token path was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  rbd_mirror_peer_direction_t mirror_peer_direction =
+    RBD_MIRROR_PEER_DIRECTION_RX_TX;
+  if (vm.count("direction")) {
+    mirror_peer_direction = vm["direction"].as<rbd_mirror_peer_direction_t>();
+  }
+
+  int fd = STDIN_FILENO;
+  if (token_path != "-") {
+    fd = open(token_path.c_str(), O_RDONLY);
+    if (fd < 0) {
+      r = -errno;
+      std::cerr << "rbd: error opening " << token_path << std::endl;
+      return r;
+    }
+  }
+
+  char token[1024];
+  memset(token, 0, sizeof(token));
+  r = safe_read(fd, token, sizeof(token) - 1);
+  if (fd != STDIN_FILENO) {
+    VOID_TEMP_FAILURE_RETRY(close(fd));
+  }
+
+  if (r < 0) {
+    std::cerr << "rbd: error reading token file: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  if (vm.count(SITE_NAME)) {
+    r = set_site_name(rados, vm[SITE_NAME].as<std::string>());
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  librbd::RBD rbd;
+  r = rbd.mirror_peer_bootstrap_import(io_ctx, mirror_peer_direction, token);
+  if (r == -ENOSYS) {
+    std::cerr << "rbd: mirroring is not enabled on remote peer" << std::endl;
+    return r;
+  } else if (r < 0) {
+    std::cerr << "rbd: failed to import peer bootstrap token" << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_peer_add_arguments(po::options_description *positional,
+                            po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  positional->add_options()
+    ("remote-cluster-spec", "remote cluster spec\n"
+     "(example: [<client name>@]<cluster name>)");
+  options->add_options()
+    ("remote-client-name", po::value<std::string>(), "remote client name")
+    ("remote-cluster", po::value<std::string>(), "remote cluster name")
+    ("remote-mon-host", po::value<std::string>(), "remote mon host(s)")
+    ("remote-key-file", po::value<std::string>(),
+     "path to file containing remote key");
+}
+
+int execute_peer_add(const po::variables_map &vm,
+                     const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string remote_client_name = g_ceph_context->_conf->name.to_str();
+  std::string remote_cluster;
+  std::map<std::string, std::string> attributes;
+  r = get_remote_cluster_spec(
+    vm, utils::get_positional_argument(vm, arg_index),
+    &remote_client_name, &remote_cluster, &attributes);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO: temporary restriction to prevent adding multiple peers
+  // until rbd-mirror daemon can properly handle the scenario
+  librbd::RBD rbd;
+  std::vector<librbd::mirror_peer_t> mirror_peers;
+  r = rbd.mirror_peer_list(io_ctx, &mirror_peers);
+  if (r < 0) {
+    std::cerr << "rbd: failed to list mirror peers" << std::endl;
+    return r;
+  }
+  if (!mirror_peers.empty()) {
+    std::cerr << "rbd: multiple peers are not currently supported" << std::endl;
+    return -EINVAL;
+  }
+
+  std::string uuid;
+  r = rbd.mirror_peer_add(io_ctx, &uuid, remote_cluster, remote_client_name);
+  if (r < 0) {
+    std::cerr << "rbd: error adding mirror peer" << std::endl;
+    return r;
+  }
+
+  if (!attributes.empty()) {
+    r = set_peer_config_key(io_ctx, uuid, std::move(attributes));
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  std::cout << uuid << std::endl;
+  return 0;
+}
+
+void get_peer_remove_arguments(po::options_description *positional,
+                               po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  add_uuid_option(positional);
+}
+
+int execute_peer_remove(const po::variables_map &vm,
+                        const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string uuid;
+  r = get_uuid(vm, arg_index, &uuid);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.mirror_peer_remove(io_ctx, uuid);
+  if (r < 0) {
+    std::cerr << "rbd: error removing mirror peer" << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_peer_set_arguments(po::options_description *positional,
+                            po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  add_uuid_option(positional);
+  positional->add_options()
+    ("key", "peer parameter [client, cluster, mon-host, key-file]")
+    ("value", "new value for specified key");
+}
+
+int execute_peer_set(const po::variables_map &vm,
+                     const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string uuid;
+  r = get_uuid(vm, arg_index++, &uuid);
+  if (r < 0) {
+    return r;
+  }
+
+  std::set<std::string> valid_keys{{"client", "cluster", "mon-host",
+                                    "key-file"}};
+  std::string key = utils::get_positional_argument(vm, arg_index++);
+  if (valid_keys.find(key) == valid_keys.end()) {
+    std::cerr << "rbd: must specify ";
+    for (auto& valid_key : valid_keys) {
+      std::cerr << "'" << valid_key << "'";
+      if (&valid_key != &(*valid_keys.rbegin())) {
+        std::cerr << ", ";
+      }
+    }
+    std::cerr <<  " key." << std::endl;
+    return -EINVAL;
+  }
+
+  std::string value = utils::get_positional_argument(vm, arg_index++);
+  if (value.empty() && (key == "client" || key == "cluster")) {
+    std::cerr << "rbd: must specify new " << key << " value." << std::endl;
+  } else if (key == "key-file") {
+    key = "key";
+    r = read_key_file(value, &value);
+    if (r < 0) {
+      return r;
+    }
+  } else if (key == "mon-host") {
+    key = "mon_host";
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  if (key == "client") {
+    r = rbd.mirror_peer_set_client(io_ctx, uuid.c_str(), value.c_str());
+  } else if (key == "cluster") {
+    r = rbd.mirror_peer_set_cluster(io_ctx, uuid.c_str(), value.c_str());
+  } else {
+    r = update_peer_config_key(io_ctx, uuid, key, value);
+    if (r  == -ENOENT) {
+      std::cerr << "rbd: mirror peer " << uuid << " does not exist"
+                << std::endl;
+    }
+  }
+
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
+void get_disable_arguments(po::options_description *positional,
+                           po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+}
+
+void get_enable_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  positional->add_options()
+    ("mode", "mirror mode [image or pool]");
+  add_site_name_optional(options);
+}
+
+int execute_enable_disable(librados::IoCtx& io_ctx,
+                           rbd_mirror_mode_t next_mirror_mode,
+                           const std::string &mode, bool ignore_no_update) {
+  librbd::RBD rbd;
+  rbd_mirror_mode_t current_mirror_mode;
+  int r = rbd.mirror_mode_get(io_ctx, &current_mirror_mode);
+  if (r < 0) {
+    std::cerr << "rbd: failed to retrieve mirror mode: "
+              << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  if (current_mirror_mode == next_mirror_mode) {
+    if (!ignore_no_update) {
+      if (mode == "disabled") {
+        std::cout << "rbd: mirroring is already " << mode << std::endl;
+      } else {
+        std::cout << "rbd: mirroring is already configured for "
+                  << mode << " mode" << std::endl;
+      }
+    }
+    return 0;
+  } else if (next_mirror_mode == RBD_MIRROR_MODE_IMAGE &&
+             current_mirror_mode == RBD_MIRROR_MODE_POOL) {
+    std::cout << "note: changing mirroring mode from pool to image"
+              << std::endl;
+  } else if (next_mirror_mode == RBD_MIRROR_MODE_POOL &&
+             current_mirror_mode == RBD_MIRROR_MODE_IMAGE) {
+    std::cout << "note: changing mirroring mode from image to pool"
+              << std::endl;
+  }
+
+  r = rbd.mirror_mode_set(io_ctx, next_mirror_mode);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
+int execute_disable(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+
+  // TODO support namespaces
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  return execute_enable_disable(io_ctx, RBD_MIRROR_MODE_DISABLED, "disabled",
+                                false);
+}
+
+int execute_enable(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  rbd_mirror_mode_t mirror_mode;
+  std::string mode = utils::get_positional_argument(vm, arg_index++);
+  if (mode == "image") {
+    mirror_mode = RBD_MIRROR_MODE_IMAGE;
+  } else if (mode == "pool") {
+    mirror_mode = RBD_MIRROR_MODE_POOL;
+  } else {
+    std::cerr << "rbd: must specify 'image' or 'pool' mode." << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+
+  // TODO support namespaces
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  bool updated = false;
+  if (vm.count(SITE_NAME)) {
+    librbd::RBD rbd;
+
+    auto site_name = vm[SITE_NAME].as<std::string>();
+    std::string original_site_name;
+    r = rbd.mirror_site_name_get(rados, &original_site_name);
+    updated = (r >= 0 && site_name != original_site_name);
+
+    r = set_site_name(rados, site_name);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  return execute_enable_disable(io_ctx, mirror_mode, mode, updated);
+}
+
+void get_info_arguments(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  at::add_format_options(options);
+  options->add_options()
+    (ALL_NAME.c_str(), po::bool_switch(), "list all attributes");
+}
+
+int execute_info(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, false, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  rbd_mirror_mode_t mirror_mode;
+  r = rbd.mirror_mode_get(io_ctx, &mirror_mode);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string site_name;
+  r = rbd.mirror_site_name_get(rados, &site_name);
+  if (r < 0 && r != -EOPNOTSUPP) {
+    return r;
+  }
+
+  std::vector<librbd::mirror_peer_t> mirror_peers;
+  r = rbd.mirror_peer_list(io_ctx, &mirror_peers);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string mirror_mode_desc;
+  switch (mirror_mode) {
+  case RBD_MIRROR_MODE_DISABLED:
+    mirror_mode_desc = "disabled";
+    break;
+  case RBD_MIRROR_MODE_IMAGE:
+    mirror_mode_desc = "image";
+    break;
+  case RBD_MIRROR_MODE_POOL:
+    mirror_mode_desc = "pool";
+    break;
+  default:
+    mirror_mode_desc = "unknown";
+    break;
+  }
+
+  if (formatter != nullptr) {
+    formatter->open_object_section("mirror");
+    formatter->dump_string("mode", mirror_mode_desc);
+  } else {
+    std::cout << "Mode: " << mirror_mode_desc << std::endl;
+  }
+
+  if (mirror_mode != RBD_MIRROR_MODE_DISABLED) {
+    if (formatter != nullptr) {
+      formatter->dump_string("site_name", site_name);
+    } else {
+      std::cout << "Site Name: " << site_name << std::endl;
+    }
+
+    r = format_mirror_peers(io_ctx, formatter, mirror_peers,
+                            vm[ALL_NAME].as<bool>());
+    if (r < 0) {
+      return r;
+    }
+  }
+  if (formatter != nullptr) {
+    formatter->close_section();
+    formatter->flush(std::cout);
+  }
+  return 0;
+}
+
+void get_status_arguments(po::options_description *positional,
+			  po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  at::add_format_options(options);
+  at::add_verbose_option(options);
+}
+
+int execute_status(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, false, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  bool verbose = vm[at::VERBOSE].as<bool>();
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+
+  std::map<librbd::mirror_image_status_state_t, int> states;
+  r = rbd.mirror_image_status_summary(io_ctx, &states);
+  if (r < 0) {
+    std::cerr << "rbd: failed to get status summary for mirrored images: "
+	      << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  if (formatter != nullptr) {
+    formatter->open_object_section("status");
+  }
+
+  enum Health {Ok = 0, Warning = 1, Error = 2} health = Ok;
+  const char *names[] = {"OK", "WARNING", "ERROR"};
+  int total = 0;
+
+  for (auto &it : states) {
+    auto &state = it.first;
+    if (health < Warning &&
+	(state != MIRROR_IMAGE_STATUS_STATE_REPLAYING &&
+	 state != MIRROR_IMAGE_STATUS_STATE_STOPPED)) {
+      health = Warning;
+    }
+    if (health < Error &&
+	state == MIRROR_IMAGE_STATUS_STATE_ERROR) {
+      health = Error;
+    }
+    total += it.second;
+  }
+
+  if (formatter != nullptr) {
+    formatter->open_object_section("summary");
+    formatter->dump_string("health", names[health]);
+    formatter->open_object_section("states");
+    for (auto &it : states) {
+      std::string state_name = utils::mirror_image_status_state(it.first);
+      formatter->dump_int(state_name.c_str(), it.second);
+    }
+    formatter->close_section(); // states
+    formatter->close_section(); // summary
+  } else {
+    std::cout << "health: " << names[health] << std::endl;
+    std::cout << "images: " << total << " total" << std::endl;
+    for (auto &it : states) {
+      std::cout << "    " << it.second << " "
+		<< utils::mirror_image_status_state(it.first) << std::endl;
+    }
+  }
+
+  int ret = 0;
+
+  if (verbose) {
+    if (formatter != nullptr) {
+      formatter->open_array_section("images");
+    }
+
+    std::map<std::string, std::string> instance_ids;
+    MirrorDaemonServiceInfo daemon_service_info(io_ctx);
+
+    std::string start_image_id;
+    while (true) {
+      std::map<std::string, std::string> ids;
+      r = rbd.mirror_image_instance_id_list(io_ctx, start_image_id, 1024, &ids);
+      if (r < 0) {
+        if (r == -EOPNOTSUPP) {
+          std::cerr << "rbd: newer release of Ceph OSDs required to map image "
+                    << "to rbd-mirror daemon instance" << std::endl;
+        } else {
+          std::cerr << "rbd: failed to get instance id list: "
+                    << cpp_strerror(r) << std::endl;
+        }
+        // not fatal
+        break;
+      }
+      if (ids.empty()) {
+        break;
+      }
+      instance_ids.insert(ids.begin(), ids.end());
+      start_image_id = ids.rbegin()->first;
+    }
+
+    if (!instance_ids.empty()) {
+      daemon_service_info.init();
+    }
+
+    ImageRequestGenerator<StatusImageRequest> generator(
+        io_ctx, instance_ids, daemon_service_info, formatter);
+    ret = generator.execute();
+
+    if (formatter != nullptr) {
+      formatter->close_section(); // images
+    }
+  }
+
+  if (formatter != nullptr) {
+    formatter->close_section(); // status
+    formatter->flush(std::cout);
+  }
+
+  return ret;
+}
+
+void get_promote_arguments(po::options_description *positional,
+			   po::options_description *options) {
+  options->add_options()
+    ("force", po::bool_switch(),
+     "promote even if not cleanly demoted by remote cluster");
+  at::add_pool_options(positional, options, false);
+}
+
+int execute_promote(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  utils::disable_cache();
+
+  std::atomic<unsigned> counter = { 0 };
+  ImageRequestGenerator<PromoteImageRequest> generator(io_ctx, &counter,
+                                                       vm["force"].as<bool>());
+  r = generator.execute();
+
+  std::cout << "Promoted " << counter.load() << " mirrored images" << std::endl;
+  return r;
+}
+
+void get_demote_arguments(po::options_description *positional,
+			   po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+}
+
+int execute_demote(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  // TODO support namespaces
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  r = validate_mirroring_enabled(io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  utils::disable_cache();
+
+  std::atomic<unsigned> counter { 0 };
+  ImageRequestGenerator<DemoteImageRequest> generator(io_ctx, &counter);
+  r = generator.execute();
+
+  std::cout << "Demoted " << counter.load() << " mirrored images" << std::endl;
+  return r;
+}
+
+Shell::Action action_bootstrap_create(
+  {"mirror", "pool", "peer", "bootstrap", "create"}, {},
+  "Create a peer bootstrap token to import in a remote cluster", "",
+  &get_peer_bootstrap_create_arguments, &execute_peer_bootstrap_create);
+Shell::Action action_bootstreap_import(
+  {"mirror", "pool", "peer", "bootstrap", "import"}, {},
+  "Import a peer bootstrap token created from a remote cluster", "",
+  &get_peer_bootstrap_import_arguments, &execute_peer_bootstrap_import);
+
+Shell::Action action_add(
+  {"mirror", "pool", "peer", "add"}, {},
+  "Add a mirroring peer to a pool.", "",
+  &get_peer_add_arguments, &execute_peer_add);
+Shell::Action action_remove(
+  {"mirror", "pool", "peer", "remove"}, {},
+  "Remove a mirroring peer from a pool.", "",
+  &get_peer_remove_arguments, &execute_peer_remove);
+Shell::Action action_set(
+  {"mirror", "pool", "peer", "set"}, {},
+  "Update mirroring peer settings.", "",
+  &get_peer_set_arguments, &execute_peer_set);
+
+Shell::Action action_disable(
+  {"mirror", "pool", "disable"}, {},
+  "Disable RBD mirroring by default within a pool.", "",
+  &get_disable_arguments, &execute_disable);
+Shell::Action action_enable(
+  {"mirror", "pool", "enable"}, {},
+  "Enable RBD mirroring by default within a pool.", "",
+  &get_enable_arguments, &execute_enable);
+Shell::Action action_info(
+  {"mirror", "pool", "info"}, {},
+  "Show information about the pool mirroring configuration.", {},
+  &get_info_arguments, &execute_info);
+Shell::Action action_status(
+  {"mirror", "pool", "status"}, {},
+  "Show status for all mirrored images in the pool.", {},
+  &get_status_arguments, &execute_status);
+Shell::Action action_promote(
+  {"mirror", "pool", "promote"}, {},
+  "Promote all non-primary images in the pool.", {},
+  &get_promote_arguments, &execute_promote);
+Shell::Action action_demote(
+  {"mirror", "pool", "demote"}, {},
+  "Demote all primary images in the pool.", {},
+  &get_demote_arguments, &execute_demote);
+
+} // namespace mirror_pool
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Namespace.cc b/src/tools/rbd/action/Namespace.cc
new file mode 100644
index 00000000..746ab40c
--- /dev/null
+++ b/src/tools/rbd/action/Namespace.cc
@@ -0,0 +1,191 @@
+
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include "include/stringify.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include <algorithm>
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace ns {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+void get_create_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_pool_options(positional, options, true);
+}
+
+int execute_create(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  std::string namespace_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              &namespace_name, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  if (namespace_name.empty()) {
+    std::cerr << "rbd: namespace name was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.namespace_create(io_ctx, namespace_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: failed to created namespace: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_remove_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_pool_options(positional, options, true);
+}
+
+int execute_remove(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  std::string namespace_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              &namespace_name, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  if (namespace_name.empty()) {
+    std::cerr << "rbd: namespace name was not specified" << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.namespace_remove(io_ctx, namespace_name.c_str());
+  if (r == -EBUSY) {
+    std::cerr << "rbd: namespace contains images which must be deleted first."
+              << std::endl;
+    return r;
+  } else if (r == -ENOENT) {
+    std::cerr << "rbd: namespace does not exist." << std::endl;
+    return r;
+  } else if (r < 0) {
+    std::cerr << "rbd: failed to remove namespace: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_list_arguments(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  at::add_format_options(options);
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  std::vector<std::string> names;
+  r = rbd.namespace_list(io_ctx, &names);
+  if (r < 0 && r != -ENOENT) {
+    std::cerr << "rbd: failed to list namespaces: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  std::sort(names.begin(), names.end());
+
+  TextTable tbl;
+  if (formatter) {
+    formatter->open_array_section("namespaces");
+  } else {
+    tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  for (auto& name : names) {
+    if (formatter) {
+      formatter->open_object_section("namespace");
+      formatter->dump_string("name", name);
+      formatter->close_section();
+    } else {
+      tbl << name << TextTable::endrow;
+    }
+  }
+
+  if (formatter) {
+    formatter->close_section();
+    formatter->flush(std::cout);
+  } else if (!names.empty()) {
+    std::cout << tbl;
+  }
+
+  return 0;
+}
+
+Shell::Action action_create(
+  {"namespace", "create"}, {},
+   "Create an RBD image namespace.", "",
+  &get_create_arguments, &execute_create);
+
+Shell::Action action_remove(
+  {"namespace", "remove"}, {"namespace", "rm"},
+   "Remove an RBD image namespace.", "",
+  &get_remove_arguments, &execute_remove);
+
+Shell::Action action_list(
+  {"namespace", "list"}, {"namespace", "ls"}, "List RBD image namespaces.", "",
+  &get_list_arguments, &execute_list);
+
+} // namespace ns
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Nbd.cc b/src/tools/rbd/action/Nbd.cc
new file mode 100644
index 00000000..5c55adea
--- /dev/null
+++ b/src/tools/rbd/action/Nbd.cc
@@ -0,0 +1,286 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/stringify.h"
+#include "common/SubProcess.h"
+#include <iostream>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace nbd {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int call_nbd_cmd(const po::variables_map &vm,
+                        const std::vector<std::string> &args,
+                        const std::vector<std::string> &ceph_global_init_args) {
+  char exe_path[PATH_MAX];
+  ssize_t exe_path_bytes = readlink("/proc/self/exe", exe_path,
+				    sizeof(exe_path) - 1);
+  if (exe_path_bytes < 0) {
+    strcpy(exe_path, "rbd-nbd");
+  } else {
+    if (snprintf(exe_path + exe_path_bytes,
+                 sizeof(exe_path) - exe_path_bytes,
+                 "-nbd") < 0) {
+      return -EOVERFLOW;
+    }
+  }
+
+  SubProcess process(exe_path, SubProcess::KEEP, SubProcess::KEEP, SubProcess::KEEP);
+
+  for (auto &arg : ceph_global_init_args) {
+    process.add_cmd_arg(arg.c_str());
+  }
+
+  for (auto &arg : args) {
+    process.add_cmd_arg(arg.c_str());
+  }
+
+  if (process.spawn()) {
+    std::cerr << "rbd: failed to run rbd-nbd: " << process.err() << std::endl;
+    return -EINVAL;
+  } else if (process.join()) {
+    std::cerr << "rbd: rbd-nbd failed with error: " << process.err() << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+int get_image_or_snap_spec(const po::variables_map &vm, std::string *spec) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string nspace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &nspace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  spec->append(pool_name);
+  spec->append("/");
+  if (!nspace_name.empty()) {
+    spec->append(nspace_name);
+    spec->append("/");
+  }
+  spec->append(image_name);
+  if (!snap_name.empty()) {
+    spec->append("@");
+    spec->append(snap_name);
+  }
+
+  return 0;
+}
+
+int parse_options(const std::vector<std::string> &options,
+                  std::vector<std::string> *args) {
+  for (auto &opts : options) {
+    std::vector<std::string> args_;
+    boost::split(args_, opts, boost::is_any_of(","));
+    for (auto &o : args_) {
+      args->push_back("--" + o);
+    }
+  }
+
+  return 0;
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+#if defined(__FreeBSD__)
+  std::cerr << "rbd: nbd device is not supported" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+  std::vector<std::string> args;
+
+  args.push_back("list-mapped");
+
+  if (vm.count("format")) {
+    args.push_back("--format");
+    args.push_back(vm["format"].as<at::Format>().value);
+  }
+  if (vm["pretty-format"].as<bool>()) {
+    args.push_back("--pretty-format");
+  }
+
+  return call_nbd_cmd(vm, args, ceph_global_init_args);
+}
+
+int execute_map(const po::variables_map &vm,
+                const std::vector<std::string> &ceph_global_init_args) {
+#if defined(__FreeBSD__)
+  std::cerr << "rbd: nbd device is not supported" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+  std::vector<std::string> args;
+
+  args.push_back("map");
+  std::string img;
+  int r = get_image_or_snap_spec(vm, &img);
+  if (r < 0) {
+    return r;
+  }
+  args.push_back(img);
+
+  if (vm["read-only"].as<bool>()) {
+    args.push_back("--read-only");
+  }
+
+  if (vm["exclusive"].as<bool>()) {
+    args.push_back("--exclusive");
+  }
+
+  if (vm.count("options")) {
+    r = parse_options(vm["options"].as<std::vector<std::string>>(), &args);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  return call_nbd_cmd(vm, args, ceph_global_init_args);
+}
+
+int execute_unmap(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+#if defined(__FreeBSD__)
+  std::cerr << "rbd: nbd device is not supported" << std::endl;
+  return -EOPNOTSUPP;
+#endif
+  std::string device_name = utils::get_positional_argument(vm, 0);
+  if (!boost::starts_with(device_name, "/dev/")) {
+    device_name.clear();
+  }
+
+  std::string image_name;
+  if (device_name.empty()) {
+    int r = get_image_or_snap_spec(vm, &image_name);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  if (device_name.empty() && image_name.empty()) {
+    std::cerr << "rbd: unmap requires either image name or device path"
+              << std::endl;
+    return -EINVAL;
+  }
+
+  std::vector<std::string> args;
+
+  args.push_back("unmap");
+  args.push_back(device_name.empty() ? image_name : device_name);
+
+  if (vm.count("options")) {
+    int r = parse_options(vm["options"].as<std::vector<std::string>>(), &args);
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  return call_nbd_cmd(vm, args, ceph_global_init_args);
+}
+
+void get_list_arguments_deprecated(po::options_description *positional,
+                                   po::options_description *options) {
+  at::add_format_options(options);
+}
+
+int execute_list_deprecated(const po::variables_map &vm,
+                            const std::vector<std::string> &ceph_global_args) {
+  std::cerr << "rbd: 'nbd list' command is deprecated, "
+            << "use 'device list -t nbd' instead" << std::endl;
+  return execute_list(vm, ceph_global_args);
+}
+
+void get_map_arguments_deprecated(po::options_description *positional,
+                                  po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_NONE);
+  options->add_options()
+    ("read-only", po::bool_switch(), "map read-only")
+    ("exclusive", po::bool_switch(), "forbid writes by other clients")
+    ("device", po::value<std::string>(), "specify nbd device")
+    ("nbds_max", po::value<std::string>(), "override module param nbds_max")
+    ("max_part", po::value<std::string>(), "override module param max_part")
+    ("timeout", po::value<std::string>(), "set nbd request timeout (seconds)");
+}
+
+int execute_map_deprecated(const po::variables_map &vm_deprecated,
+                           const std::vector<std::string> &ceph_global_args) {
+  std::cerr << "rbd: 'nbd map' command is deprecated, "
+            << "use 'device map -t nbd' instead" << std::endl;
+
+  po::options_description options;
+  options.add_options()
+    ("options,o", po::value<std::vector<std::string>>()
+                  ->default_value(std::vector<std::string>(), ""), "");
+
+  po::variables_map vm = vm_deprecated;
+  po::store(po::command_line_parser({}).options(options).run(), vm);
+
+  std::vector<std::string> opts;
+  if (vm_deprecated.count("device")) {
+    opts.push_back("device=" + vm_deprecated["device"].as<std::string>());
+  }
+  if (vm_deprecated.count("nbds_max")) {
+    opts.push_back("nbds_max=" + vm_deprecated["nbds_max"].as<std::string>());
+  }
+  if (vm_deprecated.count("max_part")) {
+    opts.push_back("max_part=" + vm_deprecated["max_part"].as<std::string>());
+  }
+  if (vm_deprecated.count("timeout")) {
+    opts.push_back("timeout=" + vm_deprecated["timeout"].as<std::string>());
+  }
+
+  vm.at("options").value() = boost::any(opts);
+
+  return execute_map(vm, ceph_global_args);
+}
+
+void get_unmap_arguments_deprecated(po::options_description *positional,
+                                    po::options_description *options) {
+  positional->add_options()
+    ("image-or-snap-or-device-spec",
+     "image, snapshot, or device specification\n"
+     "[<pool-name>/]<image-name>[@<snapshot-name>] or <device-path>");
+  at::add_pool_option(options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_snap_option(options, at::ARGUMENT_MODIFIER_NONE);
+}
+
+int execute_unmap_deprecated(const po::variables_map &vm,
+                             const std::vector<std::string> &ceph_global_args) {
+  std::cerr << "rbd: 'nbd unmap' command is deprecated, "
+            << "use 'device unmap -t nbd' instead" << std::endl;
+  return execute_unmap(vm, ceph_global_args);
+}
+
+Shell::SwitchArguments switched_arguments({"read-only", "exclusive"});
+
+Shell::Action action_show_deprecated(
+  {"nbd", "list"}, {"nbd", "ls"}, "List the nbd devices already used.", "",
+  &get_list_arguments_deprecated, &execute_list_deprecated, false);
+
+Shell::Action action_map_deprecated(
+  {"nbd", "map"}, {}, "Map image to a nbd device.", "",
+  &get_map_arguments_deprecated, &execute_map_deprecated, false);
+
+Shell::Action action_unmap_deprecated(
+  {"nbd", "unmap"}, {}, "Unmap a nbd device.", "",
+  &get_unmap_arguments_deprecated, &execute_unmap_deprecated, false);
+
+} // namespace nbd
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/ObjectMap.cc b/src/tools/rbd/action/ObjectMap.cc
new file mode 100644
index 00000000..40ee2d47
--- /dev/null
+++ b/src/tools/rbd/action/ObjectMap.cc
@@ -0,0 +1,131 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace object_map {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_object_map_rebuild(librbd::Image &image, bool no_progress)
+{
+  utils::ProgressContext pc("Object Map Rebuild", no_progress);
+  int r = image.rebuild_object_map(pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+void get_rebuild_arguments(po::options_description *positional,
+			   po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+                                     at::ARGUMENT_MODIFIER_NONE);
+  at::add_no_progress_option(options);
+}
+
+int execute_rebuild(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "",
+                                 snap_name, false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_object_map_rebuild(image, vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: rebuilding object map failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+static int do_object_map_check(librbd::Image &image, bool no_progress)
+{
+  utils::ProgressContext pc("Object Map Check", no_progress);
+  int r = image.check_object_map(pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+void get_check_arguments(po::options_description *positional,
+		   po::options_description *options) {
+  at::add_image_or_snap_spec_options(positional, options,
+				     at::ARGUMENT_MODIFIER_NONE);
+  at::add_no_progress_option(options);
+}
+
+int execute_check(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_PERMITTED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "",
+                                 snap_name, false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_object_map_check(image, vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: checking object map failed: " << cpp_strerror(r)
+	      << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action_rebuild(
+  {"object-map", "rebuild"}, {}, "Rebuild an invalid object map.", "",
+  &get_rebuild_arguments, &execute_rebuild);
+Shell::Action action_check(
+  {"object-map", "check"}, {}, "Verify the object map is correct.", "",
+  &get_check_arguments, &execute_check);
+
+} // namespace object_map
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Perf.cc b/src/tools/rbd/action/Perf.cc
new file mode 100644
index 00000000..8f76e85f
--- /dev/null
+++ b/src/tools/rbd/action/Perf.cc
@@ -0,0 +1,699 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/stringify.h"
+#include "common/ceph_context.h"
+#include "common/ceph_json.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include "global/global_context.h"
+#include <ncurses.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <iostream>
+#include <vector>
+#include <boost/algorithm/string.hpp>
+#include <boost/assign.hpp>
+#include <boost/bimap.hpp>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace perf {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+namespace {
+
+enum class StatDescriptor {
+  WRITE_OPS = 0,
+  READ_OPS,
+  WRITE_BYTES,
+  READ_BYTES,
+  WRITE_LATENCY,
+  READ_LATENCY
+};
+
+typedef boost::bimap<StatDescriptor, std::string> StatDescriptors;
+
+static const StatDescriptors STAT_DESCRIPTORS =
+  boost::assign::list_of<StatDescriptors::relation>
+    (StatDescriptor::WRITE_OPS, "write_ops")
+    (StatDescriptor::READ_OPS, "read_ops")
+    (StatDescriptor::WRITE_BYTES, "write_bytes")
+    (StatDescriptor::READ_BYTES, "read_bytes")
+    (StatDescriptor::WRITE_LATENCY, "write_latency")
+    (StatDescriptor::READ_LATENCY, "read_latency");
+
+std::ostream& operator<<(std::ostream& os, const StatDescriptor& val) {
+  auto it = STAT_DESCRIPTORS.left.find(val);
+  if (it == STAT_DESCRIPTORS.left.end()) {
+    os << "unknown (" << static_cast<int>(val) << ")";
+  } else {
+    os << it->second;
+  }
+  return os;
+}
+
+void validate(boost::any& v, const std::vector<std::string>& values,
+              StatDescriptor *target_type, int) {
+  po::validators::check_first_occurrence(v);
+  std::string s = po::validators::get_single_string(values);
+  boost::replace_all(s, "_", " ");
+  boost::replace_all(s, "-", "_");
+
+  auto it = STAT_DESCRIPTORS.right.find(s);
+  if (it == STAT_DESCRIPTORS.right.end()) {
+    throw po::validation_error(po::validation_error::invalid_option_value);
+  }
+  v = boost::any(it->second);
+}
+
+struct ImageStat {
+  ImageStat(const std::string& pool_name, const std::string& pool_namespace,
+            const std::string& image_name)
+    : pool_name(pool_name), pool_namespace(pool_namespace),
+      image_name(image_name) {
+    stats.resize(STAT_DESCRIPTORS.size());
+  }
+
+  std::string pool_name;
+  std::string pool_namespace;
+  std::string image_name;
+  std::vector<double> stats;
+};
+
+typedef std::vector<ImageStat> ImageStats;
+
+typedef std::pair<std::string, std::string> SpecPair;
+
+std::string format_pool_spec(const std::string& pool,
+                             const std::string& pool_namespace) {
+  std::string pool_spec{pool};
+  if (!pool_namespace.empty()) {
+    pool_spec += "/" + pool_namespace;
+  }
+  return pool_spec;
+}
+
+int query_iostats(librados::Rados& rados, const std::string& pool_spec,
+                  StatDescriptor sort_by, ImageStats* image_stats,
+                  std::ostream& err_os) {
+  auto sort_by_str = STAT_DESCRIPTORS.left.find(sort_by)->second;
+
+  std::string cmd = R"(
+    {
+      "prefix": "rbd perf image stats",
+      "pool_spec": ")" + pool_spec + R"(",
+      "sort_by": ")" + sort_by_str + R"(",
+      "format": "json"
+    }")";
+
+  bufferlist in_bl;
+  bufferlist out_bl;
+  std::string outs;
+  int r = rados.mgr_command(cmd, in_bl, &out_bl, &outs);
+  if (r == -EOPNOTSUPP) {
+    err_os << "rbd: 'rbd_support' mgr module is not enabled."
+           << std::endl << std::endl
+           << "Use 'ceph mgr module enable rbd_support' to enable."
+           << std::endl;
+    return r;
+  } else if (r < 0) {
+    err_os << "rbd: mgr command failed: " << cpp_strerror(r);
+    if (!outs.empty()) {
+      err_os << ": " << outs;
+    }
+    err_os << std::endl;
+    return r;
+  }
+
+  json_spirit::mValue json_root;
+  if (!json_spirit::read(out_bl.to_str(), json_root)) {
+    err_os << "rbd: error parsing perf stats" << std::endl;
+    return -EINVAL;
+  }
+
+  image_stats->clear();
+  try {
+    auto& root = json_root.get_obj();
+
+    // map JSON stat descriptor order to our internal order
+    std::map<uint32_t, uint32_t> json_to_internal_stats;
+    auto& json_stat_descriptors = root["stat_descriptors"].get_array();
+    for (size_t idx = 0; idx < json_stat_descriptors.size(); ++idx) {
+      auto it = STAT_DESCRIPTORS.right.find(
+        json_stat_descriptors[idx].get_str());
+      if (it == STAT_DESCRIPTORS.right.end()) {
+        continue;
+      }
+      json_to_internal_stats[idx] = static_cast<uint32_t>(it->second);
+    }
+
+    // cache a mapping from pool descriptors back to pool-specs
+    std::map<std::string, SpecPair> json_to_internal_pools;
+    auto& pool_descriptors = root["pool_descriptors"].get_obj();
+    for (auto& pool : pool_descriptors) {
+      auto& pool_spec = pool.second.get_str();
+      auto pos = pool_spec.rfind("/");
+
+      SpecPair pair{pool_spec.substr(0, pos), ""};
+      if (pos != std::string::npos) {
+        pair.second = pool_spec.substr(pos + 1);
+      }
+
+      json_to_internal_pools[pool.first] = pair;
+    }
+
+    auto& stats = root["stats"].get_array();
+    for (auto& stat : stats) {
+      auto& stat_obj = stat.get_obj();
+      if (!stat_obj.empty()) {
+        auto& image_spec = stat_obj.begin()->first;
+
+        auto pos = image_spec.find("/");
+        SpecPair pair{image_spec.substr(0, pos), ""};
+        if (pos != std::string::npos) {
+          pair.second = image_spec.substr(pos + 1);
+        }
+
+        const auto pool_it = json_to_internal_pools.find(pair.first);
+        if (pool_it == json_to_internal_pools.end()) {
+          continue;
+        }
+
+        image_stats->emplace_back(
+          pool_it->second.first, pool_it->second.second, pair.second);
+
+        auto& image_stat = image_stats->back();
+        auto& data = stat_obj.begin()->second.get_array();
+        for (auto& indexes : json_to_internal_stats) {
+          image_stat.stats[indexes.second] = data[indexes.first].get_real();
+        }
+      }
+    }
+  } catch (std::runtime_error &e) {
+    err_os << "rbd: error parsing perf stats: " << e.what() << std::endl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
+void format_stat(StatDescriptor stat_descriptor, double stat,
+                 std::ostream& os) {
+  switch (stat_descriptor) {
+  case StatDescriptor::WRITE_OPS:
+  case StatDescriptor::READ_OPS:
+    os << si_u_t(stat) << "/s";
+    break;
+  case StatDescriptor::WRITE_BYTES:
+  case StatDescriptor::READ_BYTES:
+    os << byte_u_t(stat) << "/s";
+    break;
+  case StatDescriptor::WRITE_LATENCY:
+  case StatDescriptor::READ_LATENCY:
+    os << std::fixed << std::setprecision(2);
+    if (stat >= 1000000000) {
+      os << (stat / 1000000000) << " s";
+    } else if (stat >= 1000000) {
+      os << (stat / 1000000) << " ms";
+    } else if (stat >= 1000) {
+      os << (stat / 1000) << " us";
+    } else {
+      os << stat << " ns";
+    }
+    break;
+  default:
+    ceph_assert(false);
+    break;
+  }
+}
+
+} // anonymous namespace
+
+namespace iostat {
+
+struct Iterations {};
+
+void validate(boost::any& v, const std::vector<std::string>& values,
+              Iterations *target_type, int) {
+  po::validators::check_first_occurrence(v);
+  auto& s = po::validators::get_single_string(values);
+
+  try {
+    auto iterations = boost::lexical_cast<uint32_t>(s);
+    if (iterations > 0) {
+      v = boost::any(iterations);
+      return;
+    }
+  } catch (const boost::bad_lexical_cast &) {
+  }
+  throw po::validation_error(po::validation_error::invalid_option_value);
+}
+
+void format(const ImageStats& image_stats, Formatter* f, bool global_search) {
+  TextTable tbl;
+  if (f) {
+    f->open_array_section("images");
+  } else {
+    tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+    for (auto& stat : STAT_DESCRIPTORS.left) {
+      std::string title;
+      switch (stat.first) {
+      case StatDescriptor::WRITE_OPS:
+        title = "WR ";
+        break;
+      case StatDescriptor::READ_OPS:
+        title = "RD ";
+        break;
+      case StatDescriptor::WRITE_BYTES:
+        title = "WR_BYTES ";
+        break;
+      case StatDescriptor::READ_BYTES:
+        title = "RD_BYTES ";
+        break;
+      case StatDescriptor::WRITE_LATENCY:
+        title = "WR_LAT ";
+        break;
+      case StatDescriptor::READ_LATENCY:
+        title = "RD_LAT ";
+        break;
+      default:
+        ceph_assert(false);
+        break;
+      }
+      tbl.define_column(title, TextTable::RIGHT, TextTable::RIGHT);
+    }
+  }
+
+  for (auto& image_stat : image_stats) {
+    if (f)  {
+      f->open_object_section("image");
+      f->dump_string("pool", image_stat.pool_name);
+      f->dump_string("pool_namespace", image_stat.pool_namespace);
+      f->dump_string("image", image_stat.image_name);
+      for (auto& pair : STAT_DESCRIPTORS.left) {
+        f->dump_float(pair.second.c_str(),
+                      image_stat.stats[static_cast<size_t>(pair.first)]);
+      }
+      f->close_section();
+    } else {
+      std::string name;
+      if (global_search) {
+        name += image_stat.pool_name + "/";
+        if (!image_stat.pool_namespace.empty()) {
+          name += image_stat.pool_namespace + "/";
+        }
+      }
+      name += image_stat.image_name;
+
+      tbl << name;
+      for (auto& pair : STAT_DESCRIPTORS.left) {
+        std::stringstream str;
+        format_stat(pair.first,
+                    image_stat.stats[static_cast<size_t>(pair.first)], str);
+        str << ' ';
+        tbl << str.str();
+      }
+      tbl << TextTable::endrow;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  } else {
+    std::cout << tbl << std::endl;
+  }
+}
+
+} // namespace iostat
+
+namespace iotop {
+
+class MainWindow {
+public:
+  MainWindow(librados::Rados& rados, const std::string& pool_spec)
+  : m_rados(rados), m_pool_spec(pool_spec) {
+    initscr();
+    curs_set(0);
+    cbreak();
+    noecho();
+    keypad(stdscr, TRUE);
+    nodelay(stdscr, TRUE);
+
+    init_columns();
+  }
+
+  int run() {
+    redraw();
+
+    int r = 0;
+    std::stringstream err_str;
+    while (true) {
+      r = query_iostats(m_rados, m_pool_spec, m_sort_by, &m_image_stats,
+                        err_str);
+      if (r < 0) {
+        break;
+        return r;
+      }
+
+      redraw();
+      wait_for_key_or_delay();
+
+      int ch = getch();
+      if (ch == 'q' || ch == 'Q') {
+        break;
+      } else if (ch == '<' || ch == KEY_LEFT) {
+        auto it = STAT_DESCRIPTORS.left.find(m_sort_by);
+        if (it != STAT_DESCRIPTORS.left.begin()) {
+          m_sort_by = (--it)->first;
+        }
+      } else if (ch == '>' || ch == KEY_RIGHT) {
+        auto it = STAT_DESCRIPTORS.left.find(m_sort_by);
+        if (it != STAT_DESCRIPTORS.left.end() &&
+            ++it != STAT_DESCRIPTORS.left.end()) {
+          m_sort_by = it->first;
+        }
+      }
+    }
+
+    endwin();
+
+    if (r < 0) {
+      std::cerr << err_str.str() << std::endl;
+    }
+    return r;
+  }
+
+private:
+  static const size_t STAT_COLUMN_WIDTH = 12;
+
+  librados::Rados& m_rados;
+  std::string m_pool_spec;
+
+  ImageStats m_image_stats;
+  StatDescriptor m_sort_by = StatDescriptor::WRITE_OPS;
+
+  bool m_pending_win_opened = false;
+  WINDOW* m_pending_win = nullptr;
+
+  int m_height = 1;
+  int m_width = 1;
+
+  std::map<StatDescriptor, std::string> m_columns;
+
+  void init_columns() {
+    m_columns.clear();
+    for (auto& pair : STAT_DESCRIPTORS.left) {
+      std::string title;
+      switch (pair.first) {
+      case StatDescriptor::WRITE_OPS:
+        title = "WRITES OPS";
+        break;
+      case StatDescriptor::READ_OPS:
+        title = "READS OPS";
+        break;
+      case StatDescriptor::WRITE_BYTES:
+        title = "WRITE BYTES";
+        break;
+      case StatDescriptor::READ_BYTES:
+        title = "READ BYTES";
+        break;
+      case StatDescriptor::WRITE_LATENCY:
+        title = "WRITE LAT";
+        break;
+      case StatDescriptor::READ_LATENCY:
+        title = "READ LAT";
+        break;
+      default:
+        ceph_assert(false);
+        break;
+      }
+      m_columns[pair.first] = (title);
+    }
+  }
+
+  void redraw() {
+    getmaxyx(stdscr, m_height, m_width);
+
+    redraw_main_window();
+    redraw_pending_window();
+
+    doupdate();
+  }
+
+  void redraw_main_window() {
+    werase(stdscr);
+    mvhline(0, 0, ' ' | A_REVERSE, m_width);
+
+    // print header for all metrics
+    int remaining_cols = m_width;
+    std::stringstream str;
+    for (auto& pair : m_columns) {
+      int attr = A_REVERSE;
+      std::string title;
+      if (pair.first == m_sort_by) {
+        title += '>';
+        attr |= A_BOLD;
+      } else {
+        title += ' ';
+      }
+      title += pair.second;
+
+      str.str("");
+      str << std::right << std::setfill(' ')
+         << std::setw(STAT_COLUMN_WIDTH)
+         << title << ' ';
+
+      attrset(attr);
+      addstr(str.str().c_str());
+      remaining_cols -= title.size();
+    }
+
+    attrset(A_REVERSE);
+    addstr("IMAGE");
+    attrset(A_NORMAL);
+
+    // print each image (one per line)
+    int row = 1;
+    int remaining_lines = m_height - 1;
+    for (auto& image_stat : m_image_stats) {
+      if (remaining_lines <= 0) {
+        break;
+      }
+      --remaining_lines;
+
+      move(row++, 0);
+      for (auto& pair : m_columns) {
+        str.str("");
+        format_stat(pair.first,
+                    image_stat.stats[static_cast<size_t>(pair.first)], str);
+        auto value = str.str().substr(0, STAT_COLUMN_WIDTH);
+
+        str.str("");
+        str << std::right << std::setfill(' ')
+            << std::setw(STAT_COLUMN_WIDTH)
+            << value << ' ';
+        addstr(str.str().c_str());
+      }
+
+      std::string image;
+      if (m_pool_spec.empty()) {
+        image = format_pool_spec(image_stat.pool_name,
+                                 image_stat.pool_namespace) + "/";
+      }
+      image += image_stat.image_name;
+      addstr(image.substr(0, remaining_cols).c_str());
+    }
+
+    wnoutrefresh(stdscr);
+  }
+
+  void redraw_pending_window() {
+    // draw a "please by patient" window while waiting
+    const char* msg = "Waiting for initial stats";
+    int height = 5;
+    int width = strlen(msg) + 4;;
+    int starty = (m_height - height) / 2;
+    int startx = (m_width - width) / 2;
+
+    if (m_image_stats.empty() && !m_pending_win_opened) {
+      m_pending_win_opened = true;
+      m_pending_win = newwin(height, width, starty, startx);
+    }
+
+    if (m_pending_win != nullptr) {
+      if (m_image_stats.empty()) {
+        box(m_pending_win, 0 , 0);
+        mvwaddstr(m_pending_win, 2, 2, msg);
+        wnoutrefresh(m_pending_win);
+      } else {
+        delwin(m_pending_win);
+        m_pending_win = nullptr;
+      }
+    }
+  }
+
+  void wait_for_key_or_delay() {
+    fd_set fds;
+    FD_ZERO(&fds);
+    FD_SET(STDIN_FILENO, &fds);
+
+    // no point to refreshing faster than the stats period
+    struct timeval tval;
+    tval.tv_sec = std::min<uint32_t>(
+      10, g_conf().get_val<int64_t>("mgr_stats_period"));
+    tval.tv_usec = 0;
+
+    select(STDIN_FILENO + 1, &fds, NULL, NULL, &tval);
+  }
+};
+
+} // namespace iotop
+
+
+void get_arguments_iostat(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_pool_options(positional, options, true);
+  options->add_options()
+    ("iterations", po::value<iostat::Iterations>(),
+     "iterations of metric collection [> 0]")
+    ("sort-by", po::value<StatDescriptor>()->default_value(StatDescriptor::WRITE_OPS),
+     "sort-by IO metric "
+     "(write-ops, read-ops, write-bytes, read-bytes, write-latency, read-latency) "
+     "[default: write-ops]");
+  at::add_format_options(options);
+}
+
+int execute_iostat(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool;
+  std::string pool_namespace;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, false, false, &pool,
+                                              &pool_namespace, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  uint32_t iterations = 0;
+  if (vm.count("iterations")) {
+    iterations = vm["iterations"].as<uint32_t>();
+  }
+  auto sort_by = vm["sort-by"].as<StatDescriptor>();
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  auto f = formatter.get();
+  if (iterations > 1 && f != nullptr) {
+    std::cerr << "rbd: specifing iterations is not valid with formatted output"
+              << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  r = utils::init_rados(&rados);
+  if (r < 0) {
+    return r;
+  }
+
+  r = rados.wait_for_latest_osdmap();
+  if (r < 0) {
+    std::cerr << "rbd: failed to retrieve OSD map" << std::endl;
+    return r;
+  }
+
+  std::string pool_spec = format_pool_spec(pool, pool_namespace);
+
+  // no point to refreshing faster than the stats period
+  auto delay = std::min<uint32_t>(10, g_conf().get_val<int64_t>("mgr_stats_period"));
+
+  ImageStats image_stats;
+  uint32_t count = 0;
+  bool printed_notice = false;
+  while (count++ < iterations || iterations == 0) {
+    r = query_iostats(rados, pool_spec, sort_by, &image_stats, std::cerr);
+    if (r < 0) {
+      return r;
+    }
+
+    if (count == 1 && image_stats.empty()) {
+      count = 0;
+      if (!printed_notice) {
+        std::cerr << "rbd: waiting for initial image stats"
+                  << std::endl << std::endl;;
+        printed_notice = true;
+      }
+    } else {
+      iostat::format(image_stats, f, pool_spec.empty());
+      if (f != nullptr) {
+        break;
+      }
+    }
+
+    sleep(delay);
+  }
+
+  return 0;
+}
+
+void get_arguments_iotop(po::options_description *positional,
+                         po::options_description *options) {
+  at::add_pool_options(positional, options, true);
+}
+
+int execute_iotop(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool;
+  std::string pool_namespace;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, false, false, &pool,
+                                              &pool_namespace, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  r = utils::init_rados(&rados);
+  if (r < 0) {
+    return r;
+  }
+
+  r = rados.wait_for_latest_osdmap();
+  if (r < 0) {
+    std::cerr << "rbd: failed to retrieve OSD map" << std::endl;
+    return r;
+  }
+
+  iotop::MainWindow mainWindow(rados, format_pool_spec(pool, pool_namespace));
+  r = mainWindow.run();
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+Shell::Action stat_action(
+  {"perf", "image", "iostat"}, {}, "Display image IO statistics.", "",
+  &get_arguments_iostat, &execute_iostat);
+Shell::Action top_action(
+  {"perf", "image", "iotop"}, {}, "Display a top-like IO monitor.", "",
+  &get_arguments_iotop, &execute_iotop);
+
+} // namespace perf
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Pool.cc b/src/tools/rbd/action/Pool.cc
new file mode 100644
index 00000000..f1718eb1
--- /dev/null
+++ b/src/tools/rbd/action/Pool.cc
@@ -0,0 +1,162 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/stringify.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace pool {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+void get_arguments_init(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_pool_options(positional, options, false);
+  options->add_options()
+      ("force", po::bool_switch(),
+       "force initialize pool for RBD use if registered by another application");
+}
+
+int execute_init(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, false, &pool_name,
+                                              nullptr, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, "", &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.pool_init(io_ctx, vm["force"].as<bool>());
+  if (r == -EOPNOTSUPP) {
+    std::cerr << "rbd: luminous or later release required." << std::endl;
+  } else if (r == -EPERM) {
+    std::cerr << "rbd: pool already registered to a different application."
+              << std::endl;
+  } else if (r < 0) {
+    std::cerr << "rbd: error registered application: " << cpp_strerror(r)
+              << std::endl;
+  }
+
+  return 0;
+}
+
+void get_arguments_stats(po::options_description *positional,
+                         po::options_description *options) {
+  at::add_pool_options(positional, options, true);
+  at::add_format_options(options);
+}
+
+int execute_stats(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  std::string namespace_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, false, &pool_name,
+                                              &namespace_name, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  uint64_t image_count;
+  uint64_t provisioned_bytes;
+  uint64_t snap_count;
+  uint64_t trash_count;
+  uint64_t trash_provisioned_bytes;
+  uint64_t trash_snap_count;
+
+  librbd::PoolStats pool_stats;
+  pool_stats.add(RBD_POOL_STAT_OPTION_IMAGES, &image_count);
+  pool_stats.add(RBD_POOL_STAT_OPTION_IMAGE_MAX_PROVISIONED_BYTES,
+                 &provisioned_bytes);
+  pool_stats.add(RBD_POOL_STAT_OPTION_IMAGE_SNAPSHOTS, &snap_count);
+  pool_stats.add(RBD_POOL_STAT_OPTION_TRASH_IMAGES, &trash_count);
+  pool_stats.add(RBD_POOL_STAT_OPTION_TRASH_MAX_PROVISIONED_BYTES,
+                 &trash_provisioned_bytes);
+  pool_stats.add(RBD_POOL_STAT_OPTION_TRASH_SNAPSHOTS, &trash_snap_count);
+
+  r = rbd.pool_stats_get(io_ctx, &pool_stats);
+  if (r < 0) {
+    std::cerr << "rbd: failed to query pool stats: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+
+  if (formatter) {
+    formatter->open_object_section("stats");
+    formatter->open_object_section("images");
+    formatter->dump_unsigned("count", image_count);
+    formatter->dump_unsigned("provisioned_bytes", provisioned_bytes);
+    formatter->dump_unsigned("snap_count", snap_count);
+    formatter->close_section();
+    formatter->open_object_section("trash");
+    formatter->dump_unsigned("count", trash_count);
+    formatter->dump_unsigned("provisioned_bytes", trash_provisioned_bytes);
+    formatter->dump_unsigned("snap_count", trash_snap_count);
+    formatter->close_section();
+    formatter->close_section();
+    formatter->flush(std::cout);
+  } else {
+    std::cout << "Total Images: " << image_count;
+    if (trash_count > 0) {
+      std::cout << " (" << trash_count << " in trash)";
+    }
+    std::cout << std::endl;
+
+    std::cout << "Total Snapshots: " << snap_count;
+    if (trash_count > 0) {
+      std::cout << " (" << trash_snap_count << " in trash)";
+    }
+    std::cout << std::endl;
+
+    std::cout << "Provisioned Size: " << byte_u_t(provisioned_bytes);
+    if (trash_count > 0) {
+      std::cout << " (" << byte_u_t(trash_provisioned_bytes) << " in trash)";
+    }
+    std::cout << std::endl;
+  }
+
+  return 0;
+}
+
+Shell::Action init_action(
+  {"pool", "init"}, {}, "Initialize pool for use by RBD.", "",
+  &get_arguments_init, &execute_init);
+Shell::Action stat_action(
+  {"pool", "stats"}, {}, "Display pool statistics.",
+  "Note: legacy v1 images are not included in stats",
+  &get_arguments_stats, &execute_stats);
+
+} // namespace pool
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Remove.cc b/src/tools/rbd/action/Remove.cc
new file mode 100644
index 00000000..337d42be
--- /dev/null
+++ b/src/tools/rbd/action/Remove.cc
@@ -0,0 +1,161 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include "include/stringify.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace remove {
+
+namespace {
+
+bool is_auto_delete_snapshot(librbd::Image* image,
+                             const librbd::snap_info_t &snap_info) {
+  librbd::snap_namespace_type_t namespace_type;
+  int r = image->snap_get_namespace_type(snap_info.id, &namespace_type);
+  if (r < 0) {
+    return false;
+  }
+
+  switch (namespace_type) {
+  case RBD_SNAP_NAMESPACE_TYPE_TRASH:
+    return true;
+  default:
+    return false;
+  }
+}
+
+} // anonymous namespace
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_delete(librbd::RBD &rbd, librados::IoCtx& io_ctx,
+                     const char *imgname, bool no_progress)
+{
+  utils::ProgressContext pc("Removing image", no_progress);
+  int r = rbd.remove_with_progress(io_ctx, imgname, pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_no_progress_option(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  io_ctx.set_osdmap_full_try();
+
+  librbd::RBD rbd;
+  r = do_delete(rbd, io_ctx, image_name.c_str(),
+                vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    if (r == -ENOTEMPTY) {
+      librbd::Image image;
+      std::vector<librbd::snap_info_t> snaps;
+      int image_r = utils::open_image(io_ctx, image_name, true, &image);
+      if (image_r >= 0) {
+        image_r = image.snap_list(snaps);
+      }
+      if (image_r >= 0) {
+        snaps.erase(std::remove_if(snaps.begin(), snaps.end(),
+			           [&image](const librbd::snap_info_t& snap) {
+                                     return is_auto_delete_snapshot(&image,
+                                                                    snap);
+                                   }),
+                    snaps.end());
+      }
+
+      if (!snaps.empty()) {
+        std::cerr << "rbd: image has snapshots - these must be deleted"
+                  << " with 'rbd snap purge' before the image can be removed."
+                  << std::endl;
+      } else {
+        std::cerr << "rbd: image has snapshots with linked clones - these must "
+                  << "be deleted or flattened before the image can be removed."
+                  << std::endl;
+      }
+    } else if (r == -EBUSY) {
+      std::cerr << "rbd: error: image still has watchers"
+                << std::endl
+                << "This means the image is still open or the client using "
+                << "it crashed. Try again after closing/unmapping it or "
+                << "waiting 30s for the crashed client to timeout."
+                << std::endl;
+    } else if (r == -EMLINK) {
+      librbd::Image image;
+      int image_r = utils::open_image(io_ctx, image_name, true, &image);
+      librbd::group_info_t group_info;
+      if (image_r == 0) {
+	image_r = image.get_group(&group_info, sizeof(group_info));
+      }
+      if (image_r == 0) {
+        std::string pool_name = "";
+        librados::Rados rados(io_ctx);
+        librados::IoCtx pool_io_ctx;
+        image_r = rados.ioctx_create2(group_info.pool, pool_io_ctx);
+        if (image_r < 0) {
+          pool_name = "<missing group pool " + stringify(group_info.pool) + ">";
+        } else {
+          pool_name = pool_io_ctx.get_pool_name();
+        }
+        std::cerr << "rbd: error: image belongs to a group "
+                  << pool_name << "/";
+        if (!io_ctx.get_namespace().empty()) {
+          std::cerr << io_ctx.get_namespace() << "/";
+        }
+        std::cerr << group_info.name;
+      } else
+	std::cerr << "rbd: error: image belongs to a group";
+
+      std::cerr << std::endl
+		<< "Remove the image from the group and try again."
+		<< std::endl;
+      image.close();
+    } else {
+      std::cerr << "rbd: delete error: " << cpp_strerror(r) << std::endl;
+    }
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"remove"}, {"rm"}, "Delete an image.", "", &get_arguments, &execute);
+
+} // namespace remove
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Rename.cc b/src/tools/rbd/action/Rename.cc
new file mode 100644
index 00000000..b4954bcb
--- /dev/null
+++ b/src/tools/rbd/action/Rename.cc
@@ -0,0 +1,94 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace rename {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_rename(librbd::RBD &rbd, librados::IoCtx& io_ctx,
+                     const char *imgname, const char *destname)
+{
+  int r = rbd.rename(io_ctx, imgname, destname);
+  if (r < 0)
+    return r;
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_SOURCE);
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_SOURCE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string dst_image_name;
+  std::string dst_snap_name;
+  std::string dst_pool_name = pool_name;
+  std::string dst_namespace_name = namespace_name;
+  r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, &dst_pool_name,
+    &dst_namespace_name, &dst_image_name, &dst_snap_name, true,
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_FULL);
+  if (r < 0) {
+    return r;
+  }
+
+  if (pool_name != dst_pool_name) {
+    std::cerr << "rbd: mv/rename across pools not supported" << std::endl
+              << "source pool: " << pool_name << " dest pool: " << dst_pool_name
+              << std::endl;
+    return -EINVAL;
+  } else if (namespace_name != dst_namespace_name) {
+    std::cerr << "rbd: mv/rename across namespaces not supported" << std::endl
+              << "source namespace: " << namespace_name << " dest namespace: "
+              << dst_namespace_name << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::RBD rbd;
+  r = do_rename(rbd, io_ctx, image_name.c_str(), dst_image_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: rename error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"rename"}, {"mv"}, "Rename image within pool.", "", &get_arguments,
+  &execute);
+
+} // namespace rename
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Resize.cc b/src/tools/rbd/action/Resize.cc
new file mode 100644
index 00000000..60c16429
--- /dev/null
+++ b/src/tools/rbd/action/Resize.cc
@@ -0,0 +1,106 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace resize {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_resize(librbd::Image& image, uint64_t size, bool allow_shrink, bool no_progress)
+{
+  utils::ProgressContext pc("Resizing image", no_progress);
+  int r = image.resize2(size, allow_shrink, pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_size_option(options);
+  options->add_options()
+    ("allow-shrink", po::bool_switch(), "permit shrinking");
+  at::add_no_progress_option(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  uint64_t size;
+  r = utils::get_image_size(vm, &size);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "",
+                                 snap_name, false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::image_info_t info;
+  r = image.stat(info, sizeof(info));
+  if (r < 0) {
+    std::cerr << "rbd: resize error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  if (info.size == size) {
+    std::cerr << "rbd: new size is equal to original size " << std::endl;
+    return -EINVAL;
+  }
+
+  if (info.size > size && !vm["allow-shrink"].as<bool>()) {
+    r = -EINVAL;
+  } else {
+    r = do_resize(image, size, vm["allow-shrink"].as<bool>(), vm[at::NO_PROGRESS].as<bool>());
+  }
+
+  if (r < 0) {
+    if (r == -EINVAL && !vm["allow-shrink"].as<bool>()) {
+      std::cerr << "rbd: shrinking an image is only allowed with the "
+                << "--allow-shrink flag" << std::endl;
+      return r;
+    }
+    std::cerr << "rbd: resize error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::SwitchArguments switched_arguments({"allow-shrink"});
+Shell::Action action(
+  {"resize"}, {}, "Resize (expand or shrink) image.", "", &get_arguments,
+  &execute);
+
+} // namespace resize
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Snap.cc b/src/tools/rbd/action/Snap.cc
new file mode 100644
index 00000000..70cf62da
--- /dev/null
+++ b/src/tools/rbd/action/Snap.cc
@@ -0,0 +1,889 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/types.h"
+#include "include/stringify.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+#include <boost/bind.hpp>
+
+namespace rbd {
+namespace action {
+namespace snap {
+
+static const std::string ALL_NAME("all");
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+int do_list_snaps(librbd::Image& image, Formatter *f, bool all_snaps, librados::Rados& rados)
+{
+  std::vector<librbd::snap_info_t> snaps;
+  TextTable t;
+  int r;
+
+  r = image.snap_list(snaps);
+  if (r < 0) {
+    std::cerr << "rbd: unable to list snapshots" << std::endl;
+    return r;
+  }
+
+  if (!all_snaps) {
+    snaps.erase(remove_if(snaps.begin(),
+                          snaps.end(),
+                          boost::bind(utils::is_not_user_snap_namespace, &image, _1)),
+                snaps.end());
+  }
+
+  if (f) {
+    f->open_array_section("snapshots");
+  } else {
+    t.define_column("SNAPID", TextTable::LEFT, TextTable::RIGHT);
+    t.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+    t.define_column("SIZE", TextTable::LEFT, TextTable::RIGHT);
+    t.define_column("PROTECTED", TextTable::LEFT, TextTable::LEFT);
+    t.define_column("TIMESTAMP", TextTable::LEFT, TextTable::RIGHT);
+    if (all_snaps) {
+      t.define_column("NAMESPACE", TextTable::LEFT, TextTable::LEFT);
+    }
+  }
+
+  std::list<std::pair<int64_t, std::string>> pool_list;
+  rados.pool_list2(pool_list);
+  std::map<int64_t, std::string> pool_map(pool_list.begin(), pool_list.end());
+
+  for (std::vector<librbd::snap_info_t>::iterator s = snaps.begin();
+       s != snaps.end(); ++s) {
+    struct timespec timestamp;
+    bool snap_protected = false;
+    image.snap_get_timestamp(s->id, &timestamp);
+    string tt_str = "";
+    if(timestamp.tv_sec != 0) {
+      time_t tt = timestamp.tv_sec;
+      tt_str = ctime(&tt);
+      tt_str = tt_str.substr(0, tt_str.length() - 1);
+    }
+
+    librbd::snap_namespace_type_t snap_namespace;
+    r = image.snap_get_namespace_type(s->id, &snap_namespace);
+    if (r < 0) {
+      std::cerr << "rbd: unable to retrieve snap namespace" << std::endl;
+      return r;
+    }
+
+    std::string snap_namespace_name = "Unknown";
+    switch (snap_namespace) {
+    case RBD_SNAP_NAMESPACE_TYPE_USER:
+      snap_namespace_name = "user";
+      break;
+    case RBD_SNAP_NAMESPACE_TYPE_GROUP:
+      snap_namespace_name = "group";
+      break;
+    case RBD_SNAP_NAMESPACE_TYPE_TRASH:
+      snap_namespace_name = "trash";
+      break;
+    }
+
+    int get_trash_res = -ENOENT;
+    std::string trash_original_name;
+    int get_group_res = -ENOENT;
+    librbd::snap_group_namespace_t group_snap;
+    if (snap_namespace == RBD_SNAP_NAMESPACE_TYPE_GROUP) {
+      get_group_res = image.snap_get_group_namespace(s->id, &group_snap,
+                                                     sizeof(group_snap));
+    } else if (snap_namespace == RBD_SNAP_NAMESPACE_TYPE_TRASH) {
+      get_trash_res = image.snap_get_trash_namespace(
+        s->id, &trash_original_name);
+    }
+
+    std::string protected_str = "";
+    if (snap_namespace == RBD_SNAP_NAMESPACE_TYPE_USER) {
+      r = image.snap_is_protected(s->name.c_str(), &snap_protected);
+      if (r < 0) {
+        std::cerr << "rbd: unable to retrieve snap protection" << std::endl;
+        return r;
+      }
+    }
+
+    if (f) {
+      protected_str = snap_protected ? "true" : "false";
+      f->open_object_section("snapshot");
+      f->dump_unsigned("id", s->id);
+      f->dump_string("name", s->name);
+      f->dump_unsigned("size", s->size);
+      f->dump_string("protected", protected_str);
+      f->dump_string("timestamp", tt_str);
+      if (all_snaps) {
+	f->open_object_section("namespace");
+        f->dump_string("type", snap_namespace_name);
+	if (get_group_res == 0) {
+	  std::string pool_name = pool_map[group_snap.group_pool];
+	  f->dump_string("pool", pool_name);
+	  f->dump_string("group", group_snap.group_name);
+	  f->dump_string("group snap", group_snap.group_snap_name);
+	} else if (get_trash_res == 0) {
+          f->dump_string("original_name", trash_original_name);
+        }
+	f->close_section();
+      }
+      f->close_section();
+    } else {
+      protected_str = snap_protected ? "yes" : "";
+      t << s->id << s->name << stringify(byte_u_t(s->size)) << protected_str << tt_str;
+
+      if (all_snaps) {
+	ostringstream oss;
+        oss << snap_namespace_name;
+
+        if (get_group_res == 0) {
+	  std::string pool_name = pool_map[group_snap.group_pool];
+	  oss << " (" << pool_name << "/"
+		      << group_snap.group_name << "@"
+		      << group_snap.group_snap_name << ")";
+        } else if (get_trash_res == 0) {
+          oss << " (" << trash_original_name << ")";
+        }
+
+	t << oss.str();
+      }
+      t << TextTable::endrow;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  } else if (snaps.size()) {
+    std::cout << t;
+  }
+
+  return 0;
+}
+
+int do_add_snap(librbd::Image& image, const char *snapname)
+{
+  int r = image.snap_create(snapname);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int do_remove_snap(librbd::Image& image, const char *snapname, bool force,
+		   bool no_progress)
+{
+  uint32_t flags = force? RBD_SNAP_REMOVE_FORCE : 0;
+  int r = 0;
+  utils::ProgressContext pc("Removing snap", no_progress);
+
+  r = image.snap_remove2(snapname, flags, pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+
+  pc.finish();
+  return 0;
+}
+
+int do_rollback_snap(librbd::Image& image, const char *snapname,
+                     bool no_progress)
+{
+  utils::ProgressContext pc("Rolling back to snapshot", no_progress);
+  int r = image.snap_rollback_with_progress(snapname, pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+int do_purge_snaps(librbd::Image& image, bool no_progress)
+{
+  utils::ProgressContext pc("Removing all snapshots", no_progress);
+  std::vector<librbd::snap_info_t> snaps;
+  bool is_protected = false;
+  int r = image.snap_list(snaps);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  } else if (0 == snaps.size()) {
+    return 0;
+  } else {
+    list<std::string> protect;
+    snaps.erase(remove_if(snaps.begin(),
+                          snaps.end(),
+                          boost::bind(utils::is_not_user_snap_namespace, &image, _1)),
+                snaps.end());
+    for (auto it = snaps.begin(); it != snaps.end();) {
+      r = image.snap_is_protected(it->name.c_str(), &is_protected);
+      if (r < 0) {
+        pc.fail();
+        return r;
+      } else if (is_protected == true) {
+        protect.push_back(it->name.c_str());
+        snaps.erase(it);
+      } else {
+        ++it;
+      }
+    }
+
+    if (!protect.empty()) {
+      std::cout << "rbd: error removing snapshot(s) '" << protect << "', which "
+                << (1 == protect.size() ? "is" : "are")
+                << " protected - these must be unprotected with "
+                << "`rbd snap unprotect`."
+                << std::endl;
+    }
+    for (size_t i = 0; i < snaps.size(); ++i) {
+      r = image.snap_remove(snaps[i].name.c_str());
+      if (r < 0) {
+        pc.fail();
+        return r;
+      }
+      pc.update_progress(i + 1, snaps.size() + protect.size());
+    }
+
+    if (!protect.empty()) {
+      pc.fail();
+    } else if (snaps.size() > 0) {
+      pc.finish();
+    }
+
+    return 0;
+  }
+}
+
+int do_protect_snap(librbd::Image& image, const char *snapname)
+{
+  int r = image.snap_protect(snapname);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int do_unprotect_snap(librbd::Image& image, const char *snapname)
+{
+  int r = image.snap_unprotect(snapname);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int do_set_limit(librbd::Image& image, uint64_t limit)
+{
+  return image.snap_set_limit(limit);
+}
+
+void get_list_arguments(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_id_option(options);
+  at::add_format_options(options);
+
+  std::string name = ALL_NAME + ",a";
+
+  options->add_options()
+    (name.c_str(), po::bool_switch(), "list snapshots from all namespaces");
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  std::string image_id;
+
+  if (vm.count(at::IMAGE_ID)) {
+    image_id = vm[at::IMAGE_ID].as<std::string>();
+  }
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, image_id.empty(),
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  if (!image_id.empty() && !image_name.empty()) {
+    std::cerr << "rbd: trying to access image using both name and id. "
+              << std::endl;
+    return -EINVAL;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name,
+                                 image_id, "", true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  bool all_snaps = vm[ALL_NAME].as<bool>();
+  r = do_list_snaps(image, formatter.get(), all_snaps, rados);
+  if (r < 0) {
+    cerr << "rbd: failed to list snapshots: " << cpp_strerror(r)
+         << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_create_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_snap_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+}
+
+int execute_create(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_REQUIRED,
+    utils::SPEC_VALIDATION_SNAP);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_add_snap(image, snap_name.c_str());
+  if (r < 0) {
+    cerr << "rbd: failed to create snapshot: " << cpp_strerror(r)
+         << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_remove_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_snap_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_id_option(options);
+  at::add_snap_id_option(options);
+  at::add_no_progress_option(options);
+
+  options->add_options()
+    ("force", po::bool_switch(), "flatten children and unprotect snapshot if needed.");
+}
+
+int execute_remove(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  std::string image_id;
+  uint64_t snap_id = CEPH_NOSNAP;
+  bool force = vm["force"].as<bool>();
+  bool no_progress = vm[at::NO_PROGRESS].as<bool>();
+
+  if (vm.count(at::IMAGE_ID)) {
+    image_id = vm[at::IMAGE_ID].as<std::string>();
+  }
+  if (vm.count(at::SNAPSHOT_ID)) {
+    snap_id = vm[at::SNAPSHOT_ID].as<uint64_t>();
+  }
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, image_id.empty(),
+    (snap_id == CEPH_NOSNAP ? utils::SNAPSHOT_PRESENCE_REQUIRED :
+                              utils::SNAPSHOT_PRESENCE_PERMITTED),
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  if (!image_id.empty() && !image_name.empty()) {
+    std::cerr << "rbd: trying to access image using both name and id."
+              << std::endl;
+    return -EINVAL;
+  } else if (!snap_name.empty() && snap_id != CEPH_NOSNAP) {
+    std::cerr << "rbd: trying to access snapshot using both name and id."
+              << std::endl;
+    return -EINVAL;
+  } else if ((force || no_progress) && snap_id != CEPH_NOSNAP) {
+    std::cerr << "rbd: force and no-progress options not permitted when "
+              << "removing by id." << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  io_ctx.set_osdmap_full_try();
+  if (image_id.empty()) {
+    r = utils::open_image(io_ctx, image_name, false, &image);
+  } else {
+    r = utils::open_image_by_id(io_ctx, image_id, false, &image);
+  }
+  if (r < 0) {
+    return r;
+  }
+
+  if (!snap_name.empty()) {
+    r = do_remove_snap(image, snap_name.c_str(), force, no_progress);
+  } else {
+    r = image.snap_remove_by_id(snap_id);
+  }
+
+  if (r < 0) {
+    if (r == -EBUSY) {
+      std::cerr << "rbd: snapshot "
+                << (snap_name.empty() ? std::string("id ") + stringify(snap_id) :
+                                        std::string("'") + snap_name + "'")
+                << " is protected from removal." << std::endl;
+    } else {
+      std::cerr << "rbd: failed to remove snapshot: " << cpp_strerror(r)
+                << std::endl;
+    }
+    return r;
+  }
+  return 0;
+}
+
+void get_purge_arguments(po::options_description *positional,
+                         po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_id_option(options);
+  at::add_no_progress_option(options);
+}
+
+int execute_purge(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  std::string image_id;
+
+  if (vm.count(at::IMAGE_ID)) {
+    image_id = vm[at::IMAGE_ID].as<std::string>();
+  }
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, image_id.empty(),
+    utils::SNAPSHOT_PRESENCE_NONE, utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  if (!image_id.empty() && !image_name.empty()) {
+    std::cerr << "rbd: trying to access image using both name and id. "
+              << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  io_ctx.set_osdmap_full_try();
+  if (image_id.empty()) {
+    r = utils::open_image(io_ctx, image_name, false, &image);
+  } else {
+    r = utils::open_image_by_id(io_ctx, image_id, false, &image);
+  }
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_purge_snaps(image, vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    if (r != -EBUSY) {
+      std::cerr << "rbd: removing snaps failed: " << cpp_strerror(r)
+                << std::endl;
+    }
+    return r;
+  }
+  return 0;
+}
+
+void get_rollback_arguments(po::options_description *positional,
+                            po::options_description *options) {
+  at::add_snap_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_no_progress_option(options);
+}
+
+int execute_rollback(const po::variables_map &vm,
+                     const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_REQUIRED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_rollback_snap(image, snap_name.c_str(),
+                       vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: rollback failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_protect_arguments(po::options_description *positional,
+                           po::options_description *options) {
+  at::add_snap_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+}
+
+int execute_protect(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_REQUIRED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  bool is_protected = false;
+  r = image.snap_is_protected(snap_name.c_str(), &is_protected);
+  if (r < 0) {
+    std::cerr << "rbd: protecting snap failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  } else if (is_protected) {
+    std::cerr << "rbd: snap is already protected" << std::endl;
+    return -EBUSY;
+  }
+
+  r = do_protect_snap(image, snap_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: protecting snap failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_unprotect_arguments(po::options_description *positional,
+                             po::options_description *options) {
+  at::add_snap_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_id_option(options);
+}
+
+int execute_unprotect(const po::variables_map &vm,
+                      const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  std::string image_id;
+
+  if (vm.count(at::IMAGE_ID)) {
+    image_id = vm[at::IMAGE_ID].as<std::string>();
+  }
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, image_id.empty(),
+    utils::SNAPSHOT_PRESENCE_REQUIRED, utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  if (!image_id.empty() && !image_name.empty()) {
+    std::cerr << "rbd: trying to access image using both name and id. "
+              << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  io_ctx.set_osdmap_full_try();
+  if (image_id.empty()) {
+    r = utils::open_image(io_ctx, image_name, false, &image);
+  } else {
+    r = utils::open_image_by_id(io_ctx, image_id, false, &image);
+  }
+  if (r < 0) {
+    return r;
+  }
+
+  bool is_protected = false;
+  r = image.snap_is_protected(snap_name.c_str(), &is_protected);
+  if (r < 0) {
+    std::cerr << "rbd: unprotecting snap failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  } else if (!is_protected) {
+    std::cerr << "rbd: snap is already unprotected" << std::endl;
+    return -EINVAL;
+  }
+
+  r = do_unprotect_snap(image, snap_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: unprotecting snap failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_set_limit_arguments(po::options_description *pos,
+			     po::options_description *opt) {
+  at::add_image_spec_options(pos, opt, at::ARGUMENT_MODIFIER_NONE);
+  at::add_limit_option(opt);
+}
+
+int execute_set_limit(const po::variables_map &vm,
+                      const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  uint64_t limit;
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  if (vm.count(at::LIMIT)) {
+    limit = vm[at::LIMIT].as<uint64_t>();
+  } else {
+    std::cerr << "rbd: must specify --limit <num>" << std::endl;
+    return -ERANGE;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+      return r;
+  }
+
+  r = do_set_limit(image, limit);
+  if (r < 0) {
+    std::cerr << "rbd: setting snapshot limit failed: " << cpp_strerror(r)
+	      << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_clear_limit_arguments(po::options_description *pos,
+			       po::options_description *opt) {
+  at::add_image_spec_options(pos, opt, at::ARGUMENT_MODIFIER_NONE);
+}
+
+int execute_clear_limit(const po::variables_map &vm,
+                        const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+      return r;
+  }
+
+  r = do_set_limit(image, UINT64_MAX);
+  if (r < 0) {
+    std::cerr << "rbd: clearing snapshot limit failed: " << cpp_strerror(r)
+	      << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_rename_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  at::add_snap_spec_options(positional, options, at::ARGUMENT_MODIFIER_SOURCE);
+  at::add_snap_spec_options(positional, options, at::ARGUMENT_MODIFIER_DEST);
+}
+
+int execute_rename(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string src_snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_SOURCE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &src_snap_name, true, utils::SNAPSHOT_PRESENCE_REQUIRED,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return -r;
+  }
+
+  std::string dest_pool_name(pool_name);
+  std::string dest_namespace_name(namespace_name);
+  std::string dest_image_name;
+  std::string dest_snap_name;
+  r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_DEST, &arg_index, &dest_pool_name,
+    &dest_namespace_name, &dest_image_name, &dest_snap_name, true,
+    utils::SNAPSHOT_PRESENCE_REQUIRED, utils::SPEC_VALIDATION_SNAP);
+  if (r < 0) {
+    return -r;
+  }
+
+  if (pool_name != dest_pool_name) {
+    std::cerr << "rbd: source and destination pool must be the same"
+              << std::endl;
+    return -EINVAL;
+  } else if (namespace_name != dest_namespace_name) {
+    std::cerr << "rbd: source and destination namespace must be the same"
+              << std::endl;
+    return -EINVAL;
+  } else if (image_name != dest_image_name) {
+    std::cerr << "rbd: source and destination image name must be the same"
+              << std::endl;
+    return -EINVAL;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = image.snap_rename(src_snap_name.c_str(), dest_snap_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: renaming snap failed: " << cpp_strerror(r)
+              << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action_list(
+  {"snap", "list"}, {"snap", "ls"}, "Dump list of image snapshots.", "",
+  &get_list_arguments, &execute_list);
+Shell::Action action_create(
+  {"snap", "create"}, {"snap", "add"}, "Create a snapshot.", "",
+  &get_create_arguments, &execute_create);
+Shell::Action action_remove(
+  {"snap", "remove"}, {"snap", "rm"}, "Delete a snapshot.", "",
+  &get_remove_arguments, &execute_remove);
+Shell::Action action_purge(
+  {"snap", "purge"}, {}, "Delete all unprotected snapshots.", "",
+  &get_purge_arguments, &execute_purge);
+Shell::Action action_rollback(
+  {"snap", "rollback"}, {"snap", "revert"}, "Rollback image to snapshot.", "",
+  &get_rollback_arguments, &execute_rollback);
+Shell::Action action_protect(
+  {"snap", "protect"}, {}, "Prevent a snapshot from being deleted.", "",
+  &get_protect_arguments, &execute_protect);
+Shell::Action action_unprotect(
+  {"snap", "unprotect"}, {}, "Allow a snapshot to be deleted.", "",
+  &get_unprotect_arguments, &execute_unprotect);
+Shell::Action action_set_limit(
+  {"snap", "limit", "set"}, {}, "Limit the number of snapshots.", "",
+  &get_set_limit_arguments, &execute_set_limit);
+Shell::Action action_clear_limit(
+  {"snap", "limit", "clear"}, {}, "Remove snapshot limit.", "",
+  &get_clear_limit_arguments, &execute_clear_limit);
+Shell::Action action_rename(
+  {"snap", "rename"}, {}, "Rename a snapshot.", "",
+  &get_rename_arguments, &execute_rename);
+
+} // namespace snap
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Sparsify.cc b/src/tools/rbd/action/Sparsify.cc
new file mode 100644
index 00000000..a345f920
--- /dev/null
+++ b/src/tools/rbd/action/Sparsify.cc
@@ -0,0 +1,82 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace sparsify {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_sparsify(librbd::Image& image, size_t sparse_size,
+                       bool no_progress)
+{
+  utils::ProgressContext pc("Image sparsify", no_progress);
+  int r = image.sparsify_with_progress(sparse_size, pc);
+  if (r < 0) {
+    pc.fail();
+    return r;
+  }
+  pc.finish();
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_no_progress_option(options);
+  at::add_sparse_size_option(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 false, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  size_t sparse_size = utils::RBD_DEFAULT_SPARSE_SIZE;
+  if (vm.count(at::IMAGE_SPARSE_SIZE)) {
+    sparse_size = vm[at::IMAGE_SPARSE_SIZE].as<size_t>();
+  }
+
+  r = do_sparsify(image, sparse_size, vm[at::NO_PROGRESS].as<bool>());
+  if (r < 0) {
+    std::cerr << "rbd: sparsify error: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"sparsify"}, {},
+  "Reclaim space for zeroed image extents.", "",
+  &get_arguments, &execute);
+
+} // namespace sparsify
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Status.cc b/src/tools/rbd/action/Status.cc
new file mode 100644
index 00000000..0a599e7f
--- /dev/null
+++ b/src/tools/rbd/action/Status.cc
@@ -0,0 +1,214 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/rbd_types.h"
+#include "include/stringify.h"
+#include "common/errno.h"
+#include "common/Formatter.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace status {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+static int do_show_status(librados::IoCtx& io_ctx, const std::string &image_name,
+                          librbd::Image &image, Formatter *f)
+{
+  int r;
+  std::list<librbd::image_watcher_t> watchers;
+
+  r = image.list_watchers(watchers);
+  if (r < 0)
+    return r;
+
+  uint64_t features;
+  r = image.features(&features);
+  if (r < 0) {
+    return r;
+  }
+
+  librbd::image_migration_status_t migration_status;
+  std::string source_pool_name;
+  std::string dest_pool_name;
+  std::string migration_state;
+  if ((features & RBD_FEATURE_MIGRATING) != 0) {
+    r = librbd::RBD().migration_status(io_ctx, image_name.c_str(),
+                                       &migration_status,
+                                       sizeof(migration_status));
+    if (r < 0) {
+      std::cerr << "rbd: getting migration status failed: " << cpp_strerror(r)
+                << std::endl;
+      // not fatal
+    } else {
+      librados::IoCtx src_io_ctx;
+      r = librados::Rados(io_ctx).ioctx_create2(migration_status.source_pool_id, src_io_ctx);
+      if (r < 0) {
+        source_pool_name = stringify(migration_status.source_pool_id);
+      } else {
+        source_pool_name = src_io_ctx.get_pool_name();
+      }
+
+      librados::IoCtx dst_io_ctx;
+      r = librados::Rados(io_ctx).ioctx_create2(migration_status.dest_pool_id, dst_io_ctx);
+      if (r < 0) {
+        dest_pool_name = stringify(migration_status.dest_pool_id);
+      } else {
+        dest_pool_name = dst_io_ctx.get_pool_name();
+      }
+
+      switch (migration_status.state) {
+      case RBD_IMAGE_MIGRATION_STATE_ERROR:
+        migration_state = "error";
+        break;
+      case RBD_IMAGE_MIGRATION_STATE_PREPARING:
+        migration_state = "preparing";
+        break;
+      case RBD_IMAGE_MIGRATION_STATE_PREPARED:
+        migration_state = "prepared";
+        break;
+      case RBD_IMAGE_MIGRATION_STATE_EXECUTING:
+        migration_state = "executing";
+        break;
+      case RBD_IMAGE_MIGRATION_STATE_EXECUTED:
+        migration_state = "executed";
+        break;
+      case RBD_IMAGE_MIGRATION_STATE_ABORTING:
+        migration_state = "aborting";
+        break;
+      default:
+        migration_state = "unknown";
+      }
+    }
+  }
+
+  if (f)
+    f->open_object_section("status");
+
+  if (f) {
+    f->open_array_section("watchers");
+    for (auto &watcher : watchers) {
+      f->open_object_section("watcher");
+      f->dump_string("address", watcher.addr);
+      f->dump_unsigned("client", watcher.id);
+      f->dump_unsigned("cookie", watcher.cookie);
+      f->close_section();
+    }
+    f->close_section(); // watchers
+    if (!migration_state.empty()) {
+      f->open_object_section("migration");
+      f->dump_string("source_pool_name", source_pool_name);
+      f->dump_string("source_pool_namespace",
+                     migration_status.source_pool_namespace);
+      f->dump_string("source_image_name", migration_status.source_image_name);
+      f->dump_string("source_image_id", migration_status.source_image_id);
+      f->dump_string("dest_pool_name", dest_pool_name);
+      f->dump_string("dest_pool_namespace",
+                     migration_status.dest_pool_namespace);
+      f->dump_string("dest_image_name", migration_status.dest_image_name);
+      f->dump_string("dest_image_id", migration_status.dest_image_id);
+      f->dump_string("state", migration_state);
+      f->dump_string("state_description", migration_status.state_description);
+      f->close_section(); // migration
+    }
+  } else {
+    if (watchers.size()) {
+      std::cout << "Watchers:" << std::endl;
+      for (auto &watcher : watchers) {
+        std::cout << "\twatcher=" << watcher.addr << " client." << watcher.id
+                  << " cookie=" << watcher.cookie << std::endl;
+      }
+    } else {
+      std::cout << "Watchers: none" << std::endl;
+    }
+    if (!migration_state.empty()) {
+      if (!migration_status.source_pool_namespace.empty()) {
+        source_pool_name += ("/" + migration_status.source_pool_namespace);
+      }
+      if (!migration_status.dest_pool_namespace.empty()) {
+        dest_pool_name += ("/" + migration_status.dest_pool_namespace);
+      }
+
+      std::cout << "Migration:" << std::endl;
+      std::cout << "\tsource: " << source_pool_name << "/"
+              << migration_status.source_image_name;
+      if (!migration_status.source_image_id.empty()) {
+        std::cout << " (" << migration_status.source_image_id <<  ")";
+      }
+      std::cout << std::endl;
+      std::cout << "\tdestination: " << dest_pool_name << "/"
+                << migration_status.dest_image_name << " ("
+                << migration_status.dest_image_id <<  ")" << std::endl;
+      std::cout << "\tstate: " << migration_state;
+      if (!migration_status.state_description.empty()) {
+        std::cout << " (" << migration_status.state_description <<  ")";
+      }
+      std::cout << std::endl;
+    }
+  }
+
+  if (f) {
+    f->close_section(); // status
+    f->flush(std::cout);
+  }
+
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_format_options(options);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_show_status(io_ctx, image_name, image, formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: show status failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"status"}, {}, "Show the status of this image.", "", &get_arguments,
+  &execute);
+
+} // namespace status
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Trash.cc b/src/tools/rbd/action/Trash.cc
new file mode 100644
index 00000000..327b20ba
--- /dev/null
+++ b/src/tools/rbd/action/Trash.cc
@@ -0,0 +1,525 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 SUSE LINUX GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "common/errno.h"
+#include "include/stringify.h"
+#include "common/Formatter.h"
+#include "common/TextTable.h"
+#include "common/Clock.h"
+#include <iostream>
+#include <sstream>
+#include <boost/program_options.hpp>
+#include <boost/bind.hpp>
+
+namespace rbd {
+namespace action {
+namespace trash {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+//Optional arguments used only by this set of commands (rbd trash *)
+static const std::string EXPIRES_AT("expires-at");
+static const std::string EXPIRED_BEFORE("expired-before");
+static const std::string THRESHOLD("threshold");
+
+static bool is_not_trash_user(const librbd::trash_image_info_t &trash_info) {
+  return trash_info.source != RBD_TRASH_IMAGE_SOURCE_USER;
+}
+
+void get_move_arguments(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+  options->add_options()
+    (EXPIRES_AT.c_str(), po::value<std::string>()->default_value("now"),
+     "set the expiration time of an image so it can be purged when it is stale");
+}
+
+int execute_move(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  utime_t now = ceph_clock_now();
+  utime_t exp_time = now;
+  std::string expires_at;
+  if (vm.find(EXPIRES_AT) != vm.end()) {
+    expires_at = vm[EXPIRES_AT].as<std::string>();
+    r = utime_t::invoke_date(expires_at, &exp_time);
+    if (r < 0) {
+      std::cerr << "rbd: error calling /bin/date: " << cpp_strerror(r)
+                << std::endl;
+      return r;
+    }
+  }
+
+  time_t dt = (exp_time - now).sec();
+  if(dt < 0) {
+    std::cerr << "rbd: cannot use a date in the past as an expiration date"
+              << std::endl;
+    return -EINVAL;
+  }
+
+  librbd::RBD rbd;
+  r = rbd.trash_move(io_ctx, image_name.c_str(), dt);
+  if (r < 0) {
+    std::cerr << "rbd: deferred delete error: " << cpp_strerror(r)
+              << std::endl;
+  }
+
+  return r;
+}
+
+void get_remove_arguments(po::options_description *positional,
+                          po::options_description *options) {
+  positional->add_options()
+    (at::IMAGE_ID.c_str(), "image id\n(example: [<pool-name>/[<namespace>/]]<image-id>)");
+  at::add_pool_option(options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_namespace_option(options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_id_option(options);
+
+  at::add_no_progress_option(options);
+  options->add_options()
+      ("force", po::bool_switch(), "force remove of non-expired delayed images");
+}
+
+int execute_remove(const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_id;
+  int r = utils::get_pool_image_id(vm, &arg_index, &pool_name, &namespace_name,
+                                   &image_id);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  io_ctx.set_osdmap_full_try();
+  librbd::RBD rbd;
+
+  utils::ProgressContext pc("Removing image", vm[at::NO_PROGRESS].as<bool>());
+  r = rbd.trash_remove_with_progress(io_ctx, image_id.c_str(),
+                                     vm["force"].as<bool>(), pc);
+  if (r < 0) {
+    if (r == -ENOTEMPTY) {
+      std::cerr << "rbd: image has snapshots - these must be deleted"
+                << " with 'rbd snap purge' before the image can be removed."
+                << std::endl;
+    } else if (r == -EBUSY) {
+      std::cerr << "rbd: error: image still has watchers"
+                << std::endl
+                << "This means the image is still open or the client using "
+                << "it crashed. Try again after closing/unmapping it or "
+                << "waiting 30s for the crashed client to timeout."
+                << std::endl;
+    } else if (r == -EMLINK) {
+      std::cerr << std::endl
+                << "Remove the image from the group and try again."
+                << std::endl;
+    } else if (r == -EPERM) {
+      std::cerr << std::endl
+                << "Deferment time has not expired, please use --force if you "
+                << "really want to remove the image"
+                << std::endl;
+    } else {
+      std::cerr << "rbd: remove error: " << cpp_strerror(r) << std::endl;
+    }
+    pc.fail();
+    return r;
+  }
+
+  pc.finish();
+
+  return r;
+}
+
+std::string delete_status(time_t deferment_end_time) {
+  time_t now = time(nullptr);
+
+  std::string time_str = ctime(&deferment_end_time);
+  time_str = time_str.substr(0, time_str.length() - 1);
+
+  std::stringstream ss;
+  if (now < deferment_end_time) {
+    ss << "protected until " << time_str;
+  } else {
+    ss << "expired at " << time_str;
+  }
+
+  return ss.str();
+}
+
+int do_list(librbd::RBD &rbd, librados::IoCtx& io_ctx, bool long_flag,
+            bool all_flag, Formatter *f) {
+  std::vector<librbd::trash_image_info_t> trash_entries;
+  int r = rbd.trash_list(io_ctx, trash_entries);
+  if (r < 0) {
+    return r;
+  }
+
+  if (!all_flag) {
+    trash_entries.erase(remove_if(trash_entries.begin(),
+                                  trash_entries.end(),
+                                  boost::bind(is_not_trash_user, _1)),
+                        trash_entries.end());
+  }
+
+  if (!long_flag) {
+    if (f) {
+      f->open_array_section("trash");
+    }
+    for (const auto& entry : trash_entries) {
+       if (f) {
+         f->open_object_section("image");
+         f->dump_string("id", entry.id);
+         f->dump_string("name", entry.name);
+         f->close_section();
+       } else {
+         std::cout << entry.id << " " << entry.name << std::endl;
+       }
+    }
+    if (f) {
+      f->close_section();
+      f->flush(std::cout);
+    }
+    return 0;
+  }
+
+  TextTable tbl;
+
+  if (f) {
+    f->open_array_section("trash");
+  } else {
+    tbl.define_column("ID", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("SOURCE", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("DELETED_AT", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("STATUS", TextTable::LEFT, TextTable::LEFT);
+    tbl.define_column("PARENT", TextTable::LEFT, TextTable::LEFT);
+  }
+
+  for (const auto& entry : trash_entries) {
+    librbd::Image im;
+
+    r = rbd.open_by_id_read_only(io_ctx, im, entry.id.c_str(), NULL);
+    // image might disappear between rbd.list() and rbd.open(); ignore
+    // that, warn about other possible errors (EPERM, say, for opening
+    // an old-format image, because you need execute permission for the
+    // class method)
+    if (r < 0) {
+      if (r != -ENOENT) {
+        std::cerr << "rbd: error opening " << entry.id << ": "
+                  << cpp_strerror(r) << std::endl;
+      }
+      // in any event, continue to next image
+      continue;
+    }
+
+    std::string del_source;
+    switch (entry.source) {
+      case RBD_TRASH_IMAGE_SOURCE_USER:
+        del_source = "USER";
+        break;
+      case RBD_TRASH_IMAGE_SOURCE_MIRRORING:
+        del_source = "MIRRORING";
+        break;
+      case RBD_TRASH_IMAGE_SOURCE_MIGRATION:
+        del_source = "MIGRATION";
+        break;
+      case RBD_TRASH_IMAGE_SOURCE_REMOVING:
+        del_source = "REMOVING";
+        break;
+    }
+
+    std::string time_str = ctime(&entry.deletion_time);
+    time_str = time_str.substr(0, time_str.length() - 1);
+
+    bool has_parent = false;
+    std::string parent;
+    librbd::linked_image_spec_t parent_image;
+    librbd::snap_spec_t parent_snap;
+    r = im.get_parent(&parent_image, &parent_snap);
+    if (r == -ENOENT) {
+      r = 0;
+    } else if (r < 0) {
+      return r;
+    } else {
+      parent = parent_image.pool_name + "/";
+      if (!parent_image.pool_namespace.empty()) {
+        parent += parent_image.pool_namespace + "/";
+      }
+      parent += parent_image.image_name + "@" + parent_snap.name;
+      has_parent = true;
+    }
+
+    if (f) {
+      f->open_object_section("image");
+      f->dump_string("id", entry.id);
+      f->dump_string("name", entry.name);
+      f->dump_string("source", del_source);
+      f->dump_string("deleted_at", time_str);
+      f->dump_string("status",
+                     delete_status(entry.deferment_end_time));
+      if (has_parent) {
+        f->open_object_section("parent");
+        f->dump_string("pool", parent_image.pool_name);
+        f->dump_string("pool_namespace", parent_image.pool_namespace);
+        f->dump_string("image", parent_image.image_name);
+        f->dump_string("snapshot", parent_snap.name);
+        f->close_section();
+      }
+      f->close_section();
+    } else {
+      tbl << entry.id
+          << entry.name
+          << del_source
+          << time_str
+          << delete_status(entry.deferment_end_time);
+      if (has_parent)
+        tbl << parent;
+      tbl << TextTable::endrow;
+    }
+  }
+
+  if (f) {
+    f->close_section();
+    f->flush(std::cout);
+  } else if (!trash_entries.empty()) {
+    std::cout << tbl;
+  }
+
+  return r < 0 ? r : 0;
+}
+
+void get_list_arguments(po::options_description *positional,
+                        po::options_description *options) {
+  at::add_pool_options(positional, options, true);
+  options->add_options()
+    ("all,a", po::bool_switch(), "list images from all sources");
+  options->add_options()
+    ("long,l", po::bool_switch(), "long listing format");
+  at::add_format_options(options);
+}
+
+int execute_list(const po::variables_map &vm,
+                 const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  std::string namespace_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, false, &pool_name,
+                                              &namespace_name, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  at::Format::Formatter formatter;
+  r = utils::get_formatter(vm, &formatter);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  utils::disable_cache();
+
+  librbd::RBD rbd;
+  r = do_list(rbd, io_ctx, vm["long"].as<bool>(), vm["all"].as<bool>(),
+              formatter.get());
+  if (r < 0) {
+    std::cerr << "rbd: trash list: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+
+  return 0;
+}
+
+void get_purge_arguments(po::options_description *positional,
+                            po::options_description *options) {
+  at::add_pool_options(positional, options, true);
+  at::add_no_progress_option(options);
+
+  options->add_options()
+      (EXPIRED_BEFORE.c_str(), po::value<std::string>()->value_name("date"),
+       "purges images that expired before the given date");
+  options->add_options()
+      (THRESHOLD.c_str(), po::value<float>(),
+       "purges images until the current pool data usage is reduced to X%, "
+       "value range: 0.0-1.0");
+}
+
+int execute_purge (const po::variables_map &vm,
+                   const std::vector<std::string> &ceph_global_init_args) {
+  std::string pool_name;
+  std::string namespace_name;
+  size_t arg_index = 0;
+  int r = utils::get_pool_and_namespace_names(vm, true, false, &pool_name,
+                                              &namespace_name, &arg_index);
+  if (r < 0) {
+    return r;
+  }
+
+  utils::disable_cache();
+
+  librbd::RBD rbd;
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  io_ctx.set_osdmap_full_try();
+
+  float threshold = -1;
+  time_t expire_ts = 0;
+
+  if (vm.find(THRESHOLD) != vm.end()) {
+    threshold = vm[THRESHOLD].as<float>();
+  } else {
+    if (vm.find(EXPIRED_BEFORE) != vm.end()) {
+      utime_t new_time;
+      r = utime_t::invoke_date(vm[EXPIRED_BEFORE].as<std::string>(), &new_time);
+      if (r < 0) {
+        std::cerr << "rbd: error calling /bin/date: " << cpp_strerror(r)
+                  << std::endl;
+        return r;
+      }
+      expire_ts = new_time.sec();
+    }
+  }
+
+  utils::ProgressContext pc("Removing images", vm[at::NO_PROGRESS].as<bool>());
+  r = rbd.trash_purge_with_progress(io_ctx, expire_ts, threshold, pc);
+  if (r < 0) {
+    pc.fail();
+  } else {
+    pc.finish();
+  }
+
+  return 0;
+}
+
+void get_restore_arguments(po::options_description *positional,
+                            po::options_description *options) {
+  positional->add_options()
+    (at::IMAGE_ID.c_str(), "image id\n(example: [<pool-name>/]<image-id>)");
+  at::add_pool_option(options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_namespace_option(options, at::ARGUMENT_MODIFIER_NONE);
+  at::add_image_id_option(options);
+  at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE, "");
+}
+
+int execute_restore(const po::variables_map &vm,
+                    const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_id;
+  int r = utils::get_pool_image_id(vm, &arg_index, &pool_name, &namespace_name,
+                                   &image_id);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  std::string name;
+  if (vm.find(at::IMAGE_NAME) != vm.end()) {
+    name = vm[at::IMAGE_NAME].as<std::string>();
+  }
+
+  librbd::RBD rbd;
+  r = rbd.trash_restore(io_ctx, image_id.c_str(), name.c_str());
+  if (r < 0) {
+    if (r == -ENOENT) {
+      std::cerr << "rbd: error: image does not exist in trash"
+                << std::endl;
+    } else if (r == -EEXIST) {
+      std::cerr << "rbd: error: an image with the same name already exists, "
+                << "try again with with a different name"
+                << std::endl;
+    } else {
+      std::cerr << "rbd: restore error: " << cpp_strerror(r) << std::endl;
+    }
+    return r;
+  }
+
+  return r;
+}
+
+
+Shell::Action action_move(
+  {"trash", "move"}, {"trash", "mv"}, "Move an image to the trash.", "",
+  &get_move_arguments, &execute_move);
+
+Shell::Action action_remove(
+  {"trash", "remove"}, {"trash", "rm"}, "Remove an image from trash.", "",
+  &get_remove_arguments, &execute_remove);
+
+Shell::Action action_purge(
+  {"trash", "purge"}, {}, "Remove all expired images from trash.", "",
+  &get_purge_arguments, &execute_purge);
+
+Shell::SwitchArguments switched_arguments({"long", "l"});
+Shell::Action action_list(
+  {"trash", "list"}, {"trash", "ls"}, "List trash images.", "",
+  &get_list_arguments, &execute_list);
+
+Shell::Action action_restore(
+  {"trash", "restore"}, {}, "Restore an image from trash.", "",
+  &get_restore_arguments, &execute_restore);
+
+} // namespace trash
+} // namespace action
+} // namespace rbd
diff --git a/src/tools/rbd/action/Watch.cc b/src/tools/rbd/action/Watch.cc
new file mode 100644
index 00000000..65f0f93d
--- /dev/null
+++ b/src/tools/rbd/action/Watch.cc
@@ -0,0 +1,149 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd/ArgumentTypes.h"
+#include "tools/rbd/Shell.h"
+#include "tools/rbd/Utils.h"
+#include "include/rbd_types.h"
+#include "librbd/WatchNotifyTypes.h"
+#include "common/errno.h"
+#include <iostream>
+#include <boost/program_options.hpp>
+
+namespace rbd {
+namespace action {
+namespace watch {
+
+namespace at = argument_types;
+namespace po = boost::program_options;
+
+class RbdWatchCtx : public librados::WatchCtx2 {
+public:
+  RbdWatchCtx(librados::IoCtx& io_ctx, const char *image_name,
+              const std::string &header_oid)
+    : m_io_ctx(io_ctx), m_image_name(image_name), m_header_oid(header_oid)
+  {
+  }
+
+  ~RbdWatchCtx() override {}
+
+  void handle_notify(uint64_t notify_id,
+                             uint64_t cookie,
+                             uint64_t notifier_id,
+                             bufferlist& bl) override {
+    using namespace librbd::watch_notify;
+    NotifyMessage notify_message;
+    if (bl.length() == 0) {
+      notify_message = NotifyMessage(HeaderUpdatePayload());
+    } else {
+      try {
+        auto iter = bl.cbegin();
+        notify_message.decode(iter);
+      } catch (const buffer::error &err) {
+        std::cerr << "rbd: failed to decode image notification" << std::endl;
+      }
+    }
+   
+    std::cout << m_image_name << " received notification: notify_id="
+              << notify_id << ", cookie=" << cookie << ", notifier_id="
+              << notifier_id << ", bl.length=" << bl.length() << ", notify_op=" 
+              << notify_message.get_notify_op()  << std::endl;
+    bufferlist reply;
+    m_io_ctx.notify_ack(m_header_oid, notify_id, cookie, reply);
+  }
+
+  void handle_error(uint64_t cookie, int err) override {
+    std::cerr << m_image_name << " received error: cookie=" << cookie << ", "
+              << "err=" << cpp_strerror(err) << std::endl;
+  }
+private:
+  librados::IoCtx m_io_ctx;
+  const char *m_image_name;
+  std::string m_header_oid;
+};
+
+static int do_watch(librados::IoCtx& pp, librbd::Image &image,
+                    const char *imgname)
+{
+  uint8_t old_format;
+  int r = image.old_format(&old_format);
+  if (r < 0) {
+    std::cerr << "failed to query format" << std::endl;
+    return r;
+  }
+
+  std::string header_oid;
+  if (old_format != 0) {
+    header_oid = std::string(imgname) + RBD_SUFFIX;
+  } else {
+    std::string id;
+    r = image.get_id(&id);
+    if (r < 0) {
+      return r;
+    }
+
+    header_oid = RBD_HEADER_PREFIX + id;
+  }
+
+  uint64_t cookie;
+  RbdWatchCtx ctx(pp, imgname, header_oid);
+  r = pp.watch2(header_oid, &cookie, &ctx);
+  if (r < 0) {
+    std::cerr << "rbd: watch failed" << std::endl;
+    return r;
+  }
+
+  std::cout << "press enter to exit..." << std::endl;
+  getchar();
+
+  r = pp.unwatch2(cookie);
+  if (r < 0) {
+    std::cerr << "rbd: unwatch failed" << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+void get_arguments(po::options_description *positional,
+                   po::options_description *options) {
+  at::add_image_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE);
+}
+
+int execute(const po::variables_map &vm,
+            const std::vector<std::string> &ceph_global_init_args) {
+  size_t arg_index = 0;
+  std::string pool_name;
+  std::string namespace_name;
+  std::string image_name;
+  std::string snap_name;
+  int r = utils::get_pool_image_snapshot_names(
+    vm, at::ARGUMENT_MODIFIER_NONE, &arg_index, &pool_name, &namespace_name,
+    &image_name, &snap_name, true, utils::SNAPSHOT_PRESENCE_NONE,
+    utils::SPEC_VALIDATION_NONE);
+  if (r < 0) {
+    return r;
+  }
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  librbd::Image image;
+  r = utils::init_and_open_image(pool_name, namespace_name, image_name, "", "",
+                                 true, &rados, &io_ctx, &image);
+  if (r < 0) {
+    return r;
+  }
+
+  r = do_watch(io_ctx, image, image_name.c_str());
+  if (r < 0) {
+    std::cerr << "rbd: watch failed: " << cpp_strerror(r) << std::endl;
+    return r;
+  }
+  return 0;
+}
+
+Shell::Action action(
+  {"watch"}, {}, "Watch events on image.", "", &get_arguments, &execute);
+
+} // namespace watch
+} // namespace action
+} // namespace rbd