Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

47 files changed, 25650 insertions, 0 deletions

diff --git a/src/test/librados/CMakeLists.txt b/src/test/librados/CMakeLists.txt
new file mode 100644
index 000000000..5d5623f06
--- /dev/null
+++ b/src/test/librados/CMakeLists.txt
@@ -0,0 +1,217 @@
+# radostest
+add_library(radostest_shared OBJECT test_shared.cc)
+target_include_directories(radostest_shared PRIVATE
+  $<TARGET_PROPERTY:GTest::GTest,INTERFACE_INCLUDE_DIRECTORIES>)
+
+add_library(radostest STATIC
+  test_common.cc
+  TestCase.cc
+  test.cc
+  $<TARGET_OBJECTS:radostest_shared>)
+target_link_libraries(radostest PUBLIC
+  GTest::GTest
+  ceph-common
+  json_spirit
+  ${GSSAPI_LIBRARIES} ${EXTRALIBS})
+add_library(radostest-cxx STATIC
+  testcase_cxx.cc
+  test_cxx.cc
+  $<TARGET_OBJECTS:radostest_shared>)
+target_link_libraries(radostest-cxx PUBLIC
+  GTest::GTest
+  ceph-common)
+
+add_executable(ceph_test_rados_api_cmd
+  cmd.cc)
+target_link_libraries(ceph_test_rados_api_cmd
+  librados ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_cmd_pp
+  cmd_cxx.cc)
+target_link_libraries(ceph_test_rados_api_cmd_pp
+  librados ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_io
+  io.cc)
+target_link_libraries(ceph_test_rados_api_io
+  librados ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_io_pp
+  io_cxx.cc)
+target_link_libraries(ceph_test_rados_api_io_pp
+  librados ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_c_write_operations
+  c_write_operations.cc)
+target_link_libraries(ceph_test_rados_api_c_write_operations
+  librados ${UNITTEST_LIBS} radostest)
+
+add_executable(ceph_test_rados_api_c_read_operations
+  c_read_operations.cc)
+target_link_libraries(ceph_test_rados_api_c_read_operations
+  librados ${UNITTEST_LIBS} radostest)
+
+add_executable(ceph_test_rados_api_aio
+  aio.cc)
+target_link_libraries(ceph_test_rados_api_aio
+  librados ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_aio_pp
+  aio_cxx.cc)
+target_link_libraries(ceph_test_rados_api_aio_pp
+  librados ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_asio asio.cc)
+target_link_libraries(ceph_test_rados_api_asio global
+  librados ${UNITTEST_LIBS} spawn)
+
+add_executable(ceph_test_rados_api_list
+  list.cc
+  $<TARGET_OBJECTS:unit-main>)
+target_link_libraries(ceph_test_rados_api_list
+  librados global ${UNITTEST_LIBS} radostest)
+
+add_executable(ceph_test_rados_api_pool
+  pool.cc)
+target_link_libraries(ceph_test_rados_api_pool
+  librados ${UNITTEST_LIBS} radostest)
+
+add_executable(ceph_test_rados_api_stat
+  stat.cc)
+target_link_libraries(ceph_test_rados_api_stat
+  librados ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_stat_pp
+  stat_cxx.cc)
+target_link_libraries(ceph_test_rados_api_stat_pp
+  librados ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_watch_notify
+  watch_notify.cc)
+target_link_libraries(ceph_test_rados_api_watch_notify
+  librados ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_watch_notify_pp
+  watch_notify_cxx.cc)
+target_link_libraries(ceph_test_rados_api_watch_notify_pp
+  librados ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_cls
+  cls.cc)
+target_link_libraries(ceph_test_rados_api_cls
+  librados ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_misc
+  misc.cc
+  $<TARGET_OBJECTS:unit-main>)
+target_link_libraries(ceph_test_rados_api_misc
+  librados global ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_misc_pp
+  misc_cxx.cc
+  $<TARGET_OBJECTS:unit-main>)
+target_link_libraries(ceph_test_rados_api_misc_pp
+  librados global ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_lock
+  lock.cc)
+target_link_libraries(ceph_test_rados_api_lock
+  librados ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_lock_pp
+  lock_cxx.cc)
+target_link_libraries(ceph_test_rados_api_lock_pp
+  librados ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_service
+  service.cc)
+target_link_libraries(ceph_test_rados_api_service
+  librados global ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_service_pp
+  service_cxx.cc)
+target_link_libraries(ceph_test_rados_api_service_pp
+  librados global ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_tier_pp
+  tier_cxx.cc
+  $<TARGET_OBJECTS:unit-main>)
+target_include_directories(ceph_test_rados_api_tier_pp
+  PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/driver/rados"
+  PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
+target_link_libraries(ceph_test_rados_api_tier_pp
+  librados global ${UNITTEST_LIBS} Boost::system radostest-cxx cls_cas_internal
+  cls_cas_client spawn)
+
+add_executable(ceph_test_rados_api_snapshots
+  snapshots.cc)
+target_link_libraries(ceph_test_rados_api_snapshots
+  librados ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_snapshots_pp
+  snapshots_cxx.cc)
+target_link_libraries(ceph_test_rados_api_snapshots_pp
+  librados ${UNITTEST_LIBS} radostest-cxx)
+add_executable(ceph_test_rados_api_snapshots_stats
+  snapshots_stats.cc)
+target_link_libraries(ceph_test_rados_api_snapshots_stats
+  librados ${UNITTEST_LIBS} radostest)
+add_executable(ceph_test_rados_api_snapshots_stats_pp
+  snapshots_stats_cxx.cc)
+target_link_libraries(ceph_test_rados_api_snapshots_stats_pp
+  librados ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_api_cls_remote_reads
+  cls_remote_reads.cc
+  $<TARGET_OBJECTS:unit-main>)
+target_link_libraries(ceph_test_rados_api_cls_remote_reads
+  librados global ${UNITTEST_LIBS} radostest-cxx)
+
+install(TARGETS
+  ceph_test_rados_api_aio
+  ceph_test_rados_api_aio_pp
+  ceph_test_rados_api_asio
+  ceph_test_rados_api_c_read_operations
+  ceph_test_rados_api_c_write_operations
+  ceph_test_rados_api_cmd
+  ceph_test_rados_api_cmd_pp
+  ceph_test_rados_api_io
+  ceph_test_rados_api_io_pp
+  ceph_test_rados_api_list
+  ceph_test_rados_api_lock
+  ceph_test_rados_api_lock_pp
+  ceph_test_rados_api_misc
+  ceph_test_rados_api_misc_pp
+  ceph_test_rados_api_pool
+  ceph_test_rados_api_service
+  ceph_test_rados_api_service_pp
+  ceph_test_rados_api_snapshots
+  ceph_test_rados_api_snapshots_pp
+  ceph_test_rados_api_stat
+  ceph_test_rados_api_stat_pp
+  ceph_test_rados_api_tier_pp
+  ceph_test_rados_api_watch_notify
+  ceph_test_rados_api_watch_notify_pp
+  ceph_test_rados_api_cls_remote_reads
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+# unittest_librados
+add_executable(unittest_librados
+  librados.cc
+  )
+add_ceph_unittest(unittest_librados)
+target_link_libraries(unittest_librados librados ${BLKID_LIBRARIES}
+	${GSSAPI_LIBRARIES})
+
+# unittest_librados_config
+add_executable(unittest_librados_config
+  librados_config.cc
+  )
+add_ceph_unittest(unittest_librados_config)
+target_link_libraries(unittest_librados_config
+  librados
+  ${BLKID_LIBRARIES} ${GSSAPI_LIBRARIES})
+
+# Removing this test. We can't shove it into Finisher as it's not a
+# Context any more, and wrapping it to adapt it would be less fair.
+
+#add_executable(ceph_test_rados_completion_speed
+#  completion_speed.cc)
+#target_link_libraries(ceph_test_rados_completion_speed
+#  librados ${UNITTEST_LIBS} radostest-cxx)
+
+add_executable(ceph_test_rados_op_speed
+  op_speed.cc)
+target_link_libraries(ceph_test_rados_op_speed
+  librados ${UNITTEST_LIBS} radostest-cxx)
diff --git a/src/test/librados/TestCase.cc b/src/test/librados/TestCase.cc
new file mode 100644
index 000000000..e99c19273
--- /dev/null
+++ b/src/test/librados/TestCase.cc
@@ -0,0 +1,203 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <errno.h>
+#include <fmt/format.h>
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+#include "include/scope_guard.h"
+#include "crimson_utils.h"
+
+std::string RadosTestNS::pool_name;
+rados_t RadosTestNS::s_cluster = NULL;
+
+
+void RadosTestNS::SetUpTestCase()
+{
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool(pool_name, &s_cluster));
+}
+
+void RadosTestNS::TearDownTestCase()
+{
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &s_cluster));
+}
+
+void RadosTestNS::SetUp()
+{
+  cluster = RadosTestNS::s_cluster;
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+  int req;
+  ASSERT_EQ(0, rados_ioctx_pool_requires_alignment2(ioctx, &req));
+  ASSERT_FALSE(req);
+}
+
+void RadosTestNS::TearDown()
+{
+  if (cleanup)
+    cleanup_all_objects(ioctx);
+  rados_ioctx_destroy(ioctx);
+}
+
+void RadosTestNS::cleanup_all_objects(rados_ioctx_t ioctx)
+{
+  // remove all objects to avoid polluting other tests
+  rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
+  rados_ioctx_set_namespace(ioctx, LIBRADOS_ALL_NSPACES);
+  rados_list_ctx_t list_ctx;
+
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &list_ctx));
+  auto sg = make_scope_guard([&] { rados_nobjects_list_close(list_ctx); });
+
+  int r;
+  const char *entry = NULL;
+  const char *key = NULL;
+  const char *nspace = NULL;
+  while ((r = rados_nobjects_list_next(list_ctx, &entry, &key, &nspace)) != -ENOENT) {
+    ASSERT_EQ(0, r);
+    rados_ioctx_locator_set_key(ioctx, key);
+    rados_ioctx_set_namespace(ioctx, nspace);
+    ASSERT_EQ(0, rados_remove(ioctx, entry));
+  }
+}
+
+std::string RadosTestECNS::pool_name;
+rados_t RadosTestECNS::s_cluster = NULL;
+
+void RadosTestECNS::SetUpTestCase()
+{
+  SKIP_IF_CRIMSON();
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_ec_pool(pool_name, &s_cluster));
+}
+
+void RadosTestECNS::TearDownTestCase()
+{
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, destroy_one_ec_pool(pool_name, &s_cluster));
+}
+
+void RadosTestECNS::SetUp()
+{
+  SKIP_IF_CRIMSON();
+  cluster = RadosTestECNS::s_cluster;
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+  int req;
+  ASSERT_EQ(0, rados_ioctx_pool_requires_alignment2(ioctx, &req));
+  ASSERT_TRUE(req);
+  ASSERT_EQ(0, rados_ioctx_pool_required_alignment2(ioctx, &alignment));
+  ASSERT_NE(0U, alignment);
+}
+
+void RadosTestECNS::TearDown()
+{
+  SKIP_IF_CRIMSON();
+  if (cleanup)
+    cleanup_all_objects(ioctx);
+  rados_ioctx_destroy(ioctx);
+}
+
+std::string RadosTest::pool_name;
+rados_t RadosTest::s_cluster = NULL;
+
+void RadosTest::SetUpTestCase()
+{
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool(pool_name, &s_cluster));
+}
+
+void RadosTest::TearDownTestCase()
+{
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &s_cluster));
+}
+
+void RadosTest::SetUp()
+{
+  cluster = RadosTest::s_cluster;
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+  nspace = get_temp_pool_name();
+  rados_ioctx_set_namespace(ioctx, nspace.c_str());
+  int req;
+  ASSERT_EQ(0, rados_ioctx_pool_requires_alignment2(ioctx, &req));
+  ASSERT_FALSE(req);
+}
+
+void RadosTest::TearDown()
+{
+  if (cleanup) {
+    cleanup_default_namespace(ioctx);
+    cleanup_namespace(ioctx, nspace);
+  }
+  rados_ioctx_destroy(ioctx);
+}
+
+void RadosTest::cleanup_default_namespace(rados_ioctx_t ioctx)
+{
+  // remove all objects from the default namespace to avoid polluting
+  // other tests
+  cleanup_namespace(ioctx, "");
+}
+
+void RadosTest::cleanup_namespace(rados_ioctx_t ioctx, std::string ns)
+{
+  rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
+  rados_ioctx_set_namespace(ioctx, ns.c_str());
+  rados_list_ctx_t list_ctx;
+
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &list_ctx));
+  auto sg = make_scope_guard([&] { rados_nobjects_list_close(list_ctx); });
+
+  int r;
+  const char *entry = NULL;
+  const char *key = NULL;
+  while ((r = rados_nobjects_list_next(list_ctx, &entry, &key, NULL)) != -ENOENT) {
+    ASSERT_EQ(0, r);
+    rados_ioctx_locator_set_key(ioctx, key);
+    ASSERT_EQ(0, rados_remove(ioctx, entry));
+  }
+}
+
+std::string RadosTestEC::pool_name;
+rados_t RadosTestEC::s_cluster = NULL;
+
+void RadosTestEC::SetUpTestCase()
+{
+  SKIP_IF_CRIMSON();
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name()); 
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_ec_pool(pool_name, &s_cluster));
+}
+
+void RadosTestEC::TearDownTestCase()
+{
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, destroy_one_ec_pool(pool_name, &s_cluster));
+}
+
+void RadosTestEC::SetUp()
+{
+  SKIP_IF_CRIMSON();
+  cluster = RadosTestEC::s_cluster;
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+  nspace = get_temp_pool_name();
+  rados_ioctx_set_namespace(ioctx, nspace.c_str());
+  int req;
+  ASSERT_EQ(0, rados_ioctx_pool_requires_alignment2(ioctx, &req));
+  ASSERT_TRUE(req);
+  ASSERT_EQ(0, rados_ioctx_pool_required_alignment2(ioctx, &alignment));
+  ASSERT_NE(0U, alignment);
+}
+
+void RadosTestEC::TearDown()
+{
+  SKIP_IF_CRIMSON();
+  if (cleanup) {
+    cleanup_default_namespace(ioctx);
+    cleanup_namespace(ioctx, nspace);
+  }
+  rados_ioctx_destroy(ioctx);
+}
+
diff --git a/src/test/librados/TestCase.h b/src/test/librados/TestCase.h
new file mode 100644
index 000000000..15fcfeb73
--- /dev/null
+++ b/src/test/librados/TestCase.h
@@ -0,0 +1,124 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_TEST_RADOS_TESTCASE_H
+#define CEPH_TEST_RADOS_TESTCASE_H
+
+#include "include/rados/librados.h"
+#include "gtest/gtest.h"
+
+#include <string>
+
+/**
+ * These test cases create a temporary pool that lives as long as the
+ * test case.  We initially use the default namespace and assume
+ * test will whatever namespaces it wants.  After each test all objects
+ * are removed.
+ *
+ * Since pool creation and deletion is slow, this allows many tests to
+ * run faster.
+ */
+class RadosTestNS : public ::testing::Test {
+public:
+  RadosTestNS(bool c=false) : cleanup(c) {}
+  ~RadosTestNS() override {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static void cleanup_all_objects(rados_ioctx_t ioctx);
+  static rados_t s_cluster;
+  static std::string pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  rados_t cluster = nullptr;
+  rados_ioctx_t ioctx = nullptr;
+  bool cleanup;
+};
+
+struct RadosTestNSCleanup : public RadosTestNS {
+  RadosTestNSCleanup() : RadosTestNS(true) {}
+};
+
+class RadosTestECNS : public RadosTestNS {
+public:
+  RadosTestECNS(bool c=false) : cleanup(c) {}
+  ~RadosTestECNS() override {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static rados_t s_cluster;
+  static std::string pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  rados_t cluster = nullptr; 
+  rados_ioctx_t ioctx = nullptr;
+  uint64_t alignment = 0;
+  bool cleanup;
+};
+
+struct RadosTestECNSCleanup : public RadosTestECNS {
+  RadosTestECNSCleanup() : RadosTestECNS(true) {}
+};
+
+/**
+ * These test cases create a temporary pool that lives as long as the
+ * test case.  Each test within a test case gets a new ioctx set to a
+ * unique namespace within the pool.
+ *
+ * Since pool creation and deletion is slow, this allows many tests to
+ * run faster.
+ */
+class RadosTest : public ::testing::Test {
+public:
+  RadosTest(bool c=false) : cleanup(c) {}
+  ~RadosTest() override {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static void cleanup_default_namespace(rados_ioctx_t ioctx);
+  static void cleanup_namespace(rados_ioctx_t ioctx, std::string ns);
+  static rados_t s_cluster;
+  static std::string pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  rados_t cluster = nullptr;
+  rados_ioctx_t ioctx = nullptr;
+  std::string nspace;
+  bool cleanup;
+};
+
+class RadosTestEC : public RadosTest {
+public:
+  RadosTestEC(bool c=false) : cleanup(c) {}
+  ~RadosTestEC() override {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static rados_t s_cluster;
+  static std::string pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  rados_t cluster = nullptr;
+  rados_ioctx_t ioctx = nullptr;
+  bool cleanup;
+  std::string nspace;
+  uint64_t alignment = 0;
+};
+
+/**
+ * Test case without creating a temporary pool in advance.
+ * This is necessary for scenarios such that we need to
+ * manually create a pool, start some long-runing tasks and
+ * then the related pool is suddenly gone.
+ */
+class RadosTestNP: public ::testing::Test {
+public:
+  RadosTestNP() {}
+  ~RadosTestNP() override {}
+};
+
+#endif
diff --git a/src/test/librados/aio.cc b/src/test/librados/aio.cc
new file mode 100644
index 000000000..68587fe87
--- /dev/null
+++ b/src/test/librados/aio.cc
@@ -0,0 +1,1724 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <string>
+#include <sstream>
+#include <utility>
+#include <boost/scoped_ptr.hpp>
+#include <fmt/format.h>
+
+#include "include/err.h"
+#include "include/rados/librados.h"
+#include "include/types.h"
+#include "include/stringify.h"
+#include "include/scope_guard.h"
+
+#include "common/errno.h"
+
+#include "gtest/gtest.h"
+
+#include "test.h"
+#include "crimson_utils.h"
+
+using std::ostringstream;
+
+class AioTestData
+{
+public:
+  AioTestData()
+    : m_cluster(NULL),
+      m_ioctx(NULL),
+      m_init(false)
+  {
+  }
+
+  ~AioTestData()
+  {
+    if (m_init) {
+      rados_ioctx_destroy(m_ioctx);
+      destroy_one_pool(m_pool_name, &m_cluster);
+    }
+  }
+
+  std::string init()
+  {
+    int ret;
+    auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+    m_pool_name = get_temp_pool_name(pool_prefix);
+    std::string err = create_one_pool(m_pool_name, &m_cluster);
+    if (!err.empty()) {
+      ostringstream oss;
+      oss << "create_one_pool(" << m_pool_name << ") failed: error " << err;
+      return oss.str();
+    }
+    ret = rados_ioctx_create(m_cluster, m_pool_name.c_str(), &m_ioctx);
+    if (ret) {
+      destroy_one_pool(m_pool_name, &m_cluster);
+      ostringstream oss;
+      oss << "rados_ioctx_create failed: error " << ret;
+      return oss.str();
+    }
+    m_init = true;
+    return "";
+  }
+
+  rados_t m_cluster;
+  rados_ioctx_t m_ioctx;
+  std::string m_pool_name;
+  bool m_init;
+};
+
+TEST(LibRadosAio, TooBig) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(-E2BIG, rados_aio_write(test_data.m_ioctx, "foo",
+                                    my_completion, buf, UINT_MAX, 0));
+  ASSERT_EQ(-E2BIG, rados_aio_write_full(test_data.m_ioctx, "foo",
+                                         my_completion, buf, UINT_MAX));
+  ASSERT_EQ(-E2BIG, rados_aio_append(test_data.m_ioctx, "foo",
+                                     my_completion, buf, UINT_MAX));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAio, SimpleWrite) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  auto sg = make_scope_guard([&] { rados_aio_release(my_completion); });
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  auto sg2 = make_scope_guard([&] { rados_aio_release(my_completion2); });
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion2, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+}
+
+TEST(LibRadosAio, WaitForSafe) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  TestAlarm alarm;
+  ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAio, RoundTrip) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[256];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, RoundTrip2) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, RoundTrip3) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+
+  rados_write_op_t op1 = rados_create_write_op();
+  rados_write_op_write(op1, buf, sizeof(buf), 0);
+  rados_write_op_set_alloc_hint2(op1, 0, 0, LIBRADOS_OP_FLAG_FADVISE_DONTNEED); 
+  ASSERT_EQ(0, rados_aio_write_op_operate(op1, test_data.m_ioctx, my_completion,
+					  "foo", NULL, 0));
+  rados_release_write_op(op1);
+
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+
+  rados_read_op_t op2 = rados_create_read_op();
+  rados_read_op_read(op2, 0, sizeof(buf2), buf2, NULL, NULL);
+  rados_read_op_set_flags(op2, LIBRADOS_OP_FLAG_FADVISE_NOCACHE |
+			       LIBRADOS_OP_FLAG_FADVISE_RANDOM);
+  ceph_le32 init_value(-1);
+  ceph_le32 checksum[2];
+  rados_read_op_checksum(op2, LIBRADOS_CHECKSUM_TYPE_CRC32C,
+			 reinterpret_cast<char *>(&init_value),
+			 sizeof(init_value), 0, 0, 0,
+                         reinterpret_cast<char *>(&checksum),
+                         sizeof(checksum), NULL);
+  ASSERT_EQ(0, rados_aio_read_op_operate(op2, test_data.m_ioctx, my_completion2,
+					 "foo", 0));
+  rados_release_read_op(op2);
+  
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion2);
+
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(1U, checksum[0]);
+  ASSERT_EQ(bl.crc32c(-1), checksum[1]);
+}
+
+TEST(LibRadosAio, RoundTripAppend) {
+  AioTestData test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  char buf3[sizeof(buf) + sizeof(buf2)];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(buf3), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(buf3 + sizeof(buf), buf2, sizeof(buf2)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAio, RemoveTest) {
+  char buf[128];
+  char buf2[sizeof(buf)];
+  rados_completion_t my_completion;
+  AioTestData test_data;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(test_data.m_ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_aio_remove(test_data.m_ioctx, "foo", my_completion));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ(-ENOENT, rados_read(test_data.m_ioctx, "foo", buf2, sizeof(buf2), 0));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAio, XattrsRoundTrip) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  // append
+  AioTestData test_data;
+  ASSERT_EQ("", test_data.init());
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(test_data.m_ioctx, "foo", buf, sizeof(buf)));
+  // async getxattr
+  rados_completion_t my_completion;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  ASSERT_EQ(0, rados_aio_getxattr(test_data.m_ioctx, "foo", my_completion, attr1, buf, sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(-ENODATA, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+  // async setxattr
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_setxattr(test_data.m_ioctx, "foo", my_completion2, attr1, attr1_buf, sizeof(attr1_buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  rados_aio_release(my_completion2);
+  // async getxattr
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_getxattr(test_data.m_ioctx, "foo", my_completion3, attr1, buf, sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(attr1_buf), rados_aio_get_return_value(my_completion3));
+  rados_aio_release(my_completion3);
+  // check content of attribute
+  ASSERT_EQ(0, memcmp(attr1_buf, buf, sizeof(attr1_buf)));
+}
+
+TEST(LibRadosAio, RmXattr) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  // append
+  memset(buf, 0xaa, sizeof(buf));
+  AioTestData test_data;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_append(test_data.m_ioctx, "foo", buf, sizeof(buf)));  
+  // async setxattr
+  rados_completion_t my_completion;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  ASSERT_EQ(0, rados_aio_setxattr(test_data.m_ioctx, "foo", my_completion, attr1, attr1_buf, sizeof(attr1_buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+  // async rmxattr
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+            nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_rmxattr(test_data.m_ioctx, "foo", my_completion2, attr1));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  rados_aio_release(my_completion2);
+  // async getxattr after deletion
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+            nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_getxattr(test_data.m_ioctx, "foo", my_completion3, attr1, buf, sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ(-ENODATA, rados_aio_get_return_value(my_completion3));
+  rados_aio_release(my_completion3);
+  // Test rmxattr on a removed object
+  char buf2[128];
+  char attr2[] = "attr2";
+  char attr2_buf[] = "foo bar baz";
+  memset(buf2, 0xbb, sizeof(buf2));
+  ASSERT_EQ(0, rados_write(test_data.m_ioctx, "foo_rmxattr", buf2, sizeof(buf2), 0));
+  // asynx setxattr
+  rados_completion_t my_completion4;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+            nullptr, &my_completion4));
+  ASSERT_EQ(0, rados_aio_setxattr(test_data.m_ioctx, "foo_rmxattr", my_completion4, attr2, attr2_buf, sizeof(attr2_buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion4));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion4));
+  rados_aio_release(my_completion4);
+  // remove object
+  ASSERT_EQ(0, rados_remove(test_data.m_ioctx, "foo_rmxattr"));
+  // async rmxattr on non existing object
+  rados_completion_t my_completion5;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+            nullptr, &my_completion5));
+  ASSERT_EQ(0, rados_aio_rmxattr(test_data.m_ioctx, "foo_rmxattr", my_completion5, attr2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion5));
+  }
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(my_completion5));
+  rados_aio_release(my_completion5);
+}
+
+TEST(LibRadosAio, XattrIter) {
+  AioTestData test_data;
+  ASSERT_EQ("", test_data.init());
+  // Create an object with 2 attributes
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  char attr2[] = "attr2";
+  char attr2_buf[256];
+  for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
+    attr2_buf[j] = j % 0xff;
+  }
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(test_data.m_ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_setxattr(test_data.m_ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ(0, rados_setxattr(test_data.m_ioctx, "foo", attr2, attr2_buf, sizeof(attr2_buf)));
+  // call async version of getxattrs and wait for completion
+  rados_completion_t my_completion;
+  ASSERT_EQ(0, rados_aio_create_completion2((void*)&test_data,
+            nullptr, &my_completion));
+  rados_xattrs_iter_t iter;
+  ASSERT_EQ(0, rados_aio_getxattrs(test_data.m_ioctx, "foo", my_completion, &iter));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  // loop over attributes
+  int num_seen = 0;
+  while (true) {
+    const char *name;
+    const char *val;
+    size_t len;
+    ASSERT_EQ(0, rados_getxattrs_next(iter, &name, &val, &len));
+    if (name == NULL) {
+      break;
+    }
+    ASSERT_LT(num_seen, 2);
+    if ((strcmp(name, attr1) == 0) && (val != NULL) && (memcmp(val, attr1_buf, len) == 0)) {
+      num_seen++;
+      continue;
+    }
+    else if ((strcmp(name, attr2) == 0) && (val != NULL) && (memcmp(val, attr2_buf, len) == 0)) {
+      num_seen++;
+      continue;
+    }
+    else {
+      ASSERT_EQ(0, 1);
+    }
+  }
+  rados_getxattrs_end(iter);
+}
+
+TEST(LibRadosAio, IsComplete) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_complete.
+    while (true) {
+      int is_complete = rados_aio_is_complete(my_completion2);
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, IsSafe) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_safe.
+    while (true) {
+      int is_safe = rados_aio_is_safe(my_completion);
+      if (is_safe)
+	break;
+    }
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, ReturnValue) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "nonexistent",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAio, Flush) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_aio_flush(test_data.m_ioctx));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, FlushAsync) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  rados_completion_t flush_completion;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &flush_completion));
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_aio_flush_async(test_data.m_ioctx, flush_completion));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(flush_completion));
+  }
+  ASSERT_EQ(1, rados_aio_is_complete(my_completion));
+  ASSERT_EQ(1, rados_aio_is_complete(flush_completion));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(flush_completion);
+}
+
+TEST(LibRadosAio, RoundTripWriteFull) {
+  AioTestData test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_write_full(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  char buf3[sizeof(buf) + sizeof(buf2)];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf2, sizeof(buf2)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAio, RoundTripWriteSame) {
+  AioTestData test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char full[128];
+  memset(full, 0xcc, sizeof(full));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, full, sizeof(full), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  /* write the same buf four times */
+  char buf[32];
+  size_t ws_write_len = sizeof(full);
+  memset(buf, 0xdd, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_writesame(test_data.m_ioctx, "foo",
+				   my_completion2, buf, sizeof(buf),
+				   ws_write_len, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, full, sizeof(full), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(full), rados_aio_get_return_value(my_completion3));
+  for (char *cmp = full; cmp < full + sizeof(full); cmp += sizeof(buf)) {
+    ASSERT_EQ(0, memcmp(cmp, buf, sizeof(buf)));
+  }
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAio, SimpleStat) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAio, OperateMtime)
+{
+  AioTestData test_data;
+  ASSERT_EQ("", test_data.init());
+
+  time_t set_mtime = 1457129052;
+  {
+    rados_write_op_t op = rados_create_write_op();
+    rados_write_op_create(op, LIBRADOS_CREATE_IDEMPOTENT, nullptr);
+    rados_completion_t completion;
+    ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &completion));
+    ASSERT_EQ(0, rados_aio_write_op_operate(op, test_data.m_ioctx, completion,
+                                            "foo", &set_mtime, 0));
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, rados_aio_wait_for_complete(completion));
+    }
+    ASSERT_EQ(0, rados_aio_get_return_value(completion));
+    rados_aio_release(completion);
+    rados_release_write_op(op);
+  }
+  {
+    uint64_t size;
+    timespec mtime;
+    ASSERT_EQ(0, rados_stat2(test_data.m_ioctx, "foo", &size, &mtime));
+    EXPECT_EQ(0, size);
+    EXPECT_EQ(set_mtime, mtime.tv_sec);
+    EXPECT_EQ(0, mtime.tv_nsec);
+  }
+}
+
+TEST(LibRadosAio, Operate2Mtime)
+{
+  AioTestData test_data;
+  ASSERT_EQ("", test_data.init());
+
+  timespec set_mtime{1457129052, 123456789};
+  {
+    rados_write_op_t op = rados_create_write_op();
+    rados_write_op_create(op, LIBRADOS_CREATE_IDEMPOTENT, nullptr);
+    rados_completion_t completion;
+    ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &completion));
+    ASSERT_EQ(0, rados_aio_write_op_operate2(op, test_data.m_ioctx, completion,
+                                             "foo", &set_mtime, 0));
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, rados_aio_wait_for_complete(completion));
+    }
+    ASSERT_EQ(0, rados_aio_get_return_value(completion));
+    rados_aio_release(completion);
+    rados_release_write_op(op);
+  }
+  {
+    uint64_t size;
+    timespec mtime;
+    ASSERT_EQ(0, rados_stat2(test_data.m_ioctx, "foo", &size, &mtime));
+    EXPECT_EQ(0, size);
+    EXPECT_EQ(set_mtime.tv_sec, mtime.tv_sec);
+    EXPECT_EQ(set_mtime.tv_nsec, mtime.tv_nsec);
+  }
+}
+
+TEST(LibRadosAio, SimpleStatNS) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  char buf2[64];
+  memset(buf2, 0xbb, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  rados_ioctx_set_namespace(test_data.m_ioctx, "");
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion3, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(sizeof(buf2), psize);
+
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAio, StatRemove) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_remove(test_data.m_ioctx, "foo", my_completion3));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion3));
+  uint64_t psize2;
+  time_t pmtime2;
+  rados_completion_t my_completion4;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion4));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion4, &psize2, &pmtime2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion4));
+  }
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(my_completion4));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+  rados_aio_release(my_completion4);
+}
+
+TEST(LibRadosAio, ExecuteClass) {
+  AioTestData test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  }
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  char out[128];
+  ASSERT_EQ(0, rados_aio_exec(test_data.m_ioctx, "foo", my_completion2,
+			      "hello", "say_hello", NULL, 0, out, sizeof(out)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(13, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, strncmp("Hello, world!", out, 13));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+using std::string;
+using std::map;
+using std::set;
+
+TEST(LibRadosAio, MultiWrite) {
+  AioTestData test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2), sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+
+  char buf3[(sizeof(buf) + sizeof(buf2)) * 3];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)(sizeof(buf) + sizeof(buf2)), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(buf3 + sizeof(buf), buf2, sizeof(buf2)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAio, AioUnlock) {
+  AioTestData test_data;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_lock_exclusive(test_data.m_ioctx, "foo", "TestLock", "Cookie", "", NULL, 0));
+  rados_completion_t my_completion;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+            nullptr, &my_completion));
+  ASSERT_EQ(0, rados_aio_unlock(test_data.m_ioctx, "foo", "TestLock", "Cookie", my_completion));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  ASSERT_EQ(0, rados_lock_exclusive(test_data.m_ioctx, "foo", "TestLock", "Cookie", "", NULL,  0));
+}
+
+// EC test cases
+class AioTestDataEC
+{
+public:
+  AioTestDataEC()
+    : m_cluster(NULL),
+      m_ioctx(NULL),
+      m_init(false)
+  {
+  }
+
+  ~AioTestDataEC()
+  {
+    if (m_init) {
+      rados_ioctx_destroy(m_ioctx);
+      destroy_one_ec_pool(m_pool_name, &m_cluster);
+    }
+  }
+
+  std::string init()
+  {
+    int ret;
+    auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+    m_pool_name = get_temp_pool_name(pool_prefix);
+    std::string err = create_one_ec_pool(m_pool_name, &m_cluster);
+    if (!err.empty()) {
+      ostringstream oss;
+      oss << "create_one_ec_pool(" << m_pool_name << ") failed: error " << err;
+      return oss.str();
+    }
+    ret = rados_ioctx_create(m_cluster, m_pool_name.c_str(), &m_ioctx);
+    if (ret) {
+      destroy_one_ec_pool(m_pool_name, &m_cluster);
+      ostringstream oss;
+      oss << "rados_ioctx_create failed: error " << ret;
+      return oss.str();
+    }
+    m_init = true;
+    return "";
+  }
+
+  rados_t m_cluster;
+  rados_ioctx_t m_ioctx;
+  std::string m_pool_name;
+  bool m_init;
+};
+
+TEST(LibRadosAioEC, SimpleWrite) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  auto sg = make_scope_guard([&] { rados_aio_release(my_completion); });
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  auto sg2 = make_scope_guard([&] { rados_aio_release(my_completion2); });
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion2, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+}
+
+TEST(LibRadosAioEC, WaitForComplete) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  TestAlarm alarm;
+  ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAioEC, RoundTrip) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[256];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, RoundTrip2) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, RoundTripAppend) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3, my_completion4;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  int req;
+  ASSERT_EQ(0, rados_ioctx_pool_requires_alignment2(test_data.m_ioctx, &req));
+  ASSERT_NE(0, req);
+  uint64_t alignment;
+  ASSERT_EQ(0, rados_ioctx_pool_required_alignment2(test_data.m_ioctx, &alignment));
+  ASSERT_NE(0U, alignment);
+
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion, buf, bsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  int hbsize = bsize / 2;
+  char *buf2 = (char *)new char[hbsize];
+  memset(buf2, 0xdd, hbsize);
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, hbsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_append(test_data.m_ioctx, "foo",
+			       my_completion3, buf2, hbsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  EXPECT_EQ(-EOPNOTSUPP, rados_aio_get_return_value(my_completion3));
+
+  int tbsize = bsize + hbsize;
+  char *buf3 = (char *)new char[tbsize];
+  memset(buf3, 0, tbsize);
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion4));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion4, buf3, bsize * 3, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion4));
+  }
+  ASSERT_EQ(tbsize, rados_aio_get_return_value(my_completion4));
+  ASSERT_EQ(0, memcmp(buf3, buf, bsize));
+  ASSERT_EQ(0, memcmp(buf3 + bsize, buf2, hbsize));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+  rados_aio_release(my_completion4);
+  delete[] buf;
+  delete[] buf2;
+  delete[] buf3;
+}
+
+TEST(LibRadosAioEC, IsComplete) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_complete.
+    while (true) {
+      int is_complete = rados_aio_is_complete(my_completion2);
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, IsSafe) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_safe.
+    while (true) {
+      int is_safe = rados_aio_is_safe(my_completion);
+      if (is_safe)
+	break;
+    }
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, ReturnValue) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "nonexistent",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(my_completion));
+  rados_aio_release(my_completion);
+}
+
+TEST(LibRadosAioEC, Flush) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_aio_flush(test_data.m_ioctx));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, FlushAsync) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  rados_completion_t flush_completion;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &flush_completion));
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_aio_flush_async(test_data.m_ioctx, flush_completion));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(flush_completion));
+  }
+  ASSERT_EQ(1, rados_aio_is_complete(my_completion));
+  ASSERT_EQ(1, rados_aio_is_complete(flush_completion));
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[128];
+  memset(buf2, 0, sizeof(buf2));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion2, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(flush_completion);
+}
+
+TEST(LibRadosAioEC, RoundTripWriteFull) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_write_full(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  char buf3[sizeof(buf) + sizeof(buf2)];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(buf2), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf2, sizeof(buf2)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAioEC, SimpleStat) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+
+TEST(LibRadosAioEC, SimpleStatNS) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  char buf2[64];
+  memset(buf2, 0xbb, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf2, sizeof(buf2), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  rados_ioctx_set_namespace(test_data.m_ioctx, "");
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+
+  rados_ioctx_set_namespace(test_data.m_ioctx, "nspace");
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion3, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(sizeof(buf2), psize);
+
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
+
+TEST(LibRadosAioEC, StatRemove) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  uint64_t psize;
+  time_t pmtime;
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion2, &psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(sizeof(buf), psize);
+  rados_completion_t my_completion3;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_remove(test_data.m_ioctx, "foo", my_completion3));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion3));
+  uint64_t psize2;
+  time_t pmtime2;
+  rados_completion_t my_completion4;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion4));
+  ASSERT_EQ(0, rados_aio_stat(test_data.m_ioctx, "foo",
+			      my_completion4, &psize2, &pmtime2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion4));
+  }
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(my_completion4));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+  rados_aio_release(my_completion4);
+}
+
+TEST(LibRadosAioEC, ExecuteClass) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+  rados_completion_t my_completion2;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  char out[128];
+  ASSERT_EQ(0, rados_aio_exec(test_data.m_ioctx, "foo", my_completion2,
+			      "hello", "say_hello", NULL, 0, out, sizeof(out)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(13, rados_aio_get_return_value(my_completion2));
+  ASSERT_EQ(0, strncmp("Hello, world!", out, 13));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+}
+
+TEST(LibRadosAioEC, MultiWrite) {
+  SKIP_IF_CRIMSON();
+  AioTestDataEC test_data;
+  rados_completion_t my_completion, my_completion2, my_completion3;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion, buf, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion));
+  }
+  ASSERT_EQ(0, rados_aio_get_return_value(my_completion));
+
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion2));
+  ASSERT_EQ(0, rados_aio_write(test_data.m_ioctx, "foo",
+			       my_completion2, buf2, sizeof(buf2), sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion2));
+  }
+  ASSERT_EQ(-EOPNOTSUPP, rados_aio_get_return_value(my_completion2));
+
+  char buf3[(sizeof(buf) + sizeof(buf2)) * 3];
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr,
+	      nullptr, &my_completion3));
+  ASSERT_EQ(0, rados_aio_read(test_data.m_ioctx, "foo",
+			      my_completion3, buf3, sizeof(buf3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rados_aio_wait_for_complete(my_completion3));
+  }
+  ASSERT_EQ((int)sizeof(buf), rados_aio_get_return_value(my_completion3));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+  rados_aio_release(my_completion);
+  rados_aio_release(my_completion2);
+  rados_aio_release(my_completion3);
+}
diff --git a/src/test/librados/aio_cxx.cc b/src/test/librados/aio_cxx.cc
new file mode 100644
index 000000000..5647bd9c0
--- /dev/null
+++ b/src/test/librados/aio_cxx.cc
@@ -0,0 +1,2467 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <boost/scoped_ptr.hpp>
+#include <fmt/format.h>
+
+#include "gtest/gtest.h"
+
+#include "common/errno.h"
+#include "include/err.h"
+#include "include/rados/librados.hpp"
+#include "include/types.h"
+#include "include/stringify.h"
+#include "include/scope_guard.h"
+#include "common/ceph_mutex.h"
+#include <fmt/format.h>
+
+#include "test_cxx.h"
+#include "crimson_utils.h"
+
+using namespace std;
+using namespace librados;
+
+class AioTestDataPP
+{
+public:
+  AioTestDataPP()
+    : m_init(false),    
+      m_oid("foo")
+  {
+  }
+
+  ~AioTestDataPP()
+  {
+    if (m_init) {
+      m_ioctx.close();
+      destroy_one_pool_pp(m_pool_name, m_cluster);
+    }
+  }
+
+  std::string init()
+  {
+      return init({});
+  }
+
+  std::string init(const std::map<std::string, std::string> &config)
+  {
+    int ret;
+    auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+    m_pool_name = get_temp_pool_name(pool_prefix);
+    std::string err = create_one_pool_pp(m_pool_name, m_cluster, config);
+    if (!err.empty()) {
+      ostringstream oss;
+      oss << "create_one_pool(" << m_pool_name << ") failed: error " << err;
+      return oss.str();
+    }
+    ret = m_cluster.ioctx_create(m_pool_name.c_str(), m_ioctx);
+    if (ret) {
+      destroy_one_pool_pp(m_pool_name, m_cluster);
+      ostringstream oss;
+      oss << "rados_ioctx_create failed: error " << ret;
+      return oss.str();
+    }
+    m_oid = fmt::format("oid_{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+    m_init = true;
+    return "";
+  }
+
+  Rados m_cluster;
+  IoCtx m_ioctx;
+  std::string m_pool_name;
+  bool m_init;
+  std::string m_oid;
+};
+
+TEST(LibRadosAio, TooBigPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+
+  bufferlist bl;
+  auto aio_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(-E2BIG, test_data.m_ioctx.aio_write(test_data.m_oid, aio_completion.get(), bl, UINT_MAX, 0));
+  ASSERT_EQ(-E2BIG, test_data.m_ioctx.aio_append(test_data.m_oid, aio_completion.get(), bl, UINT_MAX));
+  // ioctx.aio_write_full no way to overflow bl.length()
+}
+
+TEST(LibRadosAio, PoolQuotaPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+  string p = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ(0, test_data.m_cluster.pool_create(p.c_str()));
+  IoCtx ioctx;
+  ASSERT_EQ(0, test_data.m_cluster.ioctx_create(p.c_str(), ioctx));
+  ioctx.application_enable("rados", true);
+
+  bufferlist inbl;
+  ASSERT_EQ(0, test_data.m_cluster.mon_command(
+      "{\"prefix\": \"osd pool set-quota\", \"pool\": \"" + p +
+      "\", \"field\": \"max_bytes\", \"val\": \"4096\"}",
+      inbl, NULL, NULL));
+
+  bufferlist bl;
+  bufferptr z(4096);
+  bl.append(z);
+  int n;
+  for (n = 0; n < 1024; ++n) {
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    auto completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+    ASSERT_EQ(0, ioctx.aio_operate(test_data.m_oid + stringify(n),
+                                   completion.get(), &op,
+                                   librados::OPERATION_FULL_TRY));
+    completion->wait_for_complete();
+    int r = completion->get_return_value();
+    if (r == -EDQUOT)
+      break;
+    ASSERT_EQ(0, r);
+    sleep(1);
+  }
+  ASSERT_LT(n, 1024);
+
+  // make sure we have latest map that marked the pool full
+  test_data.m_cluster.wait_for_latest_osdmap();
+
+  // make sure we block without FULL_TRY
+  {
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    auto completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+    ASSERT_EQ(0, ioctx.aio_operate("bar", completion.get(), &op, 0));
+    sleep(5);
+    ASSERT_FALSE(completion->is_complete());
+  }
+
+  ioctx.close();
+  ASSERT_EQ(0, test_data.m_cluster.pool_delete(p.c_str()));
+}
+
+TEST(LibRadosAio, SimpleWritePP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  }
+
+  {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  test_data.m_ioctx.set_namespace("nspace");
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  }
+}
+
+TEST(LibRadosAio, WaitForSafePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  TestAlarm alarm;
+  ASSERT_EQ(0, my_completion->wait_for_complete());
+  ASSERT_EQ(0, my_completion->get_return_value());
+}
+
+TEST(LibRadosAio, RoundTripPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+                                          &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST(LibRadosAio, RoundTripPP2) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+   }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+                                          &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+//using ObjectWriteOperation/ObjectReadOperation with iohint
+TEST(LibRadosAio, RoundTripPP3)
+{
+  Rados cluster;
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+  std::string pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  auto my_completion1 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ObjectWriteOperation op;
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  op.write(0, bl);
+  op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", my_completion1.get(), &op);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion1->wait_for_complete());
+  }
+  EXPECT_EQ(0, my_completion1->get_return_value());
+
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  bl.clear();
+  ObjectReadOperation op1;
+  op1.read(0, sizeof(buf), &bl, NULL);
+  op1.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED|LIBRADOS_OP_FLAG_FADVISE_RANDOM);
+  bufferlist init_value_bl;
+  encode(static_cast<int32_t>(-1), init_value_bl);
+  bufferlist csum_bl;
+  op1.checksum(LIBRADOS_CHECKSUM_TYPE_CRC32C, init_value_bl,
+	       0, 0, 0, &csum_bl, nullptr);
+  ioctx.aio_operate("test_obj", my_completion2.get(), &op1, 0);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  EXPECT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(0, memcmp(buf, bl.c_str(), sizeof(buf)));
+
+  ASSERT_EQ(8U, csum_bl.length());
+  auto csum_bl_it = csum_bl.cbegin();
+  uint32_t csum_count;
+  uint32_t csum;
+  decode(csum_count, csum_bl_it);
+  ASSERT_EQ(1U, csum_count);
+  decode(csum, csum_bl_it);
+  ASSERT_EQ(bl.crc32c(-1), csum);
+  ioctx.remove("test_obj");
+  destroy_one_pool_pp(pool_name, cluster);
+}
+
+TEST(LibRadosAio, RoundTripSparseReadPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+   }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  std::map<uint64_t, uint64_t> extents;
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_sparse_read(test_data.m_oid, my_completion2.get(),
+                                                 &extents, &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  assert_eq_sparse(bl1, extents, bl2);
+}
+
+TEST(LibRadosAioPP, ReadIntoBufferlist) {
+
+  // here we test reading into a non-empty bufferlist referencing existing
+  // buffers
+
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+   }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  bufferlist bl2;
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xbb, sizeof(buf2));
+  bl2.append(buffer::create_static(sizeof(buf2), buf2));
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+                                          &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+}
+
+TEST(LibRadosAioPP, XattrsRoundTripPP) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, test_data.m_ioctx.append(test_data.m_oid, bl1, sizeof(buf)));
+  bufferlist bl2;
+  // async getxattr
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_getxattr(test_data.m_oid, my_completion.get(), attr1, bl2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(-ENODATA, my_completion->get_return_value());
+  // append
+  bufferlist bl3;
+  bl3.append(attr1_buf, sizeof(attr1_buf));
+  // async setxattr
+  AioTestDataPP test_data2;
+  ASSERT_EQ("", test_data2.init());
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_setxattr(test_data.m_oid, my_completion2.get(), attr1, bl3));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  // async getxattr
+  bufferlist bl4;
+  AioTestDataPP test_data3;
+  ASSERT_EQ("", test_data3.init());
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_getxattr(test_data.m_oid, my_completion3.get(), attr1, bl4));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(attr1_buf), my_completion3->get_return_value());
+  // check content of attribute
+  ASSERT_EQ(0, memcmp(bl4.c_str(), attr1_buf, sizeof(attr1_buf)));
+}
+
+TEST(LibRadosAioPP, RmXattrPP) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, test_data.m_ioctx.append(test_data.m_oid, bl1, sizeof(buf)));
+  // async setxattr
+  bufferlist bl2;
+  bl2.append(attr1_buf, sizeof(attr1_buf));
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_setxattr(test_data.m_oid, my_completion.get(), attr1, bl2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  // async rmxattr
+  AioTestDataPP test_data2;
+  ASSERT_EQ("", test_data2.init());
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_rmxattr(test_data.m_oid, my_completion2.get(), attr1));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  // async getxattr
+  AioTestDataPP test_data3;
+  ASSERT_EQ("", test_data3.init());
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  bufferlist bl3;
+  ASSERT_EQ(0, test_data.m_ioctx.aio_getxattr(test_data.m_oid, my_completion3.get(), attr1, bl3));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ(-ENODATA, my_completion3->get_return_value());
+  // Test rmxattr on a removed object
+  char buf2[128];
+  char attr2[] = "attr2";
+  char attr2_buf[] = "foo bar baz";
+  memset(buf2, 0xbb, sizeof(buf2));
+  bufferlist bl21;
+  bl21.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.write("foo_rmxattr", bl21, sizeof(buf2), 0));
+  bufferlist bl22;
+  bl22.append(attr2_buf, sizeof(attr2_buf));
+  // async setxattr
+  AioTestDataPP test_data4;
+  ASSERT_EQ("", test_data4.init());
+  auto my_completion4 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_setxattr("foo_rmxattr", my_completion4.get(), attr2, bl22));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion4->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion4->get_return_value());
+  // remove object
+  ASSERT_EQ(0, test_data.m_ioctx.remove("foo_rmxattr"));
+  // async rmxattr on non existing object
+  AioTestDataPP test_data5;
+  ASSERT_EQ("", test_data5.init());
+  auto my_completion5 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_rmxattr("foo_rmxattr", my_completion5.get(), attr2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion5->wait_for_complete());
+  }
+  ASSERT_EQ(-ENOENT, my_completion5->get_return_value());
+}
+
+TEST(LibRadosIoPP, XattrListPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  // create an object with 2 attributes
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  char attr2[] = "attr2";
+  char attr2_buf[256];
+  for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
+    attr2_buf[j] = j % 0xff;
+  }
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.append(test_data.m_oid, bl1, sizeof(buf)));
+  bufferlist bl2;
+  bl2.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, test_data.m_ioctx.setxattr(test_data.m_oid, attr1, bl2));
+  bufferlist bl3;
+  bl3.append(attr2_buf, sizeof(attr2_buf));
+  ASSERT_EQ(0, test_data.m_ioctx.setxattr(test_data.m_oid, attr2, bl3));
+  // call async version of getxattrs
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  std::map<std::string, bufferlist> attrset;
+  ASSERT_EQ(0, test_data.m_ioctx.aio_getxattrs(test_data.m_oid, my_completion.get(), attrset));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  for (std::map<std::string, bufferlist>::iterator i = attrset.begin();
+       i != attrset.end(); ++i) {
+    if (i->first == string(attr1)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr1_buf, sizeof(attr1_buf)));
+    }
+    else if (i->first == string(attr2)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr2_buf, sizeof(attr2_buf)));
+    }
+    else {
+      ASSERT_EQ(0, 1);
+    }
+  }
+}
+
+TEST(LibRadosAio, IsCompletePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+                                          &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test is_complete.
+    while (true) {
+      int is_complete = my_completion2->is_complete();
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST(LibRadosAio, IsSafePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_safe.
+    while (true) {
+      int is_complete = my_completion->is_complete();
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  bufferlist bl2;
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST(LibRadosAio, ReturnValuePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  bufferlist bl1;
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("nonexistent", my_completion.get(),
+                                          &bl1, 128, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(-ENOENT, my_completion->get_return_value());
+}
+
+TEST(LibRadosAio, FlushPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_flush());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST(LibRadosAio, FlushAsyncPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  auto flush_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_flush_async(flush_completion.get()));
+  {
+      TestAlarm alarm;
+      ASSERT_EQ(0, flush_completion->wait_for_complete());
+  }
+  ASSERT_EQ(1, my_completion->is_complete());
+  ASSERT_EQ(1, flush_completion->is_complete());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST(LibRadosAio, RoundTripWriteFullPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write_full(test_data.m_oid, my_completion2.get(), bl2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  bufferlist bl3;
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion3);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion3.get(),
+					  &bl3, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf2), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf2), bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
+}
+
+//using ObjectWriteOperation/ObjectReadOperation with iohint
+TEST(LibRadosAio, RoundTripWriteFullPP2)
+{
+  Rados cluster;
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+  std::string pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  auto my_completion1 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ObjectWriteOperation op;
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf);
+
+  op.write_full(bl);
+  op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", my_completion1.get(), &op);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion1->wait_for_complete());
+  }
+  EXPECT_EQ(0, my_completion1->get_return_value());
+
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  bl.clear();
+  ObjectReadOperation op1;
+  op1.read(0, sizeof(buf), &bl, NULL);
+  op1.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED|LIBRADOS_OP_FLAG_FADVISE_RANDOM);
+  ioctx.aio_operate("test_obj", my_completion2.get(), &op1, 0);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  EXPECT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(0, memcmp(buf, bl.c_str(), sizeof(buf)));
+
+  ioctx.remove("test_obj");
+  destroy_one_pool_pp(pool_name, cluster);
+}
+
+TEST(LibRadosAio, RoundTripWriteSamePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char full[128];
+  memset(full, 0xcc, sizeof(full));
+  bufferlist bl1;
+  bl1.append(full, sizeof(full));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(full), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  /* write the same buf four times */
+  char buf[32];
+  size_t ws_write_len = sizeof(full);
+  memset(buf, 0xdd, sizeof(buf));
+  bufferlist bl2;
+  bl2.append(buf, sizeof(buf));
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_writesame(test_data.m_oid, my_completion2.get(), bl2,
+					       ws_write_len, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  bufferlist bl3;
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion3);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion3.get(),
+					  &bl3, sizeof(full), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(full), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(full), bl3.length());
+  for (char *cmp = bl3.c_str(); cmp < bl3.c_str() + bl3.length();
+							cmp += sizeof(buf)) {
+    ASSERT_EQ(0, memcmp(cmp, buf, sizeof(buf)));
+  }
+}
+
+TEST(LibRadosAio, RoundTripWriteSamePP2)
+{
+  Rados cluster;
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+  std::string pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  auto wr_cmpl = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ObjectWriteOperation op;
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  op.writesame(0, sizeof(buf) * 4, bl);
+  op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", wr_cmpl.get(), &op);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, wr_cmpl->wait_for_complete());
+  }
+  EXPECT_EQ(0, wr_cmpl->get_return_value());
+
+  boost::scoped_ptr<AioCompletion>
+			rd_cmpl(cluster.aio_create_completion(0, 0));
+  char *cmp;
+  char full[sizeof(buf) * 4];
+  memset(full, 0, sizeof(full));
+  bufferlist fl;
+  fl.append(full, sizeof(full));
+  ObjectReadOperation op1;
+  op1.read(0, sizeof(full), &fl, NULL);
+  op1.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", rd_cmpl.get(), &op1, 0);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rd_cmpl->wait_for_complete());
+  }
+  EXPECT_EQ(0, rd_cmpl->get_return_value());
+  for (cmp = fl.c_str(); cmp < fl.c_str() + fl.length(); cmp += sizeof(buf)) {
+    ASSERT_EQ(0, memcmp(cmp, buf, sizeof(buf)));
+  }
+
+  ioctx.remove("test_obj");
+  destroy_one_pool_pp(pool_name, cluster);
+}
+
+TEST(LibRadosAio, SimpleStatPPNS) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat(test_data.m_oid, my_completion2.get(),
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+}
+
+TEST(LibRadosAio, SimpleStatPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat(test_data.m_oid, my_completion2.get(),
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+}
+
+TEST(LibRadosAio, OperateMtime)
+{
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+
+  time_t set_mtime = 1457129052;
+  {
+    auto c = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+    librados::ObjectWriteOperation op;
+    op.mtime(&set_mtime);
+    op.create(false);
+    ASSERT_EQ(0, test_data.m_ioctx.aio_operate(test_data.m_oid, c.get(), &op));
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, c->wait_for_complete());
+    }
+    ASSERT_EQ(0, c->get_return_value());
+  }
+  {
+    uint64_t size;
+    timespec mtime;
+    ASSERT_EQ(0, test_data.m_ioctx.stat2(test_data.m_oid, &size, &mtime));
+    EXPECT_EQ(0, size);
+    EXPECT_EQ(set_mtime, mtime.tv_sec);
+    EXPECT_EQ(0, mtime.tv_nsec);
+  }
+}
+
+TEST(LibRadosAio, OperateMtime2)
+{
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+
+  timespec set_mtime{1457129052, 123456789};
+  {
+    auto c = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+    librados::ObjectWriteOperation op;
+    op.mtime2(&set_mtime);
+    op.create(false);
+    ASSERT_EQ(0, test_data.m_ioctx.aio_operate(test_data.m_oid, c.get(), &op));
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, c->wait_for_complete());
+    }
+    ASSERT_EQ(0, c->get_return_value());
+  }
+  {
+    uint64_t size;
+    timespec mtime;
+    ASSERT_EQ(0, test_data.m_ioctx.stat2(test_data.m_oid, &size, &mtime));
+    EXPECT_EQ(0, size);
+    EXPECT_EQ(set_mtime.tv_sec, mtime.tv_sec);
+    EXPECT_EQ(set_mtime.tv_nsec, mtime.tv_nsec);
+  }
+}
+
+TEST(LibRadosAio, StatRemovePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat(test_data.m_oid, my_completion2.get(),
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+  uint64_t psize2;
+  time_t pmtime2;
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion3);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_remove(test_data.m_oid, my_completion3.get()));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion3->get_return_value());
+
+  auto my_completion4 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion4);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat(test_data.m_oid, my_completion4.get(),
+			  		&psize2, &pmtime2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion4->wait_for_complete());
+  }
+  ASSERT_EQ(-ENOENT, my_completion4->get_return_value());
+}
+
+TEST(LibRadosAio, ExecuteClassPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  bufferlist in, out;
+  ASSERT_EQ(0, test_data.m_ioctx.aio_exec(test_data.m_oid, my_completion2.get(),
+					  "hello", "say_hello", in, &out));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(std::string("Hello, world!"), std::string(out.c_str(), out.length()));
+}
+
+using std::string;
+using std::map;
+using std::set;
+
+TEST(LibRadosAio, OmapPP) {
+  Rados cluster;
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+  std::string pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  string header_str = "baz";
+  bufferptr bp(header_str.c_str(), header_str.size() + 1);
+  bufferlist header_to_set;
+  header_to_set.push_back(bp);
+  map<string, bufferlist> to_set;
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectWriteOperation op;
+    to_set["foo"] = header_to_set;
+    to_set["foo2"] = header_to_set;
+    to_set["qfoo3"] = header_to_set;
+    op.omap_set(to_set);
+
+    op.omap_set_header(header_to_set);
+
+    ioctx.aio_operate("test_obj", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+  }
+
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectReadOperation op;
+    map<string, pair<bufferlist, int> > assertions;
+    bufferlist val;
+    val.append(string("bar"));
+    assertions["foo"] = pair<bufferlist, int>(val, CEPH_OSD_CMPXATTR_OP_EQ);
+
+    int r;
+    op.omap_cmp(assertions, &r);
+
+    ioctx.aio_operate("test_obj", my_completion.get(), &op, 0);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(-ECANCELED, my_completion->get_return_value());
+    ASSERT_EQ(-ECANCELED, r);
+  }
+
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectReadOperation op;
+
+    set<string> set_got;
+    map<string, bufferlist> map_got;
+
+    set<string> to_get;
+    map<string, bufferlist> got3;
+
+    map<string, bufferlist> got4;
+
+    bufferlist header;
+
+    op.omap_get_keys2("", 1, &set_got, nullptr, 0);
+    op.omap_get_vals2("foo", 1, &map_got, nullptr, 0);
+
+    to_get.insert("foo");
+    to_get.insert("qfoo3");
+    op.omap_get_vals_by_keys(to_get, &got3, 0);
+
+    op.omap_get_header(&header, 0);
+
+    op.omap_get_vals2("foo2", "q", 1, &got4, nullptr, 0);
+
+    ioctx.aio_operate("test_obj", my_completion.get(), &op, 0);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+
+    ASSERT_EQ(header.length(), header_to_set.length());
+    ASSERT_EQ(set_got.size(), (unsigned)1);
+    ASSERT_EQ(*set_got.begin(), "foo");
+    ASSERT_EQ(map_got.size(), (unsigned)1);
+    ASSERT_EQ(map_got.begin()->first, "foo2");
+    ASSERT_EQ(got3.size(), (unsigned)2);
+    ASSERT_EQ(got3.begin()->first, "foo");
+    ASSERT_EQ(got3.rbegin()->first, "qfoo3");
+    ASSERT_EQ(got4.size(), (unsigned)1);
+    ASSERT_EQ(got4.begin()->first, "qfoo3");
+  }
+
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectWriteOperation op;
+    set<string> to_remove;
+    to_remove.insert("foo2");
+    op.omap_rm_keys(to_remove);
+    ioctx.aio_operate("test_obj", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+  }
+
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectReadOperation op;
+
+    set<string> set_got;
+    op.omap_get_keys2("", -1, &set_got, nullptr, 0);
+    ioctx.aio_operate("test_obj", my_completion.get(), &op, 0);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+    ASSERT_EQ(set_got.size(), (unsigned)2);
+  }
+
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectWriteOperation op;
+    op.omap_clear();
+    ioctx.aio_operate("test_obj", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+  }
+
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectReadOperation op;
+
+    set<string> set_got;
+    op.omap_get_keys2("", -1, &set_got, nullptr, 0);
+    ioctx.aio_operate("test_obj", my_completion.get(), &op, 0);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+    ASSERT_EQ(set_got.size(), (unsigned)0);
+  }
+
+  // omap_clear clears header *and* keys
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("some data");
+    map<string,bufferlist> to_set;
+    to_set["foo"] = bl;
+    to_set["foo2"] = bl;
+    to_set["qfoo3"] = bl;
+    op.omap_set(to_set);
+    op.omap_set_header(bl);
+    ioctx.aio_operate("foo3", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+  }
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectWriteOperation op;
+    op.omap_clear();
+    ioctx.aio_operate("foo3", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+  }
+  {
+    boost::scoped_ptr<AioCompletion> my_completion(cluster.aio_create_completion(0, 0));
+    ObjectReadOperation op;
+    set<string> set_got;
+    bufferlist hdr;
+    op.omap_get_keys2("", -1, &set_got, nullptr, 0);
+    op.omap_get_header(&hdr, NULL);
+    ioctx.aio_operate("foo3", my_completion.get(), &op, 0);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(0, my_completion->get_return_value());
+    ASSERT_EQ(set_got.size(), (unsigned)0);
+    ASSERT_EQ(hdr.length(), 0u);
+  }
+
+  ioctx.remove("test_obj");
+  destroy_one_pool_pp(pool_name, cluster);
+}
+
+TEST(LibRadosAio, MultiWritePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion2.get(),
+					   bl2, sizeof(buf2), sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+
+  bufferlist bl3;
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion3);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion3.get(),
+					  &bl3, (sizeof(buf) + sizeof(buf2) * 3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)(sizeof(buf) + sizeof(buf2)), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf) + sizeof(buf2), bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(bl3.c_str() + sizeof(buf), buf2, sizeof(buf2)));
+}
+
+TEST(LibRadosAio, AioUnlockPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, test_data.m_ioctx.lock_exclusive(test_data.m_oid, "TestLock", "Cookie", "", NULL, 0));
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_unlock(test_data.m_oid, "TestLock", "Cookie", my_completion.get()));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  ASSERT_EQ(0, test_data.m_ioctx.lock_exclusive(test_data.m_oid, "TestLock", "Cookie", "", NULL, 0));
+}
+
+class AioTestDataECPP
+{
+public:
+  AioTestDataECPP()
+    : m_init(false),
+      m_oid("foo")
+  {}
+
+  ~AioTestDataECPP()
+  {
+    if (m_init) {
+      m_ioctx.close();
+      destroy_one_ec_pool_pp(m_pool_name, m_cluster);
+    }
+  }
+
+  std::string init()
+  {
+    int ret;
+    auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+    m_pool_name = get_temp_pool_name(pool_prefix);
+    std::string err = create_one_ec_pool_pp(m_pool_name, m_cluster);
+    if (!err.empty()) {
+      ostringstream oss;
+      oss << "create_one_ec_pool(" << m_pool_name << ") failed: error " << err;
+      return oss.str();
+    }
+    ret = m_cluster.ioctx_create(m_pool_name.c_str(), m_ioctx);
+    if (ret) {
+      destroy_one_ec_pool_pp(m_pool_name, m_cluster);
+      ostringstream oss;
+      oss << "rados_ioctx_create failed: error " << ret;
+      return oss.str();
+    }
+    m_oid = fmt::format("oid_{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+    m_init = true;
+    return "";
+  }
+
+  Rados m_cluster;
+  IoCtx m_ioctx;
+  std::string m_pool_name;
+  bool m_init;
+  std::string m_oid;
+};
+
+// EC test cases
+TEST(LibRadosAioEC, SimpleWritePP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  }
+
+  {
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  test_data.m_ioctx.set_namespace("nspace");
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  }
+}
+
+TEST(LibRadosAioEC, WaitForSafePP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  TestAlarm alarm;
+  ASSERT_EQ(0, my_completion->wait_for_complete());
+  ASSERT_EQ(0, my_completion->get_return_value());
+}
+
+TEST(LibRadosAioEC, RoundTripPP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+                                          &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST(LibRadosAioEC, RoundTripPP2) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+                                          &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+//using ObjectWriteOperation/ObjectReadOperation with iohint
+TEST(LibRadosAioEC, RoundTripPP3)
+{
+  SKIP_IF_CRIMSON();
+  Rados cluster;
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+  std::string pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  auto my_completion1 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};;
+  ObjectWriteOperation op;
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf);
+
+  op.write(0, bl);
+  op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", my_completion1.get(), &op);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion1->wait_for_complete());
+  }
+  EXPECT_EQ(0, my_completion1->get_return_value());
+
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  bl.clear();
+  ObjectReadOperation op1;
+  op1.read(0, sizeof(buf), &bl, NULL);
+  op1.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED|LIBRADOS_OP_FLAG_FADVISE_RANDOM);
+  ioctx.aio_operate("test_obj", my_completion2.get(), &op1, 0);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  EXPECT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(0, memcmp(buf, bl.c_str(), sizeof(buf)));
+
+  ioctx.remove("test_obj");
+  destroy_one_pool_pp(pool_name, cluster);
+}
+
+TEST(LibRadosAio, RoundTripAppendPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append(test_data.m_oid, my_completion.get(),
+					    bl1, sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  char buf2[128];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append(test_data.m_oid, my_completion2.get(),
+					    bl2, sizeof(buf2)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  bufferlist bl3;
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion3);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion3.get(),
+                                          &bl3, 2 * sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)(sizeof(buf) * 2), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf) * 2, bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(bl3.c_str() + sizeof(buf), buf2, sizeof(buf2)));
+}
+
+TEST(LibRadosAioPP, RemoveTestPP) {
+  char buf[128];
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  ASSERT_EQ(0, test_data.m_ioctx.append(test_data.m_oid, bl1, sizeof(buf)));
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_remove(test_data.m_oid, my_completion.get()));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  ASSERT_EQ(-ENOENT, test_data.m_ioctx.read(test_data.m_oid, bl2, sizeof(buf), 0));
+}
+
+TEST(LibRadosAioEC, RoundTripSparseReadPP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  map<uint64_t, uint64_t> extents;
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_sparse_read(test_data.m_oid, my_completion2.get(),
+                                                 &extents, &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  assert_eq_sparse(bl1, extents, bl2);
+}
+
+TEST(LibRadosAioEC, RoundTripAppendPP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  bool req;
+  ASSERT_EQ(0, test_data.m_ioctx.pool_requires_alignment2(&req));
+  ASSERT_TRUE(req);
+  uint64_t alignment;
+  ASSERT_EQ(0, test_data.m_ioctx.pool_required_alignment2(&alignment));
+  ASSERT_NE((unsigned)0, alignment);
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  bufferlist bl1;
+  bl1.append(buf, bsize);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append(test_data.m_oid, my_completion.get(),
+					    bl1, bsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  int hbsize = bsize / 2;
+  char *buf2 = (char *)new char[hbsize];
+  memset(buf2, 0xdd, hbsize);
+  bufferlist bl2;
+  bl2.append(buf2, hbsize);
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append(test_data.m_oid, my_completion2.get(),
+					    bl2, hbsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion3);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_append(test_data.m_oid, my_completion3.get(),
+					    bl2, hbsize));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  EXPECT_EQ(-EOPNOTSUPP, my_completion3->get_return_value());
+
+  bufferlist bl3;
+  auto my_completion4 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion4);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion4.get(),
+                                          &bl3, bsize * 3, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion4->wait_for_complete());
+  }
+  int tbsize = bsize + hbsize;
+  ASSERT_EQ(tbsize, my_completion4->get_return_value());
+  ASSERT_EQ((unsigned)tbsize, bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+  ASSERT_EQ(0, memcmp(bl3.c_str() + bsize, buf2, hbsize));
+  delete[] buf;
+  delete[] buf2;
+}
+
+TEST(LibRadosAioEC, IsCompletePP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test is_complete.
+    while (true) {
+      int is_complete = my_completion2->is_complete();
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+TEST(LibRadosAioEC, IsSafePP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+
+    // Busy-wait until the AIO completes.
+    // Normally we wouldn't do this, but we want to test rados_aio_is_safe.
+    while (true) {
+      int is_complete = my_completion->is_complete();
+      if (is_complete)
+	break;
+    }
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  bufferlist bl2;
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST(LibRadosAioEC, ReturnValuePP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  bufferlist bl1;
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read("nonexistent", my_completion.get(),
+                                          &bl1, 128, 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(-ENOENT, my_completion->get_return_value());
+}
+
+TEST(LibRadosAioEC, FlushPP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_flush());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST(LibRadosAioEC, FlushAsyncPP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  auto flush_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xee, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_flush_async(flush_completion.get()));
+  {
+      TestAlarm alarm;
+      ASSERT_EQ(0, flush_completion->wait_for_complete());
+  }
+  ASSERT_EQ(1, my_completion->is_complete());
+  ASSERT_EQ(1, flush_completion->is_complete());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  bufferlist bl2;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion2.get(),
+					  &bl2, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl2.length());
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST(LibRadosAioEC, RoundTripWriteFullPP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write_full(test_data.m_oid, my_completion2.get(), bl2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  bufferlist bl3;
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion3);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion3.get(),
+					  &bl3, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf2), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf2), bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
+}
+
+//using ObjectWriteOperation/ObjectReadOperation with iohint
+TEST(LibRadosAioEC, RoundTripWriteFullPP2)
+{
+  SKIP_IF_CRIMSON();
+  Rados cluster;
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+  std::string pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  auto my_completion1 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ObjectWriteOperation op;
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf);
+
+  op.write_full(bl);
+  op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
+  ioctx.aio_operate("test_obj", my_completion1.get(), &op);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion1->wait_for_complete());
+  }
+  EXPECT_EQ(0, my_completion1->get_return_value());
+
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  bl.clear();
+  ObjectReadOperation op1;
+  op1.read(0, sizeof(buf), &bl, NULL);
+  op1.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_NOCACHE|LIBRADOS_OP_FLAG_FADVISE_RANDOM);
+  ioctx.aio_operate("test_obj", my_completion2.get(), &op1, 0);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  EXPECT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(0, memcmp(buf, bl.c_str(), sizeof(buf)));
+
+  ioctx.remove("test_obj");
+  destroy_one_pool_pp(pool_name, cluster);
+}
+
+TEST(LibRadosAioEC, SimpleStatPP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat(test_data.m_oid, my_completion2.get(),
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+}
+
+TEST(LibRadosAioEC, SimpleStatPPNS) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat(test_data.m_oid, my_completion2.get(),
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+}
+
+TEST(LibRadosAioEC, StatRemovePP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  uint64_t psize;
+  time_t pmtime;
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat(test_data.m_oid, my_completion2.get(),
+			  		&psize, &pmtime));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(sizeof(buf), psize);
+  uint64_t psize2;
+  time_t pmtime2;
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion3);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_remove(test_data.m_oid, my_completion3.get()));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion3->get_return_value());
+
+  auto my_completion4 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion4);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_stat(test_data.m_oid, my_completion4.get(),
+			  		&psize2, &pmtime2));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion4->wait_for_complete());
+  }
+  ASSERT_EQ(-ENOENT, my_completion4->get_return_value());
+}
+
+TEST(LibRadosAioEC, ExecuteClassPP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  bufferlist in, out;
+  ASSERT_EQ(0, test_data.m_ioctx.aio_exec(test_data.m_oid, my_completion2.get(),
+					  "hello", "say_hello", in, &out));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+  ASSERT_EQ(std::string("Hello, world!"), std::string(out.c_str(), out.length()));
+}
+
+TEST(LibRadosAioEC, OmapPP) {
+  SKIP_IF_CRIMSON();
+  Rados cluster;
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+  std::string pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_ec_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  string header_str = "baz";
+  bufferptr bp(header_str.c_str(), header_str.size() + 1);
+  bufferlist header_to_set;
+  header_to_set.push_back(bp);
+  map<string, bufferlist> to_set;
+  {
+    auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+    ObjectWriteOperation op;
+    to_set["foo"] = header_to_set;
+    to_set["foo2"] = header_to_set;
+    to_set["qfoo3"] = header_to_set;
+    op.omap_set(to_set);
+
+    op.omap_set_header(header_to_set);
+
+    ioctx.aio_operate("test_obj", my_completion.get(), &op);
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, my_completion->wait_for_complete());
+    }
+    EXPECT_EQ(-EOPNOTSUPP, my_completion->get_return_value());
+  }
+  ioctx.remove("test_obj");
+  destroy_one_ec_pool_pp(pool_name, cluster);
+}
+
+TEST(LibRadosAioEC, MultiWritePP) {
+  SKIP_IF_CRIMSON();
+  AioTestDataECPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  char buf2[64];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion2.get(),
+					   bl2, sizeof(buf2), sizeof(buf)));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(-EOPNOTSUPP, my_completion2->get_return_value());
+
+  bufferlist bl3;
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion3);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_read(test_data.m_oid, my_completion3.get(),
+					  &bl3, (sizeof(buf) + sizeof(buf2) * 3), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ((int)sizeof(buf), my_completion3->get_return_value());
+  ASSERT_EQ(sizeof(buf), bl3.length());
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+
+}
+
+TEST(LibRadosAio, RacingRemovePP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init({{"objecter_retry_writes_after_first_reply", "true"}}));
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion2);
+  ASSERT_EQ(0, test_data.m_ioctx.aio_remove(test_data.m_oid, my_completion2.get()));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+                                         bl, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    my_completion2->wait_for_complete();
+    my_completion->wait_for_complete();
+  }
+  ASSERT_EQ(-ENOENT, my_completion2->get_return_value());
+  ASSERT_EQ(0, my_completion->get_return_value());
+  ASSERT_EQ(0, test_data.m_ioctx.stat(test_data.m_oid, nullptr, nullptr));
+}
+
+TEST(LibRadosAio, RoundTripCmpExtPP) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char full[128];
+  memset(full, 0xcc, sizeof(full));
+  bufferlist bl1;
+  bl1.append(full, sizeof(full));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write(test_data.m_oid, my_completion.get(),
+					   bl1, sizeof(full), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  /* compare with match */
+  bufferlist cbl;
+  cbl.append(full, sizeof(full));
+  auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_cmpext(test_data.m_oid, my_completion2.get(), 0, cbl));
+
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion2->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion2->get_return_value());
+
+  /* compare with mismatch */
+  memset(full, 0xdd, sizeof(full));
+  cbl.clear();
+  cbl.append(full, sizeof(full));
+  auto my_completion3 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_EQ(0, test_data.m_ioctx.aio_cmpext(test_data.m_oid, my_completion3.get(), 0, cbl));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion3->wait_for_complete());
+  }
+  ASSERT_EQ(-MAX_ERRNO, my_completion3->get_return_value());
+}
+
+TEST(LibRadosAio, RoundTripCmpExtPP2)
+{
+  int ret;
+  char buf[128];
+  char miscmp_buf[128];
+  bufferlist cbl;
+  Rados cluster;
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_info()->name());
+  std::string pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  auto wr_cmpl = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ObjectWriteOperation wr_op;
+  memset(buf, 0xcc, sizeof(buf));
+  memset(miscmp_buf, 0xdd, sizeof(miscmp_buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  wr_op.write_full(bl);
+  wr_op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", wr_cmpl.get(), &wr_op);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, wr_cmpl->wait_for_complete());
+  }
+  EXPECT_EQ(0, wr_cmpl->get_return_value());
+
+  /* cmpext as write op. first match then mismatch */
+  auto wr_cmpext_cmpl = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  cbl.append(buf, sizeof(buf));
+  ret = 0;
+
+  wr_op.cmpext(0, cbl, &ret);
+  wr_op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", wr_cmpext_cmpl.get(), &wr_op);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, wr_cmpext_cmpl->wait_for_complete());
+  }
+  EXPECT_EQ(0, wr_cmpext_cmpl->get_return_value());
+  EXPECT_EQ(0, ret);
+
+  auto wr_cmpext_cmpl2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  cbl.clear();
+  cbl.append(miscmp_buf, sizeof(miscmp_buf));
+  ret = 0;
+
+  wr_op.cmpext(0, cbl, &ret);
+  wr_op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", wr_cmpext_cmpl2.get(), &wr_op);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, wr_cmpext_cmpl2->wait_for_complete());
+  }
+  EXPECT_EQ(-MAX_ERRNO, wr_cmpext_cmpl2->get_return_value());
+  EXPECT_EQ(-MAX_ERRNO, ret);
+
+  /* cmpext as read op */
+  auto rd_cmpext_cmpl = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ObjectReadOperation rd_op;
+  cbl.clear();
+  cbl.append(buf, sizeof(buf));
+  ret = 0;
+  rd_op.cmpext(0, cbl, &ret);
+  rd_op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", rd_cmpext_cmpl.get(), &rd_op, 0);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rd_cmpext_cmpl->wait_for_complete());
+  }
+  EXPECT_EQ(0, rd_cmpext_cmpl->get_return_value());
+  EXPECT_EQ(0, ret);
+
+  auto rd_cmpext_cmpl2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  cbl.clear();
+  cbl.append(miscmp_buf, sizeof(miscmp_buf));
+  ret = 0;
+
+  rd_op.cmpext(0, cbl, &ret);
+  rd_op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ioctx.aio_operate("test_obj", rd_cmpext_cmpl2.get(), &rd_op, 0);
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, rd_cmpext_cmpl2->wait_for_complete());
+  }
+  EXPECT_EQ(-MAX_ERRNO, rd_cmpext_cmpl2->get_return_value());
+  EXPECT_EQ(-MAX_ERRNO, ret);
+
+  ioctx.remove("test_obj");
+  destroy_one_pool_pp(pool_name, cluster);
+}
+
+ceph::mutex my_lock = ceph::make_mutex("my_lock");
+set<unsigned> inflight;
+unsigned max_success = 0;
+unsigned min_failed = 0;
+
+struct io_info {
+  unsigned i;
+  AioCompletion *c;
+};
+
+void pool_io_callback(completion_t cb, void *arg /* Actually AioCompletion* */)
+{
+  io_info *info = (io_info *)arg;
+  unsigned long i = info->i;
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, info->c->wait_for_complete());
+  }
+  int r = info->c->get_return_value();
+  //cout << "finish " << i << " r = " << r << std::endl;
+
+  std::scoped_lock l(my_lock);
+  inflight.erase(i);
+  if (r == 0) {
+    if (i > max_success) {
+      max_success = i;
+    }
+  } else {
+    if (!min_failed || i < min_failed) {
+      min_failed = i;
+    }
+  }
+}
+
+TEST(LibRadosAio, PoolEIOFlag) {
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+
+  bufferlist bl;
+  bl.append("some data");
+  std::thread *t = nullptr;
+  
+  unsigned max = 100;
+  unsigned timeout = max * 10;
+  unsigned long i = 1;
+  my_lock.lock();
+  for (; min_failed == 0 && i <= timeout; ++i) {
+    io_info *info = new io_info;
+    info->i = i;
+    info->c = Rados::aio_create_completion();
+    info->c->set_complete_callback((void*)info, pool_io_callback);
+    inflight.insert(i);
+    my_lock.unlock();
+    int r = test_data.m_ioctx.aio_write(test_data.m_oid, info->c, bl, bl.length(), 0);
+    //cout << "start " << i << " r = " << r << std::endl;
+
+    if (i == max / 2) {
+      cout << "setting pool EIO" << std::endl;
+      t = new std::thread(
+	[&] {
+	  bufferlist empty;
+	  ASSERT_EQ(0, test_data.m_cluster.mon_command(
+	    fmt::format(R"({{
+	                "prefix": "osd pool set",
+	                "pool": "{}",
+	                "var": "eio",
+	                "val": "true"
+	                }})", test_data.m_pool_name),
+	    empty, nullptr, nullptr));
+	});
+    }
+
+    std::this_thread::sleep_for(10ms);
+    my_lock.lock();
+    if (r < 0) {
+      inflight.erase(i);
+      break;
+    }
+  }
+  t->join();
+  delete t;
+
+  // wait for ios to finish
+  for (; !inflight.empty(); ++i) {
+    cout << "waiting for " << inflight.size() << std::endl;
+    my_lock.unlock();
+    sleep(1);
+    my_lock.lock();
+  }
+
+  cout << "max_success " << max_success << ", min_failed " << min_failed << std::endl;
+  ASSERT_TRUE(max_success + 1 == min_failed);
+  my_lock.unlock();
+}
+
+// This test case reproduces https://tracker.ceph.com/issues/57152
+TEST(LibRadosAio, MultiReads) {
+
+  // here we test multithreaded aio reads
+
+  AioTestDataPP test_data;
+  ASSERT_EQ("", test_data.init());
+  auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+  ASSERT_TRUE(my_completion);
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion.get(),
+                                           bl1, sizeof(buf), 0));
+  {
+    TestAlarm alarm;
+    ASSERT_EQ(0, my_completion->wait_for_complete());
+  }
+  ASSERT_EQ(0, my_completion->get_return_value());
+
+  // Don't use std::vector to store bufferlists (e.g for parallelizing aio_reads),
+  // as they are being moved whenever the vector resizes
+  // and will cause invalidated references.
+  std::deque<std::pair<bufferlist, std::unique_ptr<AioCompletion>>> reads;
+  for (int i = 0; i < 100; i++) {
+    // std::deque is appropriate here as emplace_back() is obliged to
+    // preserve the referenced inserted element. (Unlike insert() or erase())
+    auto& [bl, aiocp] = reads.emplace_back();
+    aiocp = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
+    ASSERT_TRUE(aiocp);
+    ASSERT_EQ(0, test_data.m_ioctx.aio_read("foo", aiocp.get(),
+                                            &bl, sizeof(buf), 0));
+  }
+  for (auto& [bl, aiocp] : reads) {
+    {
+      TestAlarm alarm;
+      ASSERT_EQ(0, aiocp->wait_for_complete());
+    }
+    ASSERT_EQ((int)sizeof(buf), aiocp->get_return_value());
+    ASSERT_EQ(0, memcmp(buf, bl.c_str(), sizeof(buf)));
+  }
+}
diff --git a/src/test/librados/asio.cc b/src/test/librados/asio.cc
new file mode 100644
index 000000000..9f86b4472
--- /dev/null
+++ b/src/test/librados/asio.cc
@@ -0,0 +1,369 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+
+#include "librados/librados_asio.h"
+#include <gtest/gtest.h>
+
+#include "common/ceph_argparse.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "global/global_init.h"
+
+#include <boost/range/begin.hpp>
+#include <boost/range/end.hpp>
+#include <spawn/spawn.hpp>
+#include <boost/asio/use_future.hpp>
+
+#define dout_subsys ceph_subsys_rados
+#define dout_context g_ceph_context
+
+using namespace std;
+
+// test fixture for global setup/teardown
+class AsioRados : public ::testing::Test {
+  static constexpr auto poolname = "ceph_test_rados_api_asio";
+
+ protected:
+  static librados::Rados rados;
+  static librados::IoCtx io;
+  // writes to snapio fail immediately with -EROFS. this is used to test errors
+  // that come from inside the initiating function, rather than passed to the
+  // AioCompletion callback
+  static librados::IoCtx snapio;
+
+ public:
+  static void SetUpTestCase() {
+    ASSERT_EQ(0, rados.init_with_context(g_ceph_context));
+    ASSERT_EQ(0, rados.connect());
+    // open/create test pool
+    int r = rados.ioctx_create(poolname, io);
+    if (r == -ENOENT) {
+      r = rados.pool_create(poolname);
+      if (r == -EEXIST) {
+        r = 0;
+      } else if (r == 0) {
+        r = rados.ioctx_create(poolname, io);
+      }
+    }
+    ASSERT_EQ(0, r);
+    ASSERT_EQ(0, rados.ioctx_create(poolname, snapio));
+    snapio.snap_set_read(1);
+    // initialize the "exist" object
+    bufferlist bl;
+    bl.append("hello");
+    ASSERT_EQ(0, io.write_full("exist", bl));
+  }
+
+  static void TearDownTestCase() {
+    rados.shutdown();
+  }
+};
+librados::Rados AsioRados::rados;
+librados::IoCtx AsioRados::io;
+librados::IoCtx AsioRados::snapio;
+
+TEST_F(AsioRados, AsyncReadCallback)
+{
+  boost::asio::io_service service;
+
+  auto success_cb = [&] (boost::system::error_code ec, bufferlist bl) {
+    EXPECT_FALSE(ec);
+    EXPECT_EQ("hello", bl.to_str());
+  };
+  librados::async_read(service, io, "exist", 256, 0, success_cb);
+
+  auto failure_cb = [&] (boost::system::error_code ec, bufferlist bl) {
+    EXPECT_EQ(boost::system::errc::no_such_file_or_directory, ec);
+  };
+  librados::async_read(service, io, "noexist", 256, 0, failure_cb);
+
+  service.run();
+}
+
+TEST_F(AsioRados, AsyncReadFuture)
+{
+  boost::asio::io_service service;
+
+  std::future<bufferlist> f1 = librados::async_read(service, io, "exist", 256,
+                                                    0, boost::asio::use_future);
+  std::future<bufferlist> f2 = librados::async_read(service, io, "noexist", 256,
+                                                    0, boost::asio::use_future);
+
+  service.run();
+
+  EXPECT_NO_THROW({
+    auto bl = f1.get();
+    EXPECT_EQ("hello", bl.to_str());
+  });
+  EXPECT_THROW(f2.get(), boost::system::system_error);
+}
+
+TEST_F(AsioRados, AsyncReadYield)
+{
+  boost::asio::io_service service;
+
+  auto success_cr = [&] (spawn::yield_context yield) {
+    boost::system::error_code ec;
+    auto bl = librados::async_read(service, io, "exist", 256, 0, yield[ec]);
+    EXPECT_FALSE(ec);
+    EXPECT_EQ("hello", bl.to_str());
+  };
+  spawn::spawn(service, success_cr);
+
+  auto failure_cr = [&] (spawn::yield_context yield) {
+    boost::system::error_code ec;
+    auto bl = librados::async_read(service, io, "noexist", 256, 0, yield[ec]);
+    EXPECT_EQ(boost::system::errc::no_such_file_or_directory, ec);
+  };
+  spawn::spawn(service, failure_cr);
+
+  service.run();
+}
+
+TEST_F(AsioRados, AsyncWriteCallback)
+{
+  boost::asio::io_service service;
+
+  bufferlist bl;
+  bl.append("hello");
+
+  auto success_cb = [&] (boost::system::error_code ec) {
+    EXPECT_FALSE(ec);
+  };
+  librados::async_write(service, io, "exist", bl, bl.length(), 0,
+                        success_cb);
+
+  auto failure_cb = [&] (boost::system::error_code ec) {
+    EXPECT_EQ(boost::system::errc::read_only_file_system, ec);
+  };
+  librados::async_write(service, snapio, "exist", bl, bl.length(), 0,
+                        failure_cb);
+
+  service.run();
+}
+
+TEST_F(AsioRados, AsyncWriteFuture)
+{
+  boost::asio::io_service service;
+
+  bufferlist bl;
+  bl.append("hello");
+
+  auto f1 = librados::async_write(service, io, "exist", bl, bl.length(), 0,
+                                  boost::asio::use_future);
+  auto f2 = librados::async_write(service, snapio, "exist", bl, bl.length(), 0,
+                                  boost::asio::use_future);
+
+  service.run();
+
+  EXPECT_NO_THROW(f1.get());
+  EXPECT_THROW(f2.get(), boost::system::system_error);
+}
+
+TEST_F(AsioRados, AsyncWriteYield)
+{
+  boost::asio::io_service service;
+
+  bufferlist bl;
+  bl.append("hello");
+
+  auto success_cr = [&] (spawn::yield_context yield) {
+    boost::system::error_code ec;
+    librados::async_write(service, io, "exist", bl, bl.length(), 0,
+                          yield[ec]);
+    EXPECT_FALSE(ec);
+    EXPECT_EQ("hello", bl.to_str());
+  };
+  spawn::spawn(service, success_cr);
+
+  auto failure_cr = [&] (spawn::yield_context yield) {
+    boost::system::error_code ec;
+    librados::async_write(service, snapio, "exist", bl, bl.length(), 0,
+                          yield[ec]);
+    EXPECT_EQ(boost::system::errc::read_only_file_system, ec);
+  };
+  spawn::spawn(service, failure_cr);
+
+  service.run();
+}
+
+TEST_F(AsioRados, AsyncReadOperationCallback)
+{
+  boost::asio::io_service service;
+  {
+    librados::ObjectReadOperation op;
+    op.read(0, 0, nullptr, nullptr);
+    auto success_cb = [&] (boost::system::error_code ec, bufferlist bl) {
+      EXPECT_FALSE(ec);
+      EXPECT_EQ("hello", bl.to_str());
+    };
+    librados::async_operate(service, io, "exist", &op, 0, success_cb);
+  }
+  {
+    librados::ObjectReadOperation op;
+    op.read(0, 0, nullptr, nullptr);
+    auto failure_cb = [&] (boost::system::error_code ec, bufferlist bl) {
+      EXPECT_EQ(boost::system::errc::no_such_file_or_directory, ec);
+    };
+    librados::async_operate(service, io, "noexist", &op, 0, failure_cb);
+  }
+  service.run();
+}
+
+TEST_F(AsioRados, AsyncReadOperationFuture)
+{
+  boost::asio::io_service service;
+  std::future<bufferlist> f1;
+  {
+    librados::ObjectReadOperation op;
+    op.read(0, 0, nullptr, nullptr);
+    f1 = librados::async_operate(service, io, "exist", &op, 0,
+                                 boost::asio::use_future);
+  }
+  std::future<bufferlist> f2;
+  {
+    librados::ObjectReadOperation op;
+    op.read(0, 0, nullptr, nullptr);
+    f2 = librados::async_operate(service, io, "noexist", &op, 0,
+                                 boost::asio::use_future);
+  }
+  service.run();
+
+  EXPECT_NO_THROW({
+    auto bl = f1.get();
+    EXPECT_EQ("hello", bl.to_str());
+  });
+  EXPECT_THROW(f2.get(), boost::system::system_error);
+}
+
+TEST_F(AsioRados, AsyncReadOperationYield)
+{
+  boost::asio::io_service service;
+
+  auto success_cr = [&] (spawn::yield_context yield) {
+    librados::ObjectReadOperation op;
+    op.read(0, 0, nullptr, nullptr);
+    boost::system::error_code ec;
+    auto bl = librados::async_operate(service, io, "exist", &op, 0,
+                                      yield[ec]);
+    EXPECT_FALSE(ec);
+    EXPECT_EQ("hello", bl.to_str());
+  };
+  spawn::spawn(service, success_cr);
+
+  auto failure_cr = [&] (spawn::yield_context yield) {
+    librados::ObjectReadOperation op;
+    op.read(0, 0, nullptr, nullptr);
+    boost::system::error_code ec;
+    auto bl = librados::async_operate(service, io, "noexist", &op, 0,
+                                      yield[ec]);
+    EXPECT_EQ(boost::system::errc::no_such_file_or_directory, ec);
+  };
+  spawn::spawn(service, failure_cr);
+
+  service.run();
+}
+
+TEST_F(AsioRados, AsyncWriteOperationCallback)
+{
+  boost::asio::io_service service;
+
+  bufferlist bl;
+  bl.append("hello");
+
+  {
+    librados::ObjectWriteOperation op;
+    op.write_full(bl);
+    auto success_cb = [&] (boost::system::error_code ec) {
+      EXPECT_FALSE(ec);
+    };
+    librados::async_operate(service, io, "exist", &op, 0, success_cb);
+  }
+  {
+    librados::ObjectWriteOperation op;
+    op.write_full(bl);
+    auto failure_cb = [&] (boost::system::error_code ec) {
+      EXPECT_EQ(boost::system::errc::read_only_file_system, ec);
+    };
+    librados::async_operate(service, snapio, "exist", &op, 0, failure_cb);
+  }
+  service.run();
+}
+
+TEST_F(AsioRados, AsyncWriteOperationFuture)
+{
+  boost::asio::io_service service;
+
+  bufferlist bl;
+  bl.append("hello");
+
+  std::future<void> f1;
+  {
+    librados::ObjectWriteOperation op;
+    op.write_full(bl);
+    f1 = librados::async_operate(service, io, "exist", &op, 0,
+                                 boost::asio::use_future);
+  }
+  std::future<void> f2;
+  {
+    librados::ObjectWriteOperation op;
+    op.write_full(bl);
+    f2 = librados::async_operate(service, snapio, "exist", &op, 0,
+                                 boost::asio::use_future);
+  }
+  service.run();
+
+  EXPECT_NO_THROW(f1.get());
+  EXPECT_THROW(f2.get(), boost::system::system_error);
+}
+
+TEST_F(AsioRados, AsyncWriteOperationYield)
+{
+  boost::asio::io_service service;
+
+  bufferlist bl;
+  bl.append("hello");
+
+  auto success_cr = [&] (spawn::yield_context yield) {
+    librados::ObjectWriteOperation op;
+    op.write_full(bl);
+    boost::system::error_code ec;
+    librados::async_operate(service, io, "exist", &op, 0, yield[ec]);
+    EXPECT_FALSE(ec);
+  };
+  spawn::spawn(service, success_cr);
+
+  auto failure_cr = [&] (spawn::yield_context yield) {
+    librados::ObjectWriteOperation op;
+    op.write_full(bl);
+    boost::system::error_code ec;
+    librados::async_operate(service, snapio, "exist", &op, 0, yield[ec]);
+    EXPECT_EQ(boost::system::errc::read_only_file_system, ec);
+  };
+  spawn::spawn(service, failure_cr);
+
+  service.run();
+}
+
+int main(int argc, char **argv)
+{
+  auto args = argv_to_vec(argc, argv);
+  env_to_vec(args);
+
+  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+                         CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(cct.get());
+
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/src/test/librados/c_read_operations.cc b/src/test/librados/c_read_operations.cc
new file mode 100644
index 000000000..a8bad0748
--- /dev/null
+++ b/src/test/librados/c_read_operations.cc
@@ -0,0 +1,895 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// Tests for the C API coverage of atomic read operations
+
+#include <cstring> // For memcpy
+#include <errno.h>
+#include <string>
+
+#include "include/buffer.h"
+#include "include/denc.h"
+#include "include/err.h"
+#include "include/rados/librados.h"
+#include "include/rbd/features.h" // For RBD_FEATURES_ALL
+#include "include/scope_guard.h"
+#include "test/librados/TestCase.h"
+#include "test/librados/test.h"
+
+const char *data = "testdata";
+const char *obj = "testobj";
+const size_t len = strlen(data);
+
+class CReadOpsTest : public RadosTest {
+protected:
+  void write_object() {
+    // Create an object and write to it
+    ASSERT_EQ(0, rados_write(ioctx, obj, data, len, 0));
+  }
+  void remove_object() {
+    ASSERT_EQ(0, rados_remove(ioctx, obj));
+  }
+  int cmp_xattr(const char *xattr, const char *value, size_t value_len,
+		uint8_t cmp_op)
+  {
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_cmpxattr(op, xattr, cmp_op, value, value_len);
+    int r = rados_read_op_operate(op, ioctx, obj, 0);
+    rados_release_read_op(op);
+    return r;
+  }
+
+  void fetch_and_verify_omap_vals(char const* const* keys,
+				  char const* const* vals,
+				  const size_t *lens,
+				  size_t len)
+  {
+    rados_omap_iter_t iter_vals, iter_keys, iter_vals_by_key;
+    int r_vals, r_keys, r_vals_by_key;
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_omap_get_vals2(op, NULL, NULL, 100, &iter_vals, NULL, &r_vals);
+    rados_read_op_omap_get_keys2(op, NULL, 100, &iter_keys, NULL, &r_keys);
+    rados_read_op_omap_get_vals_by_keys(op, keys, len,
+					&iter_vals_by_key, &r_vals_by_key);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    rados_release_read_op(op);
+    ASSERT_EQ(0, r_vals);
+    ASSERT_EQ(0, r_keys);
+    ASSERT_EQ(0, r_vals_by_key);
+
+    const char *zeros[len];
+    size_t zero_lens[len];
+    memset(zeros, 0, sizeof(zeros));
+    memset(zero_lens, 0, sizeof(zero_lens));
+    compare_omap_vals(keys, vals, lens, len, iter_vals);
+    compare_omap_vals(keys, zeros, zero_lens, len, iter_keys);
+    compare_omap_vals(keys, vals, lens, len, iter_vals_by_key);
+  }
+
+  void compare_omap_vals(char const* const* keys,
+			 char const* const* vals,
+			 const size_t *lens,
+			 size_t len,
+			 rados_omap_iter_t iter)
+  {
+    size_t i = 0;
+    char *key = NULL;
+    char *val = NULL;
+    size_t val_len = 0;
+    ASSERT_EQ(len, rados_omap_iter_size(iter));
+    while (i < len) {
+      ASSERT_EQ(0, rados_omap_get_next(iter, &key, &val, &val_len));
+      if (val_len == 0 && key == NULL && val == NULL)
+	break;
+      if (key)
+	EXPECT_EQ(std::string(keys[i]), std::string(key));
+      else
+	EXPECT_EQ(keys[i], key);
+      ASSERT_EQ(0, memcmp(vals[i], val, val_len));
+      ASSERT_EQ(lens[i], val_len);
+      ++i;
+    }
+    ASSERT_EQ(i, len);
+    ASSERT_EQ(0, rados_omap_get_next(iter, &key, &val, &val_len));
+    ASSERT_EQ((char*)NULL, key);
+    ASSERT_EQ((char*)NULL, val);
+    ASSERT_EQ(0u, val_len);
+    rados_omap_get_end(iter);
+  }
+
+  // these two used to test omap funcs that accept length for both keys and vals
+  void fetch_and_verify_omap_vals2(char const* const* keys,
+                                  char const* const* vals,
+                                  const size_t *keylens,
+                                  const size_t *vallens,
+                                  size_t len)
+  {
+    rados_omap_iter_t iter_vals_by_key;
+    int r_vals_by_key;
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_omap_get_vals_by_keys2(op, keys, len, keylens,
+                                        &iter_vals_by_key, &r_vals_by_key);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    rados_release_read_op(op);
+    ASSERT_EQ(0, r_vals_by_key);
+
+    compare_omap_vals2(keys, vals, keylens, vallens, len, iter_vals_by_key);
+  }
+
+  void compare_omap_vals2(char const* const* keys,
+                         char const* const* vals,
+                         const size_t *keylens,
+                         const size_t *vallens,
+                         size_t len,
+                         rados_omap_iter_t iter)
+  {
+    size_t i = 0;
+    char *key = NULL;
+    char *val = NULL;
+    size_t key_len = 0;
+    size_t val_len = 0;
+    ASSERT_EQ(len, rados_omap_iter_size(iter));
+    while (i < len) {
+      ASSERT_EQ(0, rados_omap_get_next2(iter, &key, &val, &key_len, &val_len));
+      if (key_len == 0 && val_len == 0 && key == NULL && val == NULL)
+        break;
+      if (key)
+        EXPECT_EQ(std::string(keys[i], keylens[i]), std::string(key, key_len));
+      else
+        EXPECT_EQ(keys[i], key);
+      ASSERT_EQ(val_len, vallens[i]);
+      ASSERT_EQ(key_len, keylens[i]);
+      ASSERT_EQ(0, memcmp(vals[i], val, val_len));
+      ++i;
+    }
+    ASSERT_EQ(i, len);
+    ASSERT_EQ(0, rados_omap_get_next2(iter, &key, &val, &key_len, &val_len));
+    ASSERT_EQ((char*)NULL, key);
+    ASSERT_EQ((char*)NULL, val);
+    ASSERT_EQ(0u, key_len);
+    ASSERT_EQ(0u, val_len);
+    rados_omap_get_end(iter);
+  }
+
+  void compare_xattrs(char const* const* keys,
+		      char const* const* vals,
+		      const size_t *lens,
+		      size_t len,
+		      rados_xattrs_iter_t iter)
+  {
+    size_t i = 0;
+    char *key = NULL;
+    char *val = NULL;
+    size_t val_len = 0;
+    while (i < len) {
+      ASSERT_EQ(0, rados_getxattrs_next(iter, (const char**) &key,
+					(const char**) &val, &val_len));
+      if (key == NULL)
+	break;
+      EXPECT_EQ(std::string(keys[i]), std::string(key));
+      if (val != NULL) {
+        EXPECT_EQ(0, memcmp(vals[i], val, val_len));
+      }
+      EXPECT_EQ(lens[i], val_len);
+      ++i;
+    }
+    ASSERT_EQ(i, len);
+    ASSERT_EQ(0, rados_getxattrs_next(iter, (const char**)&key,
+				      (const char**)&val, &val_len));
+    ASSERT_EQ((char*)NULL, key);
+    ASSERT_EQ((char*)NULL, val);
+    ASSERT_EQ(0u, val_len);
+    rados_getxattrs_end(iter);
+  }
+};
+
+TEST_F(CReadOpsTest, NewDelete) {
+  rados_read_op_t op = rados_create_read_op();
+  ASSERT_TRUE(op);
+  rados_release_read_op(op);
+}
+
+TEST_F(CReadOpsTest, SetOpFlags) {
+  write_object();
+
+  rados_read_op_t op = rados_create_read_op();
+  size_t bytes_read = 0;
+  char *out = NULL;
+  int rval = 0;
+  rados_read_op_exec(op, "rbd", "get_id", NULL, 0, &out,
+		     &bytes_read, &rval);
+  rados_read_op_set_flags(op, LIBRADOS_OP_FLAG_FAILOK);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(-EIO, rval);
+  EXPECT_EQ(0u, bytes_read);
+  EXPECT_EQ((char*)NULL, out);
+  rados_release_read_op(op);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, AssertExists) {
+  rados_read_op_t op = rados_create_read_op();
+  rados_read_op_assert_exists(op);
+
+  ASSERT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  op = rados_create_read_op();
+  rados_read_op_assert_exists(op);
+
+  rados_completion_t completion;
+  ASSERT_EQ(0, rados_aio_create_completion(NULL, NULL, NULL, &completion));
+  auto sg = make_scope_guard([&] { rados_aio_release(completion); });
+  ASSERT_EQ(0, rados_aio_read_op_operate(op, ioctx, completion, obj, 0));
+  rados_aio_wait_for_complete(completion);
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(completion));
+  rados_release_read_op(op);
+
+  write_object();
+
+  op = rados_create_read_op();
+  rados_read_op_assert_exists(op);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, AssertVersion) {
+  write_object();
+  // Write to the object a second time to guarantee that its
+  // version number is greater than 0
+  write_object();
+  uint64_t v = rados_get_last_version(ioctx);
+
+  rados_read_op_t op = rados_create_read_op();
+  rados_read_op_assert_version(op, v+1);
+  ASSERT_EQ(-EOVERFLOW, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  op = rados_create_read_op();
+  rados_read_op_assert_version(op, v-1);
+  ASSERT_EQ(-ERANGE, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  op = rados_create_read_op();
+  rados_read_op_assert_version(op, v);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, CmpXattr) {
+  write_object();
+
+  char buf[len];
+  memset(buf, 0xcc, sizeof(buf));
+
+  const char *xattr = "test";
+  rados_setxattr(ioctx, obj, xattr, buf, sizeof(buf));
+
+  // equal value
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LTE));
+
+  // < value
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf) - 1, LIBRADOS_CMPXATTR_OP_LTE));
+
+  // > value
+  memset(buf, 0xcd, sizeof(buf));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, sizeof(buf), LIBRADOS_CMPXATTR_OP_LTE));
+
+  // check that null bytes are compared correctly
+  rados_setxattr(ioctx, obj, xattr, "\0\0", 2);
+  buf[0] = '\0';
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LTE));
+
+  buf[1] = '\0';
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_EQ));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_NE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_GTE));
+  EXPECT_EQ(-ECANCELED, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LT));
+  EXPECT_EQ(1, cmp_xattr(xattr, buf, 2, LIBRADOS_CMPXATTR_OP_LTE));
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, Read) {
+  write_object();
+
+  char buf[len];
+  // check that using read_ops returns the same data with
+  // or without bytes_read and rval out params
+  {
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_read(op, 0, len, buf, NULL, NULL);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    int rval;
+    rados_read_op_read(op, 0, len, buf, NULL, &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(0, rval);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    size_t bytes_read = 0;
+    rados_read_op_read(op, 0, len, buf, &bytes_read, NULL);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(len, bytes_read);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    size_t bytes_read = 0;
+    int rval;
+    rados_read_op_read(op, 0, len, buf, &bytes_read, &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(len, bytes_read);
+    ASSERT_EQ(0, rval);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    size_t bytes_read = 0;
+    int rval;
+    rados_read_op_read(op, 0, len, buf, &bytes_read, &rval);
+    rados_read_op_set_flags(op, LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(len, bytes_read);
+    ASSERT_EQ(0, rval);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, Checksum) {
+  write_object();
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    ceph_le64 init_value(-1);
+    rados_read_op_checksum(op, LIBRADOS_CHECKSUM_TYPE_XXHASH64,
+			   reinterpret_cast<char *>(&init_value),
+			   sizeof(init_value), 0, len, 0, NULL, 0, NULL);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    rados_release_read_op(op);
+  }
+
+  {
+    ceph_le32 init_value(-1);
+    ceph_le32 crc[2];
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_checksum(op, LIBRADOS_CHECKSUM_TYPE_CRC32C,
+			   reinterpret_cast<char *>(&init_value),
+			   sizeof(init_value), 0, len, 0,
+			   reinterpret_cast<char *>(&crc), sizeof(crc),
+			   nullptr);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(1U, crc[0]);
+    uint32_t expected_crc = ceph_crc32c(
+      -1, reinterpret_cast<const uint8_t*>(data), static_cast<uint32_t>(len));
+    ASSERT_EQ(expected_crc, crc[1]);
+    rados_release_read_op(op);
+  }
+
+  {
+    ceph_le32 init_value(-1);
+    int rval;
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_checksum(op, LIBRADOS_CHECKSUM_TYPE_XXHASH32,
+                           reinterpret_cast<char *>(&init_value),
+			   sizeof(init_value), 0, len, 0, nullptr, 0, &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(0, rval);
+    rados_release_read_op(op);
+  }
+
+  {
+    ceph_le32 init_value(-1);
+    ceph_le32 crc[3];
+    int rval;
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_checksum(op, LIBRADOS_CHECKSUM_TYPE_CRC32C,
+                           reinterpret_cast<char *>(&init_value),
+			   sizeof(init_value), 0, len, 4,
+			   reinterpret_cast<char *>(&crc), sizeof(crc), &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(2U, crc[0]);
+    uint32_t expected_crc[2];
+    expected_crc[0] = ceph_crc32c(
+      -1, reinterpret_cast<const uint8_t*>(data), 4U);
+    expected_crc[1] = ceph_crc32c(
+      -1, reinterpret_cast<const uint8_t*>(data + 4), 4U);
+    ASSERT_EQ(expected_crc[0], crc[1]);
+    ASSERT_EQ(expected_crc[1], crc[2]);
+    ASSERT_EQ(0, rval);
+    rados_release_read_op(op);
+  }
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, RWOrderedRead) {
+  write_object();
+
+  char buf[len];
+  rados_read_op_t op = rados_create_read_op();
+  size_t bytes_read = 0;
+  int rval;
+  rados_read_op_read(op, 0, len, buf, &bytes_read, &rval);
+  rados_read_op_set_flags(op, LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj,
+				     LIBRADOS_OPERATION_ORDER_READS_WRITES));
+  ASSERT_EQ(len, bytes_read);
+  ASSERT_EQ(0, rval);
+  ASSERT_EQ(0, memcmp(data, buf, len));
+  rados_release_read_op(op);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, ShortRead) {
+  write_object();
+
+  char buf[len * 2];
+  // check that using read_ops returns the same data with
+  // or without bytes_read and rval out params
+  {
+    rados_read_op_t op = rados_create_read_op();
+    rados_read_op_read(op, 0, len * 2, buf, NULL, NULL);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    int rval;
+    rados_read_op_read(op, 0, len * 2, buf, NULL, &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(0, rval);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    size_t bytes_read = 0;
+    rados_read_op_read(op, 0, len * 2, buf, &bytes_read, NULL);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(len, bytes_read);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  {
+    rados_read_op_t op = rados_create_read_op();
+    size_t bytes_read = 0;
+    int rval;
+    rados_read_op_read(op, 0, len * 2, buf, &bytes_read, &rval);
+    ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+    ASSERT_EQ(len, bytes_read);
+    ASSERT_EQ(0, rval);
+    ASSERT_EQ(0, memcmp(data, buf, len));
+    rados_release_read_op(op);
+  }
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, Exec) {
+  // create object so we don't get -ENOENT
+  write_object();
+
+  rados_read_op_t op = rados_create_read_op();
+  ASSERT_TRUE(op);
+  size_t bytes_read = 0;
+  char *out = NULL;
+  int rval = 0;
+  rados_read_op_exec(op, "rbd", "get_all_features", NULL, 0, &out,
+		     &bytes_read, &rval);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+  EXPECT_EQ(0, rval);
+  EXPECT_TRUE(out);
+  uint64_t features;
+  EXPECT_EQ(sizeof(features), bytes_read);
+  // make sure buffer is at least as long as it claims
+  bufferlist bl;
+  bl.append(out, bytes_read);
+  auto it = bl.cbegin();
+  ceph::decode(features, it);
+  ASSERT_EQ(RBD_FEATURES_ALL, features);
+  rados_buffer_free(out);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, ExecUserBuf) {
+  // create object so we don't get -ENOENT
+  write_object();
+
+  rados_read_op_t op = rados_create_read_op();
+  size_t bytes_read = 0;
+  uint64_t features;
+  char out[sizeof(features)];
+  int rval = 0;
+  rados_read_op_exec_user_buf(op, "rbd", "get_all_features", NULL, 0, out,
+			      sizeof(out), &bytes_read, &rval);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+  EXPECT_EQ(0, rval);
+  EXPECT_EQ(sizeof(features), bytes_read);
+
+  // buffer too short
+  bytes_read = 1024;
+  op = rados_create_read_op();
+  rados_read_op_exec_user_buf(op, "rbd", "get_all_features", NULL, 0, out,
+			      sizeof(features) - 1, &bytes_read, &rval);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+  EXPECT_EQ(0u, bytes_read);
+  EXPECT_EQ(-ERANGE, rval);
+
+  // input buffer and no rval or bytes_read
+  op = rados_create_read_op();
+  rados_read_op_exec_user_buf(op, "rbd", "get_all_features", out, sizeof(out),
+			      out, sizeof(out), NULL, NULL);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, Stat) {
+  rados_read_op_t op = rados_create_read_op();
+  uint64_t size = 1;
+  int rval = 0;
+  rados_read_op_stat(op, &size, NULL, &rval);
+  EXPECT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(-EIO, rval);
+  EXPECT_EQ(1u, size);
+  rados_release_read_op(op);
+
+  time_t ts = 1457129052;
+  rados_write_op_t wop = rados_create_write_op();
+  rados_write_op_write(wop, data, len, 0);
+  ASSERT_EQ(0, rados_write_op_operate(wop, ioctx, obj, &ts, 0));
+  rados_release_write_op(wop);
+
+  time_t ts2;
+  op = rados_create_read_op();
+  rados_read_op_stat(op, &size, &ts2, &rval);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(0, rval);
+  EXPECT_EQ(len, size);
+  EXPECT_EQ(ts2, ts);
+  rados_release_read_op(op);
+
+  op = rados_create_read_op();
+  rados_read_op_stat(op, NULL, NULL, NULL);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  remove_object();
+
+  op = rados_create_read_op();
+  rados_read_op_stat(op, NULL, NULL, NULL);
+  EXPECT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+}
+
+TEST_F(CReadOpsTest, Stat2) {
+  rados_read_op_t op = rados_create_read_op();
+  uint64_t size = 1;
+  int rval = 0;
+  rados_read_op_stat2(op, &size, NULL, &rval);
+  EXPECT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(-EIO, rval);
+  EXPECT_EQ(1u, size);
+  rados_release_read_op(op);
+
+  struct timespec ts;
+  ts.tv_sec = 1457129052;
+  ts.tv_nsec = 123456789;
+  rados_write_op_t wop = rados_create_write_op();
+  rados_write_op_write(wop, data, len, 0);
+  ASSERT_EQ(0, rados_write_op_operate2(wop, ioctx, obj, &ts, 0));
+  rados_release_write_op(wop);
+
+  struct timespec ts2 = {};
+  op = rados_create_read_op();
+  rados_read_op_stat2(op, &size, &ts2, &rval);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(0, rval);
+  EXPECT_EQ(len, size);
+  EXPECT_EQ(ts2.tv_sec, ts.tv_sec);
+  EXPECT_EQ(ts2.tv_nsec, ts.tv_nsec);
+  rados_release_read_op(op);
+
+  op = rados_create_read_op();
+  rados_read_op_stat2(op, NULL, NULL, NULL);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  remove_object();
+
+  op = rados_create_read_op();
+  rados_read_op_stat2(op, NULL, NULL, NULL);
+  EXPECT_EQ(-ENOENT, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+}
+
+TEST_F(CReadOpsTest, Omap) {
+  char *keys[] = {(char*)"bar",
+		  (char*)"foo",
+		  (char*)"test1",
+		  (char*)"test2"};
+  char *vals[] = {(char*)"",
+		  (char*)"\0",
+		  (char*)"abc",
+		  (char*)"va\0lue"};
+  size_t lens[] = {0, 1, 3, 6};
+
+  // check for -ENOENT before the object exists and when it exists
+  // with no omap entries
+  rados_omap_iter_t iter_vals;
+  rados_read_op_t rop = rados_create_read_op();
+  rados_read_op_omap_get_vals2(rop, "", "", 10, &iter_vals, NULL, NULL);
+  ASSERT_EQ(-ENOENT, rados_read_op_operate(rop, ioctx, obj, 0));
+  rados_release_read_op(rop);
+  compare_omap_vals(NULL, NULL, NULL, 0, iter_vals);
+
+  write_object();
+
+  fetch_and_verify_omap_vals(NULL, NULL, NULL, 0);
+
+  // write and check for the k/v pairs
+  rados_write_op_t op = rados_create_write_op();
+  rados_write_op_omap_set(op, keys, vals, lens, 4);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  fetch_and_verify_omap_vals(keys, vals, lens, 4);
+
+  rados_omap_iter_t iter_keys;
+  int r_vals = -1, r_keys = -1;
+  rop = rados_create_read_op();
+  rados_read_op_omap_get_vals2(rop, "", "test", 1, &iter_vals, NULL, &r_vals);
+  rados_read_op_omap_get_keys2(rop, "test", 1, &iter_keys, NULL, &r_keys);
+  ASSERT_EQ(0, rados_read_op_operate(rop, ioctx, obj, 0));
+  rados_release_read_op(rop);
+  EXPECT_EQ(0, r_vals);
+  EXPECT_EQ(0, r_keys);
+  EXPECT_EQ(1u, rados_omap_iter_size(iter_vals));
+  EXPECT_EQ(1u, rados_omap_iter_size(iter_keys));
+
+  compare_omap_vals(&keys[2], &vals[2], &lens[2], 1, iter_vals);
+  compare_omap_vals(&keys[2], &vals[0], &lens[0], 1, iter_keys);
+
+  // check omap_cmp finds all expected values
+  rop = rados_create_read_op();
+  int rvals[4];
+  for (int i = 0; i < 4; ++i)
+    rados_read_op_omap_cmp(rop, keys[i], LIBRADOS_CMPXATTR_OP_EQ,
+			   vals[i], lens[i], &rvals[i]);
+  EXPECT_EQ(0, rados_read_op_operate(rop, ioctx, obj, 0));
+  rados_release_read_op(rop);
+  for (int i = 0; i < 4; ++i)
+    EXPECT_EQ(0, rvals[i]);
+
+  // try to remove keys with a guard that should fail
+  op = rados_create_write_op();
+  rados_write_op_omap_cmp(op, keys[2], LIBRADOS_CMPXATTR_OP_LT,
+			  vals[2], lens[2], &r_vals);
+  rados_write_op_omap_rm_keys(op, keys, 2);
+  EXPECT_EQ(-ECANCELED, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  // see http://tracker.ceph.com/issues/19518
+  //ASSERT_EQ(-ECANCELED, r_vals);
+
+  // verifying the keys are still there, and then remove them
+  op = rados_create_write_op();
+  rados_write_op_omap_cmp(op, keys[0], LIBRADOS_CMPXATTR_OP_EQ,
+			  vals[0], lens[0], NULL);
+  rados_write_op_omap_cmp(op, keys[1], LIBRADOS_CMPXATTR_OP_EQ,
+			  vals[1], lens[1], NULL);
+  rados_write_op_omap_rm_keys(op, keys, 2);
+  EXPECT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  fetch_and_verify_omap_vals(&keys[2], &vals[2], &lens[2], 2);
+
+  // clear the rest and check there are none left
+  op = rados_create_write_op();
+  rados_write_op_omap_clear(op);
+  EXPECT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  fetch_and_verify_omap_vals(NULL, NULL, NULL, 0);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, OmapNuls) {
+  char *keys[] = {(char*)"1\0bar",
+                      (char*)"2baar\0",
+                      (char*)"3baa\0rr"};
+  char *vals[] = {(char*)"_\0var",
+                      (char*)"_vaar\0",
+                      (char*)"__vaa\0rr"};
+  size_t nklens[] = {5, 6, 7};
+  size_t nvlens[] = {5, 6, 8};
+  const int paircount = 3;
+
+  // check for -ENOENT before the object exists and when it exists
+  // with no omap entries
+  rados_omap_iter_t iter_vals;
+  rados_read_op_t rop = rados_create_read_op();
+  rados_read_op_omap_get_vals2(rop, "", "", 10, &iter_vals, NULL, NULL);
+  ASSERT_EQ(-ENOENT, rados_read_op_operate(rop, ioctx, obj, 0));
+  rados_release_read_op(rop);
+  compare_omap_vals(NULL, NULL, NULL, 0, iter_vals);
+
+  write_object();
+
+  fetch_and_verify_omap_vals(NULL, NULL, NULL, 0);
+
+  // write and check for the k/v pairs
+  rados_write_op_t op = rados_create_write_op();
+  rados_write_op_omap_set2(op, keys, vals, nklens, nvlens, paircount);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  fetch_and_verify_omap_vals2(keys, vals, nklens, nvlens, paircount);
+
+  // check omap_cmp finds all expected values
+  rop = rados_create_read_op();
+  int rvals[4];
+  for (int i = 0; i < paircount; ++i)
+    rados_read_op_omap_cmp2(rop, keys[i], LIBRADOS_CMPXATTR_OP_EQ,
+                           vals[i], nklens[i], nvlens[i], &rvals[i]);
+  EXPECT_EQ(0, rados_read_op_operate(rop, ioctx, obj, 0));
+  rados_release_read_op(rop);
+  for (int i = 0; i < paircount; ++i)
+    EXPECT_EQ(0, rvals[i]);
+
+  // try to remove keys with a guard that should fail
+  int r_vals = -1;
+  op = rados_create_write_op();
+  rados_write_op_omap_cmp2(op, keys[2], LIBRADOS_CMPXATTR_OP_LT,
+                          vals[2], nklens[2], nvlens[2], &r_vals);
+  rados_write_op_omap_rm_keys(op, keys, 2);
+  EXPECT_EQ(-ECANCELED, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  // verifying the keys are still there, and then remove them
+  op = rados_create_write_op();
+  rados_write_op_omap_cmp2(op, keys[0], LIBRADOS_CMPXATTR_OP_EQ,
+                          vals[0], nklens[0], nvlens[0], NULL);
+  rados_write_op_omap_cmp2(op, keys[1], LIBRADOS_CMPXATTR_OP_EQ,
+                          vals[1], nklens[1], nvlens[1], NULL);
+  rados_write_op_omap_rm_keys2(op, keys, nklens, 2);
+  EXPECT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  fetch_and_verify_omap_vals2(&keys[2], &vals[2], &nklens[2], &nvlens[2], 1);
+
+  // clear the rest and check there are none left
+  op = rados_create_write_op();
+  rados_write_op_omap_clear(op);
+  EXPECT_EQ(0, rados_write_op_operate(op, ioctx, obj, NULL, 0));
+  rados_release_write_op(op);
+
+  fetch_and_verify_omap_vals(NULL, NULL, NULL, 0);
+
+  remove_object();
+}
+TEST_F(CReadOpsTest, GetXattrs) {
+  write_object();
+
+  char *keys[] = {(char*)"bar",
+		  (char*)"foo",
+		  (char*)"test1",
+		  (char*)"test2"};
+  char *vals[] = {(char*)"",
+		  (char*)"\0",
+		  (char*)"abc",
+		  (char*)"va\0lue"};
+  size_t lens[] = {0, 1, 3, 6};
+
+  int rval = 1;
+  rados_read_op_t op = rados_create_read_op();
+  rados_xattrs_iter_t it;
+  rados_read_op_getxattrs(op, &it, &rval);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(0, rval);
+  rados_release_read_op(op);
+  compare_xattrs(keys, vals, lens, 0, it);
+
+  for (int i = 0; i < 4; ++i)
+    rados_setxattr(ioctx, obj, keys[i], vals[i], lens[i]);
+
+  rval = 1;
+  op = rados_create_read_op();
+  rados_read_op_getxattrs(op, &it, &rval);
+  EXPECT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  EXPECT_EQ(0, rval);
+  rados_release_read_op(op);
+  compare_xattrs(keys, vals, lens, 4, it);
+
+  remove_object();
+}
+
+TEST_F(CReadOpsTest, CmpExt) {
+  char buf[len];
+  size_t bytes_read = 0;
+  int cmpext_val = 0;
+  int read_val = 0;
+
+  write_object();
+
+  // cmpext with match should ensure that the following read is successful
+  rados_read_op_t op = rados_create_read_op();
+  ASSERT_TRUE(op);
+  // @obj, @data and @len correspond to object initialised by write_object()
+  rados_read_op_cmpext(op, data, len, 0, &cmpext_val);
+  rados_read_op_read(op, 0, len, buf, &bytes_read, &read_val);
+  ASSERT_EQ(0, rados_read_op_operate(op, ioctx, obj, 0));
+  ASSERT_EQ(len, bytes_read);
+  ASSERT_EQ(0, memcmp(data, buf, len));
+  ASSERT_EQ(cmpext_val, 0);
+  rados_release_read_op(op);
+
+  // cmpext with mismatch should fail and fill mismatch_buf accordingly
+  memset(buf, 0, sizeof(buf));
+  bytes_read = 0;
+  cmpext_val = 0;
+  read_val = 0;
+  op = rados_create_read_op();
+  ASSERT_TRUE(op);
+  // @obj, @data and @len correspond to object initialised by write_object()
+  rados_read_op_cmpext(op, "mismatch", strlen("mismatch"), 0, &cmpext_val);
+  rados_read_op_read(op, 0, len, buf, &bytes_read, &read_val);
+  ASSERT_EQ(-MAX_ERRNO, rados_read_op_operate(op, ioctx, obj, 0));
+  rados_release_read_op(op);
+
+  ASSERT_EQ(-MAX_ERRNO, cmpext_val);
+
+  remove_object();
+}
diff --git a/src/test/librados/c_write_operations.cc b/src/test/librados/c_write_operations.cc
new file mode 100644
index 000000000..558d0942f
--- /dev/null
+++ b/src/test/librados/c_write_operations.cc
@@ -0,0 +1,279 @@
+// Tests for the C API coverage of atomic write operations
+
+#include <errno.h>
+#include "gtest/gtest.h"
+#include "include/err.h"
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+
+TEST(LibradosCWriteOps, NewDelete) {
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_release_write_op(op);
+}
+
+TEST(LibRadosCWriteOps, assertExists) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_assert_exists(op);
+
+  // -2, ENOENT
+  ASSERT_EQ(-2, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  rados_write_op_t op2 = rados_create_write_op();
+  ASSERT_TRUE(op2);
+  rados_write_op_assert_exists(op2);
+
+  rados_completion_t completion;
+  ASSERT_EQ(0, rados_aio_create_completion(NULL, NULL, NULL, &completion));
+  ASSERT_EQ(0, rados_aio_write_op_operate(op2, ioctx, completion, "test", NULL, 0));
+  rados_aio_wait_for_complete(completion);
+  ASSERT_EQ(-2, rados_aio_get_return_value(completion));
+  rados_aio_release(completion);
+
+  rados_ioctx_destroy(ioctx);
+  rados_release_write_op(op2);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, WriteOpAssertVersion) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  // Write to the object a second time to guarantee that its
+  // version number is greater than 0
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_write_full(op, "hi", 2);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  uint64_t v = rados_get_last_version(ioctx);
+
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_assert_version(op, v+1);
+  ASSERT_EQ(-EOVERFLOW, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_assert_version(op, v-1);
+  ASSERT_EQ(-ERANGE, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_assert_version(op, v);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, Xattrs) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  // Create an object with an xattr
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
+  rados_write_op_setxattr(op, "key", "value", 5);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  // Check that xattr exists, if it does, delete it.
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_create(op, LIBRADOS_CREATE_IDEMPOTENT, NULL);
+  rados_write_op_cmpxattr(op, "key", LIBRADOS_CMPXATTR_OP_EQ, "value", 5);
+  rados_write_op_rmxattr(op, "key");
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  // Check the xattr exits, if it does, add it again (will fail) with -125
+  // (ECANCELED)
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_cmpxattr(op, "key", LIBRADOS_CMPXATTR_OP_EQ, "value", 5);
+  rados_write_op_setxattr(op, "key", "value", 5);
+  ASSERT_EQ(-ECANCELED, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+
+  rados_release_write_op(op);
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, Write) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  // Create an object, write and write full to it
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
+  rados_write_op_write(op, "four", 4, 0);
+  rados_write_op_write_full(op, "hi", 2);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  char hi[4];
+  ASSERT_EQ(2, rados_read(ioctx, "test", hi, 4, 0));
+  rados_release_write_op(op);
+
+  //create write op with iohint
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_write_full(op, "ceph", 4);
+  rados_write_op_set_flags(op, LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  ASSERT_EQ(4, rados_read(ioctx, "test", hi, 4, 0));
+  rados_release_write_op(op);
+
+  // Truncate and append
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_truncate(op, 1);
+  rados_write_op_append(op, "hi", 2);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  ASSERT_EQ(3, rados_read(ioctx, "test", hi, 4, 0));
+  rados_release_write_op(op);
+
+  // zero and remove
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_zero(op, 0, 3);
+  rados_write_op_remove(op);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  // ENOENT
+  ASSERT_EQ(-2, rados_read(ioctx, "test", hi, 4, 0));
+  rados_release_write_op(op);
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, Exec) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  int rval = 1;
+  rados_write_op_t op = rados_create_write_op();
+  rados_write_op_exec(op, "hello", "record_hello", "test", 4, &rval);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+  ASSERT_EQ(0, rval);
+
+  char hi[100];
+  ASSERT_EQ(12, rados_read(ioctx, "test", hi, 100, 0));
+  hi[12] = '\0';
+  ASSERT_EQ(0, strcmp("Hello, test!", hi));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, WriteSame) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  // Create an object, write to it using writesame
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
+  rados_write_op_writesame(op, "four", 4, 4 * 4, 0);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  char hi[4 * 4];
+  ASSERT_EQ(sizeof(hi), static_cast<std::size_t>(
+		rados_read(ioctx, "test", hi,sizeof(hi), 0)));
+  rados_release_write_op(op);
+  ASSERT_EQ(0, memcmp("fourfourfourfour", hi, sizeof(hi)));
+
+  // cleanup
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_remove(op);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosCWriteOps, CmpExt) {
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  rados_ioctx_create(cluster, pool_name.c_str(), &ioctx);
+
+  // create an object, write to it using writesame
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_create(op, LIBRADOS_CREATE_EXCLUSIVE, NULL);
+  rados_write_op_write(op, "four", 4, 0);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  rados_release_write_op(op);
+  char hi[4];
+  ASSERT_EQ(sizeof(hi), static_cast<std::size_t>(rados_read(ioctx, "test", hi, sizeof(hi), 0)));
+  ASSERT_EQ(0, memcmp("four", hi, sizeof(hi)));
+
+  // compare and overwrite on (expected) match
+  int val = 0;
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_cmpext(op, "four", 4, 0, &val);
+  rados_write_op_write(op, "five", 4, 0);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+  ASSERT_EQ(0, val);
+  rados_release_write_op(op);
+  ASSERT_EQ(sizeof(hi), static_cast<std::size_t>(rados_read(ioctx, "test", hi, sizeof(hi), 0)));
+  ASSERT_EQ(0, memcmp("five", hi, sizeof(hi)));
+
+  // compare and bail before write due to mismatch
+  val = 0;
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_cmpext(op, "four", 4, 0, &val);
+  rados_write_op_write(op, "six ", 4, 0);
+  ASSERT_EQ(-MAX_ERRNO - 1, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+
+  ASSERT_EQ(-MAX_ERRNO - 1, val);
+
+  // cleanup
+  op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_remove(op);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "test", NULL, 0));
+
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
diff --git a/src/test/librados/cls.cc b/src/test/librados/cls.cc
new file mode 100644
index 000000000..c4f24954d
--- /dev/null
+++ b/src/test/librados/cls.cc
@@ -0,0 +1,36 @@
+#include <errno.h>
+#include <map>
+#include <sstream>
+#include <string>
+
+#include "gtest/gtest.h"
+
+#include "include/rados/librados.hpp"
+#include "test/librados/test_cxx.h"
+
+using namespace librados;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+TEST(LibRadosCls, DNE) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  // create an object
+  string oid = "foo";
+  bufferlist bl;
+  ASSERT_EQ(0, ioctx.write(oid, bl, bl.length(), 0));
+
+  // call a bogus class
+  ASSERT_EQ(-EOPNOTSUPP, ioctx.exec(oid, "doesnotexistasdfasdf", "method", bl, bl));
+
+  // call a bogus method on existent class
+  ASSERT_EQ(-EOPNOTSUPP, ioctx.exec(oid, "lock", "doesnotexistasdfasdfasdf", bl, bl));
+
+  ioctx.close();
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
diff --git a/src/test/librados/cls_remote_reads.cc b/src/test/librados/cls_remote_reads.cc
new file mode 100644
index 000000000..4256c072f
--- /dev/null
+++ b/src/test/librados/cls_remote_reads.cc
@@ -0,0 +1,55 @@
+#include <set>
+#include <string>
+
+#include "common/ceph_json.h"
+#include "gtest/gtest.h"
+#include "test/librados/test_cxx.h"
+
+#include "crimson_utils.h"
+
+using namespace librados;
+
+TEST(ClsTestRemoteReads, TestGather) {
+  SKIP_IF_CRIMSON();
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  bufferlist in, out;
+  int object_size = 4096;
+  char buf[object_size];
+  memset(buf, 1, sizeof(buf));
+
+  // create source objects from which data are gathered
+  in.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write_full("src_object.1", in));
+  in.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write_full("src_object.2", in));
+  in.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write_full("src_object.3", in));
+
+  // construct JSON request passed to "test_gather" method, and in turn, to "test_read" method
+  JSONFormatter *formatter = new JSONFormatter(true);
+  formatter->open_object_section("foo");
+  std::set<std::string> src_objects;
+  src_objects.insert("src_object.1");
+  src_objects.insert("src_object.2");
+  src_objects.insert("src_object.3");
+  encode_json("src_objects", src_objects, formatter);
+  encode_json("cls", "test_remote_reads", formatter);
+  encode_json("method", "test_read", formatter);
+  encode_json("pool", pool_name, formatter);
+  formatter->close_section();
+  in.clear();
+  formatter->flush(in);
+
+  // create target object by combining data gathered from source objects using "test_read" method
+  ASSERT_EQ(0, ioctx.exec("tgt_object", "test_remote_reads", "test_gather", in, out));
+
+  // read target object and check its size
+  ASSERT_EQ(3*object_size, ioctx.read("tgt_object", out, 0, 0));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
diff --git a/src/test/librados/cmd.cc b/src/test/librados/cmd.cc
new file mode 100644
index 000000000..1d110f73b
--- /dev/null
+++ b/src/test/librados/cmd.cc
@@ -0,0 +1,229 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "test/librados/test.h"
+
+#include "gtest/gtest.h"
+#include <errno.h>
+#include <condition_variable>
+#include <map>
+#include <sstream>
+#include <string>
+
+using std::cout;
+using std::list;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+TEST(LibRadosCmd, MonDescribe) {
+  rados_t cluster;
+  ASSERT_EQ("", connect_cluster(&cluster));
+
+  char *buf, *st;
+  size_t buflen, stlen;
+  char *cmd[2];
+
+  cmd[1] = NULL;
+
+  cmd[0] = (char *)"{\"prefix\":\"get_command_descriptions\"}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  ASSERT_LT(0u, buflen);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"get_command_descriptions";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"asdfqwer";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "{}", 2, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "{}", 2, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"{}";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"{\"abc\":\"something\"}";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"{\"prefix\":\"\"}";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"{\"prefix\":\"    \"}";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"{\"prefix\":\";;;,,,;;,,\"}";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"{\"prefix\":\"extra command\"}";
+  ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  cmd[0] = (char *)"{\"prefix\":\"quorum_status\"}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  ASSERT_LT(0u, buflen);
+  //ASSERT_LT(0u, stlen);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+  rados_shutdown(cluster);
+}
+
+TEST(LibRadosCmd, OSDCmd) {
+  rados_t cluster;
+  ASSERT_EQ("", connect_cluster(&cluster));
+  int r;
+  char *buf, *st;
+  size_t buflen, stlen;
+  char *cmd[2];
+  cmd[1] = NULL;
+
+  // note: tolerate NXIO here in case the cluster is thrashing out underneath us.
+  cmd[0] = (char *)"asdfasdf";
+  r = rados_osd_command(cluster, 0, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+  ASSERT_TRUE(r == -22 || r == -ENXIO);
+  cmd[0] = (char *)"version";
+  r = rados_osd_command(cluster, 0, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+  ASSERT_TRUE(r == -22 || r == -ENXIO);
+  cmd[0] = (char *)"{\"prefix\":\"version\"}";
+  r = rados_osd_command(cluster, 0, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen);
+  ASSERT_TRUE((r == 0 && buflen > 0) || (r == -ENXIO && buflen == 0));
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+  rados_shutdown(cluster);
+}
+
+TEST(LibRadosCmd, PGCmd) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+
+  char *buf, *st;
+  size_t buflen, stlen;
+  char *cmd[2];
+  cmd[1] = NULL;
+
+  int64_t poolid = rados_pool_lookup(cluster, pool_name.c_str());
+  ASSERT_LT(0, poolid);
+
+  string pgid = stringify(poolid) + ".0";
+
+  cmd[0] = (char *)"asdfasdf";
+  // note: tolerate NXIO here in case the cluster is thrashing out underneath us.
+  int r = rados_pg_command(cluster, pgid.c_str(), (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen);
+  ASSERT_TRUE(r == -22 || r == -ENXIO);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  // make sure the pg exists on the osd before we query it
+  rados_ioctx_t io;
+  rados_ioctx_create(cluster, pool_name.c_str(), &io);
+  for (int i=0; i<100; i++) {
+    string oid = "obj" + stringify(i);
+    ASSERT_EQ(-ENOENT, rados_stat(io, oid.c_str(), NULL, NULL));
+  }
+  rados_ioctx_destroy(io);
+
+  string qstr = "{\"prefix\":\"pg\", \"cmd\":\"query\", \"pgid\":\"" +  pgid + "\"}";
+  cmd[0] = (char *)qstr.c_str();
+  // note: tolerate ENOENT/ENXIO here if hte osd is thrashing out underneath us
+  r = rados_pg_command(cluster, pgid.c_str(), (const char **)cmd, 1, "", 0,  &buf, &buflen, &st, &stlen);
+  ASSERT_TRUE(r == 0 || r == -ENOENT || r == -ENXIO);
+
+  ASSERT_LT(0u, buflen);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+struct Log {
+  list<string> log;
+  std::condition_variable cond;
+  std::mutex lock;
+
+  bool contains(const string& str) {
+    std::lock_guard<std::mutex> l(lock);
+    for (list<string>::iterator p = log.begin(); p != log.end(); ++p) {
+      if (p->find(str) != std::string::npos)
+	return true;
+    }
+    return false;
+  }
+};
+
+void log_cb(void *arg,
+	     const char *line,
+	     const char *who, uint64_t stampsec, uint64_t stamp_nsec,
+	     uint64_t seq, const char *level,
+	     const char *msg) {
+  Log *l = static_cast<Log *>(arg);
+  std::lock_guard<std::mutex> locker(l->lock);
+  l->log.push_back(line);
+  l->cond.notify_all();
+  cout << "got: " << line << std::endl;
+}
+
+TEST(LibRadosCmd, WatchLog) {
+  rados_t cluster;
+  ASSERT_EQ("", connect_cluster(&cluster));
+  char *buf, *st;
+  char *cmd[2];
+  cmd[1] = NULL;
+  size_t buflen, stlen;
+  Log l;
+
+  ASSERT_EQ(0, rados_monitor_log(cluster, "info", log_cb, &l));
+  cmd[0] = (char *)"{\"prefix\":\"log\", \"logtext\":[\"onexx\"]}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  for (int i=0; !l.contains("onexx"); i++) {
+    ASSERT_TRUE(i<100);
+    sleep(1);
+  }
+  ASSERT_TRUE(l.contains("onexx"));
+
+  cmd[0] = (char *)"{\"prefix\":\"log\", \"logtext\":[\"twoxx\"]}";
+  ASSERT_EQ(0, rados_monitor_log(cluster, "err", log_cb, &l));
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  sleep(2);
+  ASSERT_FALSE(l.contains("twoxx"));
+
+  ASSERT_EQ(0, rados_monitor_log(cluster, "info", log_cb, &l));
+  cmd[0] = (char *)"{\"prefix\":\"log\", \"logtext\":[\"threexx\"]}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  for (int i=0; !l.contains("threexx"); i++) {
+    ASSERT_TRUE(i<100);
+    sleep(1);
+  }
+
+  ASSERT_EQ(0, rados_monitor_log(cluster, "info", NULL, NULL));
+  cmd[0] = (char *)"{\"prefix\":\"log\", \"logtext\":[\"fourxx\"]}";
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+  sleep(2);
+  ASSERT_FALSE(l.contains("fourxx"));
+  rados_shutdown(cluster);
+}
diff --git a/src/test/librados/cmd_cxx.cc b/src/test/librados/cmd_cxx.cc
new file mode 100644
index 000000000..d67e2613b
--- /dev/null
+++ b/src/test/librados/cmd_cxx.cc
@@ -0,0 +1,92 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <errno.h>
+#include <condition_variable>
+#include <map>
+#include <sstream>
+#include <string>
+
+#include "gtest/gtest.h"
+
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "test/librados/test_cxx.h"
+
+using namespace librados;
+using std::map;
+using std::ostringstream;
+using std::string;
+
+TEST(LibRadosCmd, MonDescribePP) {
+  Rados cluster;
+  ASSERT_EQ("", connect_cluster_pp(cluster));
+  bufferlist inbl, outbl;
+  string outs;
+  ASSERT_EQ(0, cluster.mon_command("{\"prefix\": \"get_command_descriptions\"}",
+				   inbl, &outbl, &outs));
+  ASSERT_LT(0u, outbl.length());
+  ASSERT_LE(0u, outs.length());
+  cluster.shutdown();
+}
+
+TEST(LibRadosCmd, OSDCmdPP) {
+  Rados cluster;
+  ASSERT_EQ("", connect_cluster_pp(cluster));
+  int r;
+  bufferlist inbl, outbl;
+  string outs;
+  string cmd;
+
+  // note: tolerate NXIO here in case the cluster is thrashing out underneath us.
+  cmd = "asdfasdf";
+  r = cluster.osd_command(0, cmd, inbl, &outbl, &outs);
+  ASSERT_TRUE(r == -22 || r == -ENXIO);
+  cmd = "version";
+  r = cluster.osd_command(0, cmd, inbl, &outbl, &outs);
+  ASSERT_TRUE(r == -22 || r == -ENXIO);
+  cmd = "{\"prefix\":\"version\"}";
+  r = cluster.osd_command(0, cmd, inbl, &outbl, &outs);
+  ASSERT_TRUE((r == 0 && outbl.length() > 0) || (r == -ENXIO && outbl.length() == 0));
+  cluster.shutdown();
+}
+
+TEST(LibRadosCmd, PGCmdPP) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+
+  int r;
+  bufferlist inbl, outbl;
+  string outs;
+  string cmd;
+
+  int64_t poolid = cluster.pool_lookup(pool_name.c_str());
+  ASSERT_LT(0, poolid);
+
+  string pgid = stringify(poolid) + ".0";
+
+  cmd = "asdfasdf";
+  // note: tolerate NXIO here in case the cluster is thrashing out underneath us.
+  r = cluster.pg_command(pgid.c_str(), cmd, inbl, &outbl, &outs);
+  ASSERT_TRUE(r == -22 || r == -ENXIO);
+
+  // make sure the pg exists on the osd before we query it
+  IoCtx io;
+  cluster.ioctx_create(pool_name.c_str(), io);
+  for (int i=0; i<100; i++) {
+    string oid = "obj" + stringify(i);
+    ASSERT_EQ(-ENOENT, io.stat(oid, NULL, NULL));
+  }
+  io.close();
+
+  cmd = "{\"prefix\":\"pg\", \"cmd\":\"query\", \"pgid\":\"" +  pgid + "\"}";
+  // note: tolerate ENOENT/ENXIO here if hte osd is thrashing out underneath us
+  r = cluster.pg_command(pgid.c_str(), cmd, inbl, &outbl, &outs);
+  ASSERT_TRUE(r == 0 || r == -ENOENT || r == -ENXIO);
+
+  ASSERT_LT(0u, outbl.length());
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
diff --git a/src/test/librados/completion_speed.cc b/src/test/librados/completion_speed.cc
new file mode 100644
index 000000000..708a58425
--- /dev/null
+++ b/src/test/librados/completion_speed.cc
@@ -0,0 +1,38 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/rados/librados.hpp"
+#include "common/ceph_context.h"
+#include "common/Finisher.h"
+#include "librados/AioCompletionImpl.h"
+
+
+constexpr int max_completions = 10'000'000;
+int completed = 0;
+auto cct = (new CephContext(CEPH_ENTITY_TYPE_CLIENT))->get();
+Finisher f(cct);
+
+void completion_cb(librados::completion_t cb, void* arg) {
+  auto c = static_cast<librados::AioCompletion*>(arg);
+  delete c;
+  if (++completed < max_completions) {
+    auto aio = librados::Rados::aio_create_completion();
+    aio->set_complete_callback(static_cast<void*>(aio), &completion_cb);
+    f.queue(new librados::C_AioComplete(aio->pc));
+  }
+}
+
+int main(void) {
+  auto aio = librados::Rados::aio_create_completion();
+  aio->set_complete_callback(static_cast<void*>(aio), &completion_cb);
+  f.queue(new librados::C_AioComplete(aio->pc));
+  f.start();
+
+  while (completed < max_completions)
+    f.wait_for_empty();
+
+  f.stop();
+
+  assert(completed == max_completions);
+  cct->put();
+}
diff --git a/src/test/librados/crimson_utils.h b/src/test/librados/crimson_utils.h
new file mode 100644
index 000000000..4adcb69d9
--- /dev/null
+++ b/src/test/librados/crimson_utils.h
@@ -0,0 +1,15 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <cstdlib>
+
+static inline bool is_crimson_cluster() {
+  return getenv("CRIMSON_COMPAT") != nullptr;
+}
+
+#define SKIP_IF_CRIMSON()             \
+  if (is_crimson_cluster()) {         \
+    GTEST_SKIP() << "Not supported by crimson yet. Skipped"; \
+  }
diff --git a/src/test/librados/io.cc b/src/test/librados/io.cc
new file mode 100644
index 000000000..7814463d4
--- /dev/null
+++ b/src/test/librados/io.cc
@@ -0,0 +1,461 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
+#include <climits>
+
+#include "include/rados/librados.h"
+#include "include/encoding.h"
+#include "include/err.h"
+#include "include/scope_guard.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+#include <errno.h>
+#include "gtest/gtest.h"
+#include "crimson_utils.h"
+
+using std::string;
+
+typedef RadosTest LibRadosIo;
+typedef RadosTestEC LibRadosIoEC;
+
+TEST_F(LibRadosIo, SimpleWrite) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIo, TooBig) {
+  char buf[1] = { 0 };
+  ASSERT_EQ(-E2BIG, rados_write(ioctx, "A", buf, UINT_MAX, 0));
+  ASSERT_EQ(-E2BIG, rados_append(ioctx, "A", buf, UINT_MAX));
+  ASSERT_EQ(-E2BIG, rados_write_full(ioctx, "A", buf, UINT_MAX));
+  ASSERT_EQ(-E2BIG, rados_writesame(ioctx, "A", buf, sizeof(buf), UINT_MAX, 0));
+}
+
+TEST_F(LibRadosIo, ReadTimeout) {
+  char buf[128];
+  memset(buf, 'a', sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+
+  {
+    // set up a second client
+    rados_t cluster;
+    rados_ioctx_t ioctx;
+    ASSERT_EQ(0, rados_create(&cluster, "admin"));
+    ASSERT_EQ(0, rados_conf_read_file(cluster, NULL));
+    ASSERT_EQ(0, rados_conf_parse_env(cluster, NULL));
+    ASSERT_EQ(0, rados_conf_set(cluster, "rados_osd_op_timeout", "1")); // use any small value that will result in a timeout
+    ASSERT_EQ(0, rados_conf_set(cluster, "ms_inject_internal_delays", "2")); // create a 2 second delay
+    ASSERT_EQ(0, rados_connect(cluster));
+    ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+    rados_ioctx_set_namespace(ioctx, nspace.c_str());
+
+    // then we show that the buffer is changed after rados_read returned
+    // with a timeout
+    for (int i=0; i<5; i++) {
+      char buf2[sizeof(buf)];
+      memset(buf2, 0, sizeof(buf2));
+      int err = rados_read(ioctx, "foo", buf2, sizeof(buf2), 0);
+      if (err == -110) {
+	int startIndex = 0;
+	// find the index until which librados already read the object before the timeout occurred
+	for (unsigned b=0; b<sizeof(buf); b++) {
+	  if (buf2[b] != buf[b]) {
+	    startIndex = b;
+	    break;
+	  }
+	}
+
+	// wait some time to give librados a change to do something
+	sleep(1);
+
+	// then check if the buffer was changed after the call
+	if (buf2[startIndex] == 'a') {
+	  printf("byte at index %d was changed after the timeout to %d\n",
+		 startIndex, (int)buf[startIndex]);
+	  ASSERT_TRUE(0);
+	  break;
+	}
+      } else {
+	printf("no timeout :/\n");
+      }
+    }
+    rados_ioctx_destroy(ioctx);
+    rados_shutdown(cluster);
+  }
+}
+
+
+TEST_F(LibRadosIo, RoundTrip) {
+  char buf[128];
+  char buf2[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+
+  uint64_t off = 19;
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "bar", buf, sizeof(buf), off));
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "bar", buf2, sizeof(buf2), off));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+}
+
+TEST_F(LibRadosIo, Checksum) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+
+  uint32_t expected_crc = ceph_crc32c(-1, reinterpret_cast<const uint8_t*>(buf),
+                                      sizeof(buf));
+  ceph_le32 init_value(-1);
+  ceph_le32 crc[2];
+  ASSERT_EQ(0, rados_checksum(ioctx, "foo", LIBRADOS_CHECKSUM_TYPE_CRC32C,
+			      reinterpret_cast<char*>(&init_value),
+			      sizeof(init_value), sizeof(buf), 0, 0,
+			      reinterpret_cast<char*>(&crc), sizeof(crc)));
+  ASSERT_EQ(1U, crc[0]);
+  ASSERT_EQ(expected_crc, crc[1]);
+}
+
+TEST_F(LibRadosIo, OverlappingWriteRoundTrip) {
+  char buf[128];
+  char buf2[64];
+  char buf3[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+  memset(buf3, 0xdd, sizeof(buf3));
+  ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf3, buf2, sizeof(buf2)));
+  ASSERT_EQ(0, memcmp(buf3 + sizeof(buf2), buf, sizeof(buf) - sizeof(buf2)));
+}
+
+TEST_F(LibRadosIo, WriteFullRoundTrip) {
+  char buf[128];
+  char buf2[64];
+  char buf3[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_write_full(ioctx, "foo", buf2, sizeof(buf2)));
+  memset(buf3, 0x00, sizeof(buf3));
+  ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf2, buf3, sizeof(buf2)));
+}
+
+TEST_F(LibRadosIo, AppendRoundTrip) {
+  char buf[64];
+  char buf2[64];
+  char buf3[sizeof(buf) + sizeof(buf2)];
+  memset(buf, 0xde, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  memset(buf2, 0xad, sizeof(buf2));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf2, sizeof(buf2)));
+  memset(buf3, 0, sizeof(buf3));
+  ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(buf3 + sizeof(buf), buf2, sizeof(buf2)));
+}
+
+TEST_F(LibRadosIo, ZeroLenZero) {
+  rados_write_op_t op = rados_create_write_op();
+  ASSERT_TRUE(op);
+  rados_write_op_zero(op, 0, 0);
+  ASSERT_EQ(0, rados_write_op_operate(op, ioctx, "foo", NULL, 0));
+  rados_release_write_op(op);
+}
+
+TEST_F(LibRadosIo, TruncTest) {
+  char buf[128];
+  char buf2[sizeof(buf)];
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_trunc(ioctx, "foo", sizeof(buf) / 2));
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ((int)(sizeof(buf)/2), rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)/2));
+}
+
+TEST_F(LibRadosIo, RemoveTest) {
+  char buf[128];
+  char buf2[sizeof(buf)];
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ(-ENOENT, rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
+}
+
+TEST_F(LibRadosIo, XattrsRoundTrip) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(-ENODATA, rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ((int)sizeof(attr1_buf),
+	    rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(attr1_buf, buf, sizeof(attr1_buf)));
+}
+
+TEST_F(LibRadosIo, RmXattr) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0,
+      rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ(0, rados_rmxattr(ioctx, "foo", attr1));
+  ASSERT_EQ(-ENODATA, rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
+
+  // Test rmxattr on a removed object
+  char buf2[128];
+  char attr2[] = "attr2";
+  char attr2_buf[] = "foo bar baz";
+  memset(buf2, 0xbb, sizeof(buf2));
+  ASSERT_EQ(0, rados_write(ioctx, "foo_rmxattr", buf2, sizeof(buf2), 0));
+  ASSERT_EQ(0,
+      rados_setxattr(ioctx, "foo_rmxattr", attr2, attr2_buf, sizeof(attr2_buf)));
+  ASSERT_EQ(0, rados_remove(ioctx, "foo_rmxattr"));
+  ASSERT_EQ(-ENOENT, rados_rmxattr(ioctx, "foo_rmxattr", attr2));
+}
+
+TEST_F(LibRadosIo, XattrIter) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  char attr2[] = "attr2";
+  char attr2_buf[256];
+  for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
+    attr2_buf[j] = j % 0xff;
+  }
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr2, attr2_buf, sizeof(attr2_buf)));
+  rados_xattrs_iter_t iter;
+  ASSERT_EQ(0, rados_getxattrs(ioctx, "foo", &iter));
+  int num_seen = 0;
+  while (true) {
+    const char *name;
+    const char *val;
+    size_t len;
+    ASSERT_EQ(0, rados_getxattrs_next(iter, &name, &val, &len));
+    if (name == NULL) {
+      break;
+    }
+    ASSERT_LT(num_seen, 2);
+    if ((strcmp(name, attr1) == 0) && (val != NULL) && (memcmp(val, attr1_buf, len) == 0)) {
+      num_seen++;
+      continue;
+    }
+    else if ((strcmp(name, attr2) == 0) && (val != NULL) && (memcmp(val, attr2_buf, len) == 0)) {
+      num_seen++;
+      continue;
+    }
+    else {
+      ASSERT_EQ(0, 1);
+    }
+  }
+  rados_getxattrs_end(iter);
+}
+
+TEST_F(LibRadosIoEC, SimpleWrite) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIoEC, RoundTrip) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char buf2[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+
+  uint64_t off = 19;
+  ASSERT_EQ(-EOPNOTSUPP, rados_write(ioctx, "bar", buf, sizeof(buf), off));
+}
+
+TEST_F(LibRadosIoEC, OverlappingWriteRoundTrip) {
+  SKIP_IF_CRIMSON();
+  int bsize = alignment;
+  int dbsize = bsize * 2;
+  char *buf = (char *)new char[dbsize];
+  char *buf2 = (char *)new char[bsize];
+  char *buf3 = (char *)new char[dbsize];
+  auto cleanup = [&] {
+    delete[] buf;
+    delete[] buf2;
+    delete[] buf3;
+  };
+  scope_guard<decltype(cleanup)> sg(std::move(cleanup));
+  memset(buf, 0xcc, dbsize);
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, dbsize, 0));
+  memset(buf2, 0xdd, bsize);
+  ASSERT_EQ(-EOPNOTSUPP, rados_write(ioctx, "foo", buf2, bsize, 0));
+  memset(buf3, 0xdd, dbsize);
+  ASSERT_EQ(dbsize, rados_read(ioctx, "foo", buf3, dbsize, 0));
+  // Read the same as first write
+  ASSERT_EQ(0, memcmp(buf3, buf, dbsize));
+}
+
+TEST_F(LibRadosIoEC, WriteFullRoundTrip) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char buf2[64];
+  char buf3[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_write_full(ioctx, "foo", buf2, sizeof(buf2)));
+  memset(buf3, 0xee, sizeof(buf3));
+  ASSERT_EQ((int)sizeof(buf2), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf3, buf2, sizeof(buf2)));
+}
+
+TEST_F(LibRadosIoEC, AppendRoundTrip) {
+  SKIP_IF_CRIMSON();
+  char *buf = (char *)new char[alignment];
+  char *buf2 = (char *)new char[alignment];
+  char *buf3 = (char *)new char[alignment *2];
+  int uasize = alignment/2;
+  char *unalignedbuf = (char *)new char[uasize];
+  auto cleanup = [&] {
+    delete[] buf;
+    delete[] buf2;
+    delete[] buf3;
+    delete[] unalignedbuf;
+  };
+  scope_guard<decltype(cleanup)> sg(std::move(cleanup));
+  memset(buf, 0xde, alignment);
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, alignment));
+  memset(buf2, 0xad, alignment);
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf2, alignment));
+  memset(buf3, 0, alignment*2);
+  ASSERT_EQ((int)alignment*2, rados_read(ioctx, "foo", buf3, alignment*2, 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, alignment));
+  ASSERT_EQ(0, memcmp(buf3 + alignment, buf2, alignment));
+  memset(unalignedbuf, 0, uasize);
+  ASSERT_EQ(0, rados_append(ioctx, "foo", unalignedbuf, uasize));
+  ASSERT_EQ(-EOPNOTSUPP, rados_append(ioctx, "foo", unalignedbuf, uasize));
+}
+
+TEST_F(LibRadosIoEC, TruncTest) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char buf2[sizeof(buf)];
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(-EOPNOTSUPP, rados_trunc(ioctx, "foo", sizeof(buf) / 2));
+  memset(buf2, 0, sizeof(buf2));
+  // Same size
+  ASSERT_EQ((int)sizeof(buf), rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
+  // No change
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+}
+
+TEST_F(LibRadosIoEC, RemoveTest) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char buf2[sizeof(buf)];
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+  memset(buf2, 0, sizeof(buf2));
+  ASSERT_EQ(-ENOENT, rados_read(ioctx, "foo", buf2, sizeof(buf2), 0));
+}
+
+TEST_F(LibRadosIoEC, XattrsRoundTrip) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(-ENODATA, rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ((int)sizeof(attr1_buf),
+	    rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(attr1_buf, buf, sizeof(attr1_buf)));
+}
+
+TEST_F(LibRadosIoEC, RmXattr) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0,
+      rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ(0, rados_rmxattr(ioctx, "foo", attr1));
+  ASSERT_EQ(-ENODATA, rados_getxattr(ioctx, "foo", attr1, buf, sizeof(buf)));
+
+  // Test rmxattr on a removed object
+  char buf2[128];
+  char attr2[] = "attr2";
+  char attr2_buf[] = "foo bar baz";
+  memset(buf2, 0xbb, sizeof(buf2));
+  ASSERT_EQ(0, rados_write(ioctx, "foo_rmxattr", buf2, sizeof(buf2), 0));
+  ASSERT_EQ(0,
+      rados_setxattr(ioctx, "foo_rmxattr", attr2, attr2_buf, sizeof(attr2_buf)));
+  ASSERT_EQ(0, rados_remove(ioctx, "foo_rmxattr"));
+  ASSERT_EQ(-ENOENT, rados_rmxattr(ioctx, "foo_rmxattr", attr2));
+}
+
+TEST_F(LibRadosIoEC, XattrIter) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  char attr2[] = "attr2";
+  char attr2_buf[256];
+  for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
+    attr2_buf[j] = j % 0xff;
+  }
+  memset(buf, 0xaa, sizeof(buf));
+  ASSERT_EQ(0, rados_append(ioctx, "foo", buf, sizeof(buf)));
+  ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr1, attr1_buf, sizeof(attr1_buf)));
+  ASSERT_EQ(0, rados_setxattr(ioctx, "foo", attr2, attr2_buf, sizeof(attr2_buf)));
+  rados_xattrs_iter_t iter;
+  ASSERT_EQ(0, rados_getxattrs(ioctx, "foo", &iter));
+  int num_seen = 0;
+  while (true) {
+    const char *name;
+    const char *val;
+    size_t len;
+    ASSERT_EQ(0, rados_getxattrs_next(iter, &name, &val, &len));
+    if (name == NULL) {
+      break;
+    }
+    ASSERT_LT(num_seen, 2);
+    if ((strcmp(name, attr1) == 0) && (val != NULL) && (memcmp(val, attr1_buf, len) == 0)) {
+      num_seen++;
+      continue;
+    }
+    else if ((strcmp(name, attr2) == 0) && (val != NULL) && (memcmp(val, attr2_buf, len) == 0)) {
+      num_seen++;
+      continue;
+    }
+    else {
+      ASSERT_EQ(0, 1);
+    }
+  }
+  rados_getxattrs_end(iter);
+}
diff --git a/src/test/librados/io_cxx.cc b/src/test/librados/io_cxx.cc
new file mode 100644
index 000000000..d9606d16b
--- /dev/null
+++ b/src/test/librados/io_cxx.cc
@@ -0,0 +1,986 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
+#include <climits>
+#include <errno.h>
+
+#include "gtest/gtest.h"
+
+#include "include/rados/librados.hpp"
+#include "include/encoding.h"
+#include "include/err.h"
+#include "include/scope_guard.h"
+#include "test/librados/test_cxx.h"
+#include "test/librados/testcase_cxx.h"
+
+#include "crimson_utils.h"
+
+using namespace librados;
+using std::string;
+
+typedef RadosTestPP LibRadosIoPP;
+typedef RadosTestECPP LibRadosIoECPP;
+
+TEST_F(LibRadosIoPP, TooBigPP) {
+  IoCtx ioctx;
+  bufferlist bl;
+  ASSERT_EQ(-E2BIG, ioctx.write("foo", bl, UINT_MAX, 0));
+  ASSERT_EQ(-E2BIG, ioctx.append("foo", bl, UINT_MAX));
+  // ioctx.write_full no way to overflow bl.length()
+  ASSERT_EQ(-E2BIG, ioctx.writesame("foo", bl, UINT_MAX, 0));
+}
+
+TEST_F(LibRadosIoPP, SimpleWritePP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIoPP, ReadOpPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+
+  {
+      bufferlist op_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist op_bl;
+      ObjectReadOperation op;
+      op.read(0, 0, NULL, NULL); //len=0 mean read the whole object data.
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl, op_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist op_bl;
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, rval);
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl, op_bl;
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, rval);
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl1, read_bl2, op_bl;
+      int rval1 = 1000, rval2 = 1002;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl1, &rval1);
+      op.read(0, sizeof(buf), &read_bl2, &rval2);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), read_bl1.length());
+      ASSERT_EQ(sizeof(buf), read_bl2.length());
+      ASSERT_EQ(sizeof(buf) * 2, op_bl.length());
+      ASSERT_EQ(0, rval1);
+      ASSERT_EQ(0, rval2);
+      ASSERT_EQ(0, memcmp(read_bl1.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl2.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(op_bl.c_str() + sizeof(buf), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist op_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(0, rval);
+  }
+
+  {
+      bufferlist read_bl;
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(0, rval);
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl1, read_bl2;
+      int rval1 = 1000, rval2 = 1002;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl1, &rval1);
+      op.read(0, sizeof(buf), &read_bl2, &rval2);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(sizeof(buf), read_bl1.length());
+      ASSERT_EQ(sizeof(buf), read_bl2.length());
+      ASSERT_EQ(0, rval1);
+      ASSERT_EQ(0, rval2);
+      ASSERT_EQ(0, memcmp(read_bl1.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl2.c_str(), buf, sizeof(buf)));
+  }
+
+  // read into a preallocated buffer with a cached crc
+  {
+      bufferlist op_bl;
+      op_bl.append(std::string(sizeof(buf), 'x'));
+      ASSERT_NE(op_bl.crc32c(0), bl.crc32c(0));  // cache 'x' crc
+
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(op_bl.crc32c(0), bl.crc32c(0));
+  }
+}
+
+TEST_F(LibRadosIoPP, SparseReadOpPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+
+  {
+    std::map<uint64_t, uint64_t> extents;
+    bufferlist read_bl;
+    int rval = -1;
+    ObjectReadOperation op;
+    op.sparse_read(0, sizeof(buf), &extents, &read_bl, &rval);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, nullptr));
+    ASSERT_EQ(0, rval);
+    assert_eq_sparse(bl, extents, read_bl);
+  }
+  {
+    bufferlist bl;
+    bl.append(buf, sizeof(buf) / 2);
+
+    std::map<uint64_t, uint64_t> extents;
+    bufferlist read_bl;
+    int rval = -1;
+    ObjectReadOperation op;
+    op.sparse_read(0, sizeof(buf), &extents, &read_bl, &rval, sizeof(buf) / 2, 1);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, nullptr));
+    ASSERT_EQ(0, rval);
+    assert_eq_sparse(bl, extents, read_bl);
+  }
+}
+
+TEST_F(LibRadosIoPP, RoundTripPP) {
+  char buf[128];
+  Rados cluster;
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  bufferlist cl;
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", cl, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(buf, cl.c_str(), sizeof(buf)));
+}
+
+TEST_F(LibRadosIoPP, RoundTripPP2)
+{
+  bufferlist bl;
+  bl.append("ceph");
+  ObjectWriteOperation write;
+  write.write(0, bl);
+  write.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ASSERT_EQ(0, ioctx.operate("foo", &write));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  read.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_NOCACHE|LIBRADOS_OP_FLAG_FADVISE_RANDOM);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "ceph", 4));
+}
+
+TEST_F(LibRadosIoPP, Checksum) {
+  char buf[128];
+  Rados cluster;
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  bufferlist init_value_bl;
+  encode(static_cast<uint32_t>(-1), init_value_bl);
+  bufferlist csum_bl;
+  ASSERT_EQ(0, ioctx.checksum("foo", LIBRADOS_CHECKSUM_TYPE_CRC32C,
+			      init_value_bl, sizeof(buf), 0, 0, &csum_bl));
+  auto csum_bl_it = csum_bl.cbegin();
+  uint32_t csum_count;
+  decode(csum_count, csum_bl_it);
+  ASSERT_EQ(1U, csum_count);
+  uint32_t csum;
+  decode(csum, csum_bl_it);
+  ASSERT_EQ(bl.crc32c(-1), csum);
+}
+
+TEST_F(LibRadosIoPP, ReadIntoBufferlist) {
+
+  // here we test reading into a non-empty bufferlist referencing existing
+  // buffers
+
+  char buf[128];
+  Rados cluster;
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  bufferlist bl2;
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xbb, sizeof(buf2));
+  bl2.append(buffer::create_static(sizeof(buf2), buf2));
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl2, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(buf, buf2, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(buf, bl2.c_str(), sizeof(buf)));
+}
+
+TEST_F(LibRadosIoPP, OverlappingWriteRoundTripPP) {
+  char buf[128];
+  char buf2[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), 0));
+  bufferlist bl3;
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
+  ASSERT_EQ(0, memcmp(bl3.c_str() + sizeof(buf2), buf, sizeof(buf) - sizeof(buf2)));
+}
+
+TEST_F(LibRadosIoPP, WriteFullRoundTripPP) {
+  char buf[128];
+  char buf2[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.write_full("foo", bl2));
+  bufferlist bl3;
+  ASSERT_EQ((int)sizeof(buf2), ioctx.read("foo", bl3, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
+}
+
+TEST_F(LibRadosIoPP, WriteFullRoundTripPP2)
+{
+  bufferlist bl;
+  bl.append("ceph");
+  ObjectWriteOperation write;
+  write.write_full(bl);
+  write.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_NOCACHE);
+  ASSERT_EQ(0, ioctx.operate("foo", &write));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  read.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED|LIBRADOS_OP_FLAG_FADVISE_RANDOM);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "ceph", 4));
+}
+
+TEST_F(LibRadosIoPP, AppendRoundTripPP) {
+  char buf[64];
+  char buf2[64];
+  memset(buf, 0xde, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  memset(buf2, 0xad, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.append("foo", bl2, sizeof(buf2)));
+  bufferlist bl3;
+  ASSERT_EQ((int)(sizeof(buf) + sizeof(buf2)),
+	    ioctx.read("foo", bl3, (sizeof(buf) + sizeof(buf2)), 0));
+  const char *bl3_str = bl3.c_str();
+  ASSERT_EQ(0, memcmp(bl3_str, buf, sizeof(buf)));
+  ASSERT_EQ(0, memcmp(bl3_str + sizeof(buf), buf2, sizeof(buf2)));
+}
+
+TEST_F(LibRadosIoPP, TruncTestPP) {
+  char buf[128];
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl, sizeof(buf)));
+  ASSERT_EQ(0, ioctx.trunc("foo", sizeof(buf) / 2));
+  bufferlist bl2;
+  ASSERT_EQ((int)(sizeof(buf)/2), ioctx.read("foo", bl2, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(bl2.c_str(), buf, sizeof(buf)/2));
+}
+
+TEST_F(LibRadosIoPP, RemoveTestPP) {
+  char buf[128];
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  ASSERT_EQ(0, ioctx.remove("foo"));
+  bufferlist bl2;
+  ASSERT_EQ(-ENOENT, ioctx.read("foo", bl2, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIoPP, XattrsRoundTripPP) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  bufferlist bl2;
+  ASSERT_EQ(-ENODATA, ioctx.getxattr("foo", attr1, bl2));
+  bufferlist bl3;
+  bl3.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr1, bl3));
+  bufferlist bl4;
+  ASSERT_EQ((int)sizeof(attr1_buf),
+      ioctx.getxattr("foo", attr1, bl4));
+  ASSERT_EQ(0, memcmp(bl4.c_str(), attr1_buf, sizeof(attr1_buf)));
+}
+
+TEST_F(LibRadosIoPP, RmXattrPP) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  bufferlist bl2;
+  bl2.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr1, bl2));
+  ASSERT_EQ(0, ioctx.rmxattr("foo", attr1));
+  bufferlist bl3;
+  ASSERT_EQ(-ENODATA, ioctx.getxattr("foo", attr1, bl3));
+
+  // Test rmxattr on a removed object
+  char buf2[128];
+  char attr2[] = "attr2";
+  char attr2_buf[] = "foo bar baz";
+  memset(buf2, 0xbb, sizeof(buf2));
+  bufferlist bl21;
+  bl21.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo_rmxattr", bl21, sizeof(buf2), 0));
+  bufferlist bl22;
+  bl22.append(attr2_buf, sizeof(attr2_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo_rmxattr", attr2, bl22));
+  ASSERT_EQ(0, ioctx.remove("foo_rmxattr"));
+  ASSERT_EQ(-ENOENT, ioctx.rmxattr("foo_rmxattr", attr2));
+}
+
+TEST_F(LibRadosIoPP, XattrListPP) {
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  char attr2[] = "attr2";
+  char attr2_buf[256];
+  for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
+    attr2_buf[j] = j % 0xff;
+  }
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  bufferlist bl2;
+  bl2.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr1, bl2));
+  bufferlist bl3;
+  bl3.append(attr2_buf, sizeof(attr2_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr2, bl3));
+  std::map<std::string, bufferlist> attrset;
+  ASSERT_EQ(0, ioctx.getxattrs("foo", attrset));
+  for (std::map<std::string, bufferlist>::iterator i = attrset.begin();
+       i != attrset.end(); ++i) {
+    if (i->first == string(attr1)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr1_buf, sizeof(attr1_buf)));
+    }
+    else if (i->first == string(attr2)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr2_buf, sizeof(attr2_buf)));
+    }
+    else {
+      ASSERT_EQ(0, 1);
+    }
+  }
+}
+
+TEST_F(LibRadosIoECPP, SimpleWritePP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIoECPP, ReadOpPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+
+  {
+      bufferlist op_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+    bufferlist op_bl;
+    ObjectReadOperation op;
+    op.read(0, 0, NULL, NULL); //len=0 mean read the whole object data
+    ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+    ASSERT_EQ(sizeof(buf), op_bl.length());
+    ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl, op_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist op_bl;
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, rval);
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl, op_bl;
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, rval);
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl1, read_bl2, op_bl;
+      int rval1 = 1000, rval2 = 1002;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl1, &rval1);
+      op.read(0, sizeof(buf), &read_bl2, &rval2);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), read_bl1.length());
+      ASSERT_EQ(sizeof(buf), read_bl2.length());
+      ASSERT_EQ(sizeof(buf) * 2, op_bl.length());
+      ASSERT_EQ(0, rval1);
+      ASSERT_EQ(0, rval2);
+      ASSERT_EQ(0, memcmp(read_bl1.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl2.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(op_bl.c_str() + sizeof(buf), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist op_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(0, rval);
+  }
+
+  {
+      bufferlist read_bl;
+      int rval = 1000;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl, &rval);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(sizeof(buf), read_bl.length());
+      ASSERT_EQ(0, rval);
+      ASSERT_EQ(0, memcmp(read_bl.c_str(), buf, sizeof(buf)));
+  }
+
+  {
+      bufferlist read_bl1, read_bl2;
+      int rval1 = 1000, rval2 = 1002;
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), &read_bl1, &rval1);
+      op.read(0, sizeof(buf), &read_bl2, &rval2);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+      ASSERT_EQ(sizeof(buf), read_bl1.length());
+      ASSERT_EQ(sizeof(buf), read_bl2.length());
+      ASSERT_EQ(0, rval1);
+      ASSERT_EQ(0, rval2);
+      ASSERT_EQ(0, memcmp(read_bl1.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(0, memcmp(read_bl2.c_str(), buf, sizeof(buf)));
+  }
+
+  // read into a preallocated buffer with a cached crc
+  {
+      bufferlist op_bl;
+      op_bl.append(std::string(sizeof(buf), 'x'));
+      ASSERT_NE(op_bl.crc32c(0), bl.crc32c(0));  // cache 'x' crc
+
+      ObjectReadOperation op;
+      op.read(0, sizeof(buf), NULL, NULL);
+      ASSERT_EQ(0, ioctx.operate("foo", &op, &op_bl));
+
+      ASSERT_EQ(sizeof(buf), op_bl.length());
+      ASSERT_EQ(0, memcmp(op_bl.c_str(), buf, sizeof(buf)));
+      ASSERT_EQ(op_bl.crc32c(0), bl.crc32c(0));
+  }
+}
+
+TEST_F(LibRadosIoECPP, SparseReadOpPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+
+  {
+    std::map<uint64_t, uint64_t> extents;
+    bufferlist read_bl;
+    int rval = -1;
+    ObjectReadOperation op;
+    op.sparse_read(0, sizeof(buf), &extents, &read_bl, &rval);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, nullptr));
+    ASSERT_EQ(0, rval);
+    assert_eq_sparse(bl, extents, read_bl);
+  }
+}
+
+TEST_F(LibRadosIoECPP, RoundTripPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  Rados cluster;
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  bufferlist cl;
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", cl, sizeof(buf) * 3, 0));
+  ASSERT_EQ(0, memcmp(buf, cl.c_str(), sizeof(buf)));
+}
+
+TEST_F(LibRadosIoECPP, RoundTripPP2)
+{
+  SKIP_IF_CRIMSON();
+  bufferlist bl;
+  bl.append("ceph");
+  ObjectWriteOperation write;
+  write.write(0, bl);
+  write.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ASSERT_EQ(0, ioctx.operate("foo", &write));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  read.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED|LIBRADOS_OP_FLAG_FADVISE_RANDOM);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "ceph", 4));
+}
+
+TEST_F(LibRadosIoECPP, OverlappingWriteRoundTripPP) {
+  SKIP_IF_CRIMSON();
+  int bsize = alignment;
+  int dbsize = bsize * 2;
+  char *buf = (char *)new char[dbsize];
+  char *buf2 = (char *)new char[bsize];
+  auto cleanup = [&] {
+    delete[] buf;
+    delete[] buf2;
+  };
+  scope_guard<decltype(cleanup)> sg(std::move(cleanup));
+  memset(buf, 0xcc, dbsize);
+  bufferlist bl1;
+  bl1.append(buf, dbsize);
+  ASSERT_EQ(0, ioctx.write("foo", bl1, dbsize, 0));
+  memset(buf2, 0xdd, bsize);
+  bufferlist bl2;
+  bl2.append(buf2, bsize);
+  ASSERT_EQ(-EOPNOTSUPP, ioctx.write("foo", bl2, bsize, 0));
+  bufferlist bl3;
+  ASSERT_EQ(dbsize, ioctx.read("foo", bl3, dbsize, 0));
+  // Read the same as first write
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, dbsize));
+}
+
+TEST_F(LibRadosIoECPP, WriteFullRoundTripPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char buf2[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.write_full("foo", bl2));
+  bufferlist bl3;
+  ASSERT_EQ((int)sizeof(buf2), ioctx.read("foo", bl3, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf2, sizeof(buf2)));
+}
+
+TEST_F(LibRadosIoECPP, WriteFullRoundTripPP2)
+{
+  SKIP_IF_CRIMSON();
+  bufferlist bl;
+  bl.append("ceph");
+  ObjectWriteOperation write;
+  write.write_full(bl);
+  write.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+  ASSERT_EQ(0, ioctx.operate("foo", &write));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  read.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED|LIBRADOS_OP_FLAG_FADVISE_RANDOM);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "ceph", 4));
+}
+
+TEST_F(LibRadosIoECPP, AppendRoundTripPP) {
+  SKIP_IF_CRIMSON();
+  char *buf = (char *)new char[alignment];
+  char *buf2 = (char *)new char[alignment];
+  auto cleanup = [&] {
+    delete[] buf;
+    delete[] buf2;
+  };
+  scope_guard<decltype(cleanup)> sg(std::move(cleanup));
+  memset(buf, 0xde, alignment);
+  bufferlist bl1;
+  bl1.append(buf, alignment);
+  ASSERT_EQ(0, ioctx.append("foo", bl1, alignment));
+  memset(buf2, 0xad, alignment);
+  bufferlist bl2;
+  bl2.append(buf2, alignment);
+  ASSERT_EQ(0, ioctx.append("foo", bl2, alignment));
+  bufferlist bl3;
+  ASSERT_EQ((int)(alignment * 2),
+	    ioctx.read("foo", bl3, (alignment * 4), 0));
+  const char *bl3_str = bl3.c_str();
+  ASSERT_EQ(0, memcmp(bl3_str, buf, alignment));
+  ASSERT_EQ(0, memcmp(bl3_str + alignment, buf2, alignment));
+}
+
+TEST_F(LibRadosIoECPP, TruncTestPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl, sizeof(buf)));
+  ASSERT_EQ(-EOPNOTSUPP, ioctx.trunc("foo", sizeof(buf) / 2));
+  bufferlist bl2;
+  // Same size
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl2, sizeof(buf), 0));
+  // No change
+  ASSERT_EQ(0, memcmp(bl2.c_str(), buf, sizeof(buf)));
+}
+
+TEST_F(LibRadosIoECPP, RemoveTestPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  ASSERT_EQ(0, ioctx.remove("foo"));
+  bufferlist bl2;
+  ASSERT_EQ(-ENOENT, ioctx.read("foo", bl2, sizeof(buf), 0));
+}
+
+TEST_F(LibRadosIoECPP, XattrsRoundTripPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  bufferlist bl2;
+  ASSERT_EQ(-ENODATA, ioctx.getxattr("foo", attr1, bl2));
+  bufferlist bl3;
+  bl3.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr1, bl3));
+  bufferlist bl4;
+  ASSERT_EQ((int)sizeof(attr1_buf),
+      ioctx.getxattr("foo", attr1, bl4));
+  ASSERT_EQ(0, memcmp(bl4.c_str(), attr1_buf, sizeof(attr1_buf)));
+}
+
+TEST_F(LibRadosIoECPP, RmXattrPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  bufferlist bl2;
+  bl2.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr1, bl2));
+  ASSERT_EQ(0, ioctx.rmxattr("foo", attr1));
+  bufferlist bl3;
+  ASSERT_EQ(-ENODATA, ioctx.getxattr("foo", attr1, bl3));
+
+  // Test rmxattr on a removed object
+  char buf2[128];
+  char attr2[] = "attr2";
+  char attr2_buf[] = "foo bar baz";
+  memset(buf2, 0xbb, sizeof(buf2));
+  bufferlist bl21;
+  bl21.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo_rmxattr", bl21, sizeof(buf2), 0));
+  bufferlist bl22;
+  bl22.append(attr2_buf, sizeof(attr2_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo_rmxattr", attr2, bl22));
+  ASSERT_EQ(0, ioctx.remove("foo_rmxattr"));
+  ASSERT_EQ(-ENOENT, ioctx.rmxattr("foo_rmxattr", attr2));
+}
+
+TEST_F(LibRadosIoECPP, XattrListPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char attr1[] = "attr1";
+  char attr1_buf[] = "foo bar baz";
+  char attr2[] = "attr2";
+  char attr2_buf[256];
+  for (size_t j = 0; j < sizeof(attr2_buf); ++j) {
+    attr2_buf[j] = j % 0xff;
+  }
+  memset(buf, 0xaa, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.append("foo", bl1, sizeof(buf)));
+  bufferlist bl2;
+  bl2.append(attr1_buf, sizeof(attr1_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr1, bl2));
+  bufferlist bl3;
+  bl3.append(attr2_buf, sizeof(attr2_buf));
+  ASSERT_EQ(0, ioctx.setxattr("foo", attr2, bl3));
+  std::map<std::string, bufferlist> attrset;
+  ASSERT_EQ(0, ioctx.getxattrs("foo", attrset));
+  for (std::map<std::string, bufferlist>::iterator i = attrset.begin();
+       i != attrset.end(); ++i) {
+    if (i->first == string(attr1)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr1_buf, sizeof(attr1_buf)));
+    }
+    else if (i->first == string(attr2)) {
+      ASSERT_EQ(0, memcmp(i->second.c_str(), attr2_buf, sizeof(attr2_buf)));
+    }
+    else {
+      ASSERT_EQ(0, 1);
+    }
+  }
+}
+
+TEST_F(LibRadosIoPP, CmpExtPP) {
+  bufferlist bl;
+  bl.append("ceph");
+  ObjectWriteOperation write1;
+  write1.write(0, bl);
+  ASSERT_EQ(0, ioctx.operate("foo", &write1));
+
+  bufferlist new_bl;
+  new_bl.append("CEPH");
+  ObjectWriteOperation write2;
+  write2.cmpext(0, bl, nullptr);
+  write2.write(0, new_bl);
+  ASSERT_EQ(0, ioctx.operate("foo", &write2));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "CEPH", 4));
+}
+
+TEST_F(LibRadosIoPP, CmpExtDNEPP) {
+  bufferlist bl;
+  bl.append(std::string(4, '\0'));
+
+  bufferlist new_bl;
+  new_bl.append("CEPH");
+  ObjectWriteOperation write;
+  write.cmpext(0, bl, nullptr);
+  write.write(0, new_bl);
+  ASSERT_EQ(0, ioctx.operate("foo", &write));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "CEPH", 4));
+}
+
+TEST_F(LibRadosIoPP, CmpExtMismatchPP) {
+  bufferlist bl;
+  bl.append("ceph");
+  ObjectWriteOperation write1;
+  write1.write(0, bl);
+  ASSERT_EQ(0, ioctx.operate("foo", &write1));
+
+  bufferlist new_bl;
+  new_bl.append("CEPH");
+  ObjectWriteOperation write2;
+  write2.cmpext(0, new_bl, nullptr);
+  write2.write(0, new_bl);
+  ASSERT_EQ(-MAX_ERRNO, ioctx.operate("foo", &write2));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "ceph", 4));
+}
+
+TEST_F(LibRadosIoECPP, CmpExtPP) {
+  SKIP_IF_CRIMSON();
+  bufferlist bl;
+  bl.append("ceph");
+  ObjectWriteOperation write1;
+  write1.write(0, bl);
+  ASSERT_EQ(0, ioctx.operate("foo", &write1));
+
+  bufferlist new_bl;
+  new_bl.append("CEPH");
+  ObjectWriteOperation write2;
+  write2.cmpext(0, bl, nullptr);
+  write2.write_full(new_bl);
+  ASSERT_EQ(0, ioctx.operate("foo", &write2));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "CEPH", 4));
+}
+
+TEST_F(LibRadosIoECPP, CmpExtDNEPP) {
+  SKIP_IF_CRIMSON();
+  bufferlist bl;
+  bl.append(std::string(4, '\0'));
+
+  bufferlist new_bl;
+  new_bl.append("CEPH");
+  ObjectWriteOperation write;
+  write.cmpext(0, bl, nullptr);
+  write.write_full(new_bl);
+  ASSERT_EQ(0, ioctx.operate("foo", &write));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "CEPH", 4));
+}
+
+TEST_F(LibRadosIoECPP, CmpExtMismatchPP) {
+  SKIP_IF_CRIMSON();
+  bufferlist bl;
+  bl.append("ceph");
+  ObjectWriteOperation write1;
+  write1.write(0, bl);
+  ASSERT_EQ(0, ioctx.operate("foo", &write1));
+
+  bufferlist new_bl;
+  new_bl.append("CEPH");
+  ObjectWriteOperation write2;
+  write2.cmpext(0, new_bl, nullptr);
+  write2.write_full(new_bl);
+  ASSERT_EQ(-MAX_ERRNO, ioctx.operate("foo", &write2));
+
+  ObjectReadOperation read;
+  read.read(0, bl.length(), NULL, NULL);
+  ASSERT_EQ(0, ioctx.operate("foo", &read, &bl));
+  ASSERT_EQ(0, memcmp(bl.c_str(), "ceph", 4));
+}
diff --git a/src/test/librados/librados.cc b/src/test/librados/librados.cc
new file mode 100644
index 000000000..c688724da
--- /dev/null
+++ b/src/test/librados/librados.cc
@@ -0,0 +1,13 @@
+//#include "common/config.h"
+#include "include/rados/librados.h"
+
+#include "gtest/gtest.h"
+
+TEST(Librados, CreateShutdown) {
+  rados_t cluster;
+  int err;
+  err = rados_create(&cluster, "someid");
+  EXPECT_EQ(err, 0);
+
+  rados_shutdown(cluster);
+}
diff --git a/src/test/librados/librados_config.cc b/src/test/librados/librados_config.cc
new file mode 100644
index 000000000..d30fb30ef
--- /dev/null
+++ b/src/test/librados/librados_config.cc
@@ -0,0 +1,98 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/rados/librados.h"
+
+#include <sstream>
+#include <string>
+#include <string.h>
+#include <errno.h>
+
+using std::string;
+
+TEST(LibRadosConfig, SimpleSet) {
+  rados_t cl;
+  int ret = rados_create(&cl, NULL);
+  ASSERT_EQ(ret, 0);
+
+  ret = rados_conf_set(cl, "leveldb_max_open_files", "21");
+  ASSERT_EQ(ret, 0);
+
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "leveldb_max_open_files", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("21"), string(buf));
+
+  rados_shutdown(cl);
+}
+
+TEST(LibRadosConfig, ArgV) {
+  rados_t cl;
+  int ret = rados_create(&cl, NULL);
+  ASSERT_EQ(ret, 0);
+
+  const char *argv[] = { "foo", "--leveldb-max-open-files", "2",
+			 "--key", "my-key", NULL };
+  size_t argc = (sizeof(argv) / sizeof(argv[0])) - 1;
+  rados_conf_parse_argv(cl, argc, argv);
+
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "key", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("my-key"), string(buf));
+
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "leveldb_max_open_files", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(string("2"), string(buf));
+
+  rados_shutdown(cl);
+}
+
+TEST(LibRadosConfig, DebugLevels) {
+  rados_t cl;
+  int ret = rados_create(&cl, NULL);
+  ASSERT_EQ(ret, 0);
+
+  ret = rados_conf_set(cl, "debug_rados", "3");
+  ASSERT_EQ(ret, 0);
+
+  char buf[128];
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "debug_rados", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(0, strcmp("3/3", buf));
+
+  ret = rados_conf_set(cl, "debug_rados", "7/8");
+  ASSERT_EQ(ret, 0);
+
+  memset(buf, 0, sizeof(buf));
+  ret = rados_conf_get(cl, "debug_rados", buf, sizeof(buf));
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(0, strcmp("7/8", buf));
+
+  ret = rados_conf_set(cl, "debug_rados", "foo");
+  ASSERT_EQ(ret, -EINVAL);
+
+  ret = rados_conf_set(cl, "debug_asdkfasdjfajksdf", "foo");
+  ASSERT_EQ(ret, -ENOENT);
+
+  ret = rados_conf_get(cl, "debug_radfjadfsdados", buf, sizeof(buf));
+  ASSERT_EQ(ret, -ENOENT);
+
+  rados_shutdown(cl);
+}
diff --git a/src/test/librados/list.cc b/src/test/librados/list.cc
new file mode 100644
index 000000000..ef3488c0b
--- /dev/null
+++ b/src/test/librados/list.cc
@@ -0,0 +1,555 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "test/librados/test.h"
+#include "test/librados/test_common.h"
+#include "test/librados/TestCase.h"
+#include "global/global_context.h"
+
+#include "include/types.h"
+#include "common/hobject.h"
+#include "gtest/gtest.h"
+#include <errno.h>
+#include <string>
+#include <stdexcept>
+
+#include "crimson_utils.h"
+
+using namespace std;
+using namespace librados;
+
+typedef RadosTestNSCleanup LibRadosList;
+typedef RadosTestECNSCleanup LibRadosListEC;
+typedef RadosTestNP LibRadosListNP;
+
+
+TEST_F(LibRadosList, ListObjects) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  const char *entry;
+  bool foundit = false;
+  while (rados_nobjects_list_next(ctx, &entry, NULL, NULL) != -ENOENT) {
+    foundit = true;
+    ASSERT_EQ(std::string(entry), "foo");
+  }
+  ASSERT_TRUE(foundit);
+  rados_nobjects_list_close(ctx);
+}
+
+TEST_F(LibRadosList, ListObjectsZeroInName) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo\0bar", buf, sizeof(buf), 0));
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  const char *entry;
+  size_t entry_size;
+  bool foundit = false;
+  while (rados_nobjects_list_next2(ctx, &entry, NULL, NULL,
+				   &entry_size, NULL, NULL) != -ENOENT) {
+    foundit = true;
+    ASSERT_EQ(std::string(entry, entry_size), "foo\0bar");
+  }
+  ASSERT_TRUE(foundit);
+  rados_nobjects_list_close(ctx);
+}
+
+static void check_list(
+  std::set<std::string>& myset,
+  rados_list_ctx_t& ctx,
+  const std::string &check_nspace)
+{
+  const char *entry, *nspace;
+  cout << "myset " << myset << std::endl;
+  // we should see every item exactly once.
+  int ret;
+  while ((ret = rados_nobjects_list_next(ctx, &entry, NULL, &nspace)) == 0) {
+    std::string test_name;
+    if (check_nspace == all_nspaces) {
+      test_name = std::string(nspace) + ":" + std::string(entry);
+    } else {
+      ASSERT_TRUE(std::string(nspace) == check_nspace);
+      test_name = std::string(entry);
+    }
+    cout << test_name << std::endl;
+
+    ASSERT_TRUE(myset.end() != myset.find(test_name));
+    myset.erase(test_name);
+  }
+  ASSERT_EQ(-ENOENT, ret);
+  ASSERT_TRUE(myset.empty());
+}
+
+TEST_F(LibRadosList, ListObjectsNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  // Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo2", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo3", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_write(ioctx, "foo4", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo5", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns2");
+  ASSERT_EQ(0, rados_write(ioctx, "foo6", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo7", buf, sizeof(buf), 0));
+
+  char nspace[4];
+  ASSERT_EQ(-ERANGE, rados_ioctx_get_namespace(ioctx, nspace, 3));
+  ASSERT_EQ(static_cast<int>(strlen("ns2")),
+	    rados_ioctx_get_namespace(ioctx, nspace, sizeof(nspace)));
+  ASSERT_EQ(0, strcmp("ns2", nspace));
+
+  std::set<std::string> def, ns1, ns2, all;
+  def.insert(std::string("foo1"));
+  def.insert(std::string("foo2"));
+  def.insert(std::string("foo3"));
+  ns1.insert(std::string("foo1"));
+  ns1.insert(std::string("foo4"));
+  ns1.insert(std::string("foo5"));
+  ns2.insert(std::string("foo6"));
+  ns2.insert(std::string("foo7"));
+  all.insert(std::string(":foo1"));
+  all.insert(std::string(":foo2"));
+  all.insert(std::string(":foo3"));
+  all.insert(std::string("ns1:foo1"));
+  all.insert(std::string("ns1:foo4"));
+  all.insert(std::string("ns1:foo5"));
+  all.insert(std::string("ns2:foo6"));
+  all.insert(std::string("ns2:foo7"));
+
+  rados_list_ctx_t ctx;
+  // Check default namespace ""
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  check_list(def, ctx, "");
+  rados_nobjects_list_close(ctx);
+
+  // Check namespace "ns1"
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  check_list(ns1, ctx, "ns1");
+  rados_nobjects_list_close(ctx);
+
+  // Check namespace "ns2"
+  rados_ioctx_set_namespace(ioctx, "ns2");
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  check_list(ns2, ctx, "ns2");
+  rados_nobjects_list_close(ctx);
+
+  // Check ALL namespaces
+  rados_ioctx_set_namespace(ioctx, LIBRADOS_ALL_NSPACES);
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  check_list(all, ctx, all_nspaces);
+  rados_nobjects_list_close(ctx);
+}
+
+
+TEST_F(LibRadosList, ListObjectsStart) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+
+  for (int i=0; i<16; ++i) {
+    string n = stringify(i);
+    ASSERT_EQ(0, rados_write(ioctx, n.c_str(), buf, sizeof(buf), 0));
+  }
+
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  std::map<int, std::set<std::string> > pg_to_obj;
+  const char *entry;
+  while (rados_nobjects_list_next(ctx, &entry, NULL, NULL) == 0) {
+    uint32_t pos = rados_nobjects_list_get_pg_hash_position(ctx);
+    std::cout << entry << " " << pos << std::endl;
+    pg_to_obj[pos].insert(entry);
+  }
+  rados_nobjects_list_close(ctx);
+
+  std::map<int, std::set<std::string> >::reverse_iterator p =
+    pg_to_obj.rbegin();
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  while (p != pg_to_obj.rend()) {
+    ASSERT_EQ((uint32_t)p->first, rados_nobjects_list_seek(ctx, p->first));
+    ASSERT_EQ(0, rados_nobjects_list_next(ctx, &entry, NULL, NULL));
+    std::cout << "have " << entry << " expect one of " << p->second << std::endl;
+    ASSERT_TRUE(p->second.count(entry));
+    ++p;
+  }
+  rados_nobjects_list_close(ctx);
+}
+
+// this function replicates
+// librados::operator<<(std::ostream& os, const librados::ObjectCursor& oc)
+// because we don't want to use librados in librados client.
+std::ostream& operator<<(std::ostream&os, const rados_object_list_cursor& oc)
+{
+  if (oc) {
+    os << *(hobject_t *)oc;
+  } else {
+    os << hobject_t{};
+  }
+  return os;
+}
+
+TEST_F(LibRadosList, ListObjectsCursor) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+
+  const int max_objs = 16;
+
+  for (int i=0; i<max_objs; ++i) {
+    string n = stringify(i);
+    ASSERT_EQ(0, rados_write(ioctx, n.c_str(), buf, sizeof(buf), 0));
+  }
+
+  {
+    rados_list_ctx_t ctx;
+    const char *entry;
+    rados_object_list_cursor cursor;
+    ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+    ASSERT_EQ(rados_nobjects_list_get_cursor(ctx, &cursor), 0);
+    rados_object_list_cursor first_cursor = cursor;
+    cout << "x cursor=" << cursor << std::endl;
+    while (rados_nobjects_list_next(ctx, &entry, NULL, NULL) == 0) {
+      string oid = entry;
+      ASSERT_EQ(rados_nobjects_list_get_cursor(ctx, &cursor), 0);
+      cout << "> oid=" << oid << " cursor=" << cursor << std::endl;
+    }
+    rados_nobjects_list_seek_cursor(ctx, first_cursor);
+    ASSERT_EQ(rados_nobjects_list_next(ctx, &entry, NULL, NULL), 0);
+    cout << "FIRST> seek to " << first_cursor << " oid=" << string(entry) << std::endl;
+  }
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+
+  std::map<rados_object_list_cursor, string> cursor_to_obj;
+  int count = 0;
+
+  const char *entry;
+  while (rados_nobjects_list_next(ctx, &entry, NULL, NULL) == 0) {
+    rados_object_list_cursor cursor;
+    ASSERT_EQ(rados_nobjects_list_get_cursor(ctx, &cursor), 0);
+    string oid = entry;
+    cout << ": oid=" << oid << " cursor=" << cursor << std::endl;
+    cursor_to_obj[cursor] = oid;
+
+    rados_nobjects_list_seek_cursor(ctx, cursor);
+    cout << ": seek to " << cursor << std::endl;
+    ASSERT_EQ(rados_nobjects_list_next(ctx, &entry, NULL, NULL), 0);
+    cout << "> " << cursor << " -> " << entry << std::endl;
+    ASSERT_EQ(string(entry), oid);
+    ASSERT_LT(count, max_objs); /* avoid infinite loops due to bad seek */
+
+    ++count;
+  }
+
+  ASSERT_EQ(count, max_objs);
+
+  auto p = cursor_to_obj.rbegin();
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  while (p != cursor_to_obj.rend()) {
+    cout << ": seek to " << p->first << std::endl;
+    rados_object_list_cursor cursor;
+    rados_object_list_cursor oid(p->first);
+    rados_nobjects_list_seek_cursor(ctx, oid);
+    ASSERT_EQ(rados_nobjects_list_get_cursor(ctx, &cursor), 0);
+    cout << ": cursor()=" << cursor << " expected=" << oid << std::endl;
+    // ASSERT_EQ(ObjectCursor(oid), ObjectCursor(cursor));
+    ASSERT_EQ(rados_nobjects_list_next(ctx, &entry, NULL, NULL), 0);
+    cout << "> " << cursor << " -> " << entry << std::endl;
+    cout << ": entry=" << entry << " expected=" << p->second << std::endl;
+    ASSERT_EQ(p->second, string(entry));
+
+    ++p;
+
+    rados_object_list_cursor_free(ctx, cursor);
+  }
+}
+
+TEST_F(LibRadosListEC, ListObjects) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  const char *entry;
+  bool foundit = false;
+  while (rados_nobjects_list_next(ctx, &entry, NULL, NULL) != -ENOENT) {
+    foundit = true;
+    ASSERT_EQ(std::string(entry), "foo");
+  }
+  ASSERT_TRUE(foundit);
+  rados_nobjects_list_close(ctx);
+}
+
+TEST_F(LibRadosListEC, ListObjectsNS) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  // Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_write(ioctx, "foo1", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo2", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo3", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_write(ioctx, "foo4", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo5", buf, sizeof(buf), 0));
+  rados_ioctx_set_namespace(ioctx, "ns2");
+  ASSERT_EQ(0, rados_write(ioctx, "foo6", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo7", buf, sizeof(buf), 0));
+
+  std::set<std::string> def, ns1, ns2, all;
+  def.insert(std::string("foo1"));
+  def.insert(std::string("foo2"));
+  def.insert(std::string("foo3"));
+  ns1.insert(std::string("foo1"));
+  ns1.insert(std::string("foo4"));
+  ns1.insert(std::string("foo5"));
+  ns2.insert(std::string("foo6"));
+  ns2.insert(std::string("foo7"));
+  all.insert(std::string(":foo1"));
+  all.insert(std::string(":foo2"));
+  all.insert(std::string(":foo3"));
+  all.insert(std::string("ns1:foo1"));
+  all.insert(std::string("ns1:foo4"));
+  all.insert(std::string("ns1:foo5"));
+  all.insert(std::string("ns2:foo6"));
+  all.insert(std::string("ns2:foo7"));
+
+  rados_list_ctx_t ctx;
+  // Check default namespace ""
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  check_list(def, ctx, "");
+  rados_nobjects_list_close(ctx);
+
+  // Check default namespace "ns1"
+  rados_ioctx_set_namespace(ioctx, "ns1");
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  check_list(ns1, ctx, "ns1");
+  rados_nobjects_list_close(ctx);
+
+  // Check default namespace "ns2"
+  rados_ioctx_set_namespace(ioctx, "ns2");
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  check_list(ns2, ctx, "ns2");
+  rados_nobjects_list_close(ctx);
+
+  // Check all namespaces
+  rados_ioctx_set_namespace(ioctx, LIBRADOS_ALL_NSPACES);
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  check_list(all, ctx, all_nspaces);
+  rados_nobjects_list_close(ctx);
+}
+
+
+TEST_F(LibRadosListEC, ListObjectsStart) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+
+  for (int i=0; i<16; ++i) {
+    string n = stringify(i);
+    ASSERT_EQ(0, rados_write(ioctx, n.c_str(), buf, sizeof(buf), 0));
+  }
+
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  std::map<int, std::set<std::string> > pg_to_obj;
+  const char *entry;
+  while (rados_nobjects_list_next(ctx, &entry, NULL, NULL) == 0) {
+    uint32_t pos = rados_nobjects_list_get_pg_hash_position(ctx);
+    std::cout << entry << " " << pos << std::endl;
+    pg_to_obj[pos].insert(entry);
+  }
+  rados_nobjects_list_close(ctx);
+
+  std::map<int, std::set<std::string> >::reverse_iterator p =
+    pg_to_obj.rbegin();
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  while (p != pg_to_obj.rend()) {
+    ASSERT_EQ((uint32_t)p->first, rados_nobjects_list_seek(ctx, p->first));
+    ASSERT_EQ(0, rados_nobjects_list_next(ctx, &entry, NULL, NULL));
+    std::cout << "have " << entry << " expect one of " << p->second << std::endl;
+    ASSERT_TRUE(p->second.count(entry));
+    ++p;
+  }
+  rados_nobjects_list_close(ctx);
+}
+
+TEST_F(LibRadosListNP, ListObjectsError) {
+  std::string pool_name;
+  rados_t cluster;
+  rados_ioctx_t ioctx;
+  pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+
+  //ASSERT_EQ(0, rados_pool_delete(cluster, pool_name.c_str()));
+  {
+    char *buf, *st;
+    size_t buflen, stlen;
+    string c = "{\"prefix\":\"osd pool rm\",\"pool\": \"" + pool_name +
+      "\",\"pool2\":\"" + pool_name +
+      "\",\"yes_i_really_really_mean_it_not_faking\": true}";
+    const char *cmd[2] = { c.c_str(), 0 };
+    ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
+    ASSERT_EQ(0, rados_wait_for_latest_osdmap(cluster));
+  }
+
+  rados_list_ctx_t ctx;
+  ASSERT_EQ(0, rados_nobjects_list_open(ioctx, &ctx));
+  const char *entry;
+  ASSERT_EQ(-ENOENT, rados_nobjects_list_next(ctx, &entry, NULL, NULL));
+  rados_nobjects_list_close(ctx);
+  rados_ioctx_destroy(ioctx);
+  rados_shutdown(cluster);
+}
+
+
+
+// ---------------------------------------------
+
+TEST_F(LibRadosList, EnumerateObjects) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+
+  const uint32_t n_objects = 16;
+  for (unsigned i=0; i<n_objects; ++i) {
+    ASSERT_EQ(0, rados_write(ioctx, stringify(i).c_str(), buf, sizeof(buf), 0));
+  }
+
+  // Ensure a non-power-of-two PG count to avoid only
+  // touching the easy path.
+  if (!is_crimson_cluster()) {
+    ASSERT_TRUE(set_pg_num(&s_cluster, pool_name, 11).empty());
+    ASSERT_TRUE(set_pgp_num(&s_cluster, pool_name, 11).empty());
+  }
+  std::set<std::string> saw_obj;
+  rados_object_list_cursor c = rados_object_list_begin(ioctx);
+  rados_object_list_cursor end = rados_object_list_end(ioctx);
+  while(!rados_object_list_is_end(ioctx, c))
+  {
+    rados_object_list_item results[12];
+    memset(results, 0, sizeof(rados_object_list_item) * 12);
+    rados_object_list_cursor temp_end = rados_object_list_end(ioctx);
+    int r = rados_object_list(ioctx, c, temp_end,
+            12, NULL, 0, results, &c);
+    rados_object_list_cursor_free(ioctx, temp_end);
+    ASSERT_GE(r, 0);
+    for (int i = 0; i < r; ++i) {
+      std::string oid(results[i].oid, results[i].oid_length);
+      if (saw_obj.count(oid)) {
+          std::cerr << "duplicate obj " << oid << std::endl;
+      }
+      ASSERT_FALSE(saw_obj.count(oid));
+      saw_obj.insert(oid);
+    }
+    rados_object_list_free(12, results);
+  }
+  rados_object_list_cursor_free(ioctx, c);
+  rados_object_list_cursor_free(ioctx, end);
+
+  for (unsigned i=0; i<n_objects; ++i) {
+    if (!saw_obj.count(stringify(i))) {
+        std::cerr << "missing object " << i << std::endl;
+    }
+    ASSERT_TRUE(saw_obj.count(stringify(i)));
+  }
+  ASSERT_EQ(n_objects, saw_obj.size());
+}
+
+TEST_F(LibRadosList, EnumerateObjectsSplit) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+
+  const uint32_t n_objects = 16;
+  for (unsigned i=0; i<n_objects; ++i) {
+    ASSERT_EQ(0, rados_write(ioctx, stringify(i).c_str(), buf, sizeof(buf), 0));
+  }
+
+  // Ensure a non-power-of-two PG count to avoid only
+  // touching the easy path.
+  if (!is_crimson_cluster()) {
+    if (auto error = set_pg_num(&s_cluster, pool_name, 11); !error.empty()) {
+      GTEST_FAIL() << error;
+    }
+    if (auto error = set_pgp_num(&s_cluster, pool_name, 11); !error.empty()) {
+      GTEST_FAIL() << error;
+    }
+  }
+
+  rados_object_list_cursor begin = rados_object_list_begin(ioctx);
+  rados_object_list_cursor end = rados_object_list_end(ioctx);
+
+  // Step through an odd number of shards
+  unsigned m = 5;
+  std::set<std::string> saw_obj;
+  for (unsigned n = 0; n < m; ++n) {
+      rados_object_list_cursor shard_start = rados_object_list_begin(ioctx);;
+      rados_object_list_cursor shard_end = rados_object_list_end(ioctx);;
+
+      rados_object_list_slice(
+        ioctx,
+        begin,
+        end,
+        n,
+        m,
+        &shard_start,
+        &shard_end);
+      std::cout << "split " << n << "/" << m << " -> "
+		<< *(hobject_t*)shard_start << " "
+		<< *(hobject_t*)shard_end << std::endl;
+
+      rados_object_list_cursor c = shard_start;
+      //while(c < shard_end)
+      while(rados_object_list_cursor_cmp(ioctx, c, shard_end) == -1)
+      {
+        rados_object_list_item results[12];
+        memset(results, 0, sizeof(rados_object_list_item) * 12);
+        int r = rados_object_list(ioctx,
+                c, shard_end,
+                12, NULL, 0, results, &c);
+        ASSERT_GE(r, 0);
+        for (int i = 0; i < r; ++i) {
+          std::string oid(results[i].oid, results[i].oid_length);
+          if (saw_obj.count(oid)) {
+              std::cerr << "duplicate obj " << oid << std::endl;
+          }
+          ASSERT_FALSE(saw_obj.count(oid));
+          saw_obj.insert(oid);
+        }
+        rados_object_list_free(12, results);
+      }
+      rados_object_list_cursor_free(ioctx, shard_start);
+      rados_object_list_cursor_free(ioctx, shard_end);
+  }
+
+  rados_object_list_cursor_free(ioctx, begin);
+  rados_object_list_cursor_free(ioctx, end);
+
+  for (unsigned i=0; i<n_objects; ++i) {
+    if (!saw_obj.count(stringify(i))) {
+        std::cerr << "missing object " << i << std::endl;
+    }
+    ASSERT_TRUE(saw_obj.count(stringify(i)));
+  }
+  ASSERT_EQ(n_objects, saw_obj.size());
+}
diff --git a/src/test/librados/list_cxx.cc b/src/test/librados/list_cxx.cc
new file mode 100644
index 000000000..112231302
--- /dev/null
+++ b/src/test/librados/list_cxx.cc
@@ -0,0 +1,782 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include <errno.h>
+#include <string>
+#include <stdexcept>
+
+#include "gtest/gtest.h"
+
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "include/types.h"
+#include "common/hobject.h"
+#include "test/librados/test_cxx.h"
+#include "test/librados/test_common.h"
+#include "test/librados/testcase_cxx.h"
+#include "global/global_context.h"
+
+#include "crimson_utils.h"
+
+using namespace librados;
+
+typedef RadosTestPPNSCleanup LibRadosListPP;
+typedef RadosTestECPPNSCleanup LibRadosListECPP;
+
+TEST_F(LibRadosListPP, ListObjectsPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  NObjectIterator iter(ioctx.nobjects_begin());
+  bool foundit = false;
+  while (iter != ioctx.nobjects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).get_oid(), "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListPP, ListObjectsTwicePP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  NObjectIterator iter(ioctx.nobjects_begin());
+  bool foundit = false;
+  while (iter != ioctx.nobjects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).get_oid(), "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+  ++iter;
+  ASSERT_TRUE(iter == ioctx.nobjects_end());
+  foundit = false;
+  iter.seek(0);
+  while (iter != ioctx.nobjects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).get_oid(), "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListPP, ListObjectsCopyIterPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  // make sure this is still valid after the original iterators are gone
+  NObjectIterator iter3;
+  {
+    NObjectIterator iter(ioctx.nobjects_begin());
+    NObjectIterator iter2(iter);
+    iter3 = iter2;
+    ASSERT_EQ((*iter).get_oid(), "foo");
+    ++iter;
+    ASSERT_TRUE(iter == ioctx.nobjects_end());
+    ++iter;
+    ASSERT_TRUE(iter == ioctx.nobjects_end());
+
+    ASSERT_EQ(iter2->get_oid(), "foo");
+    ASSERT_EQ(iter3->get_oid(), "foo");
+    ++iter2;
+    ASSERT_TRUE(iter2 == ioctx.nobjects_end());
+  }
+
+  ASSERT_EQ(iter3->get_oid(), "foo");
+  iter3 = iter3;
+  ASSERT_EQ(iter3->get_oid(), "foo");
+  ++iter3;
+  ASSERT_TRUE(iter3 == ioctx.nobjects_end());
+}
+
+TEST_F(LibRadosListPP, ListObjectsEndIter) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  NObjectIterator iter(ioctx.nobjects_begin());
+  NObjectIterator iter_end(ioctx.nobjects_end());
+  NObjectIterator iter_end2 = ioctx.nobjects_end();
+  ASSERT_TRUE(iter_end == iter_end2);
+  ASSERT_TRUE(iter_end == ioctx.nobjects_end());
+  ASSERT_TRUE(iter_end2 == ioctx.nobjects_end());
+
+  ASSERT_EQ(iter->get_oid(), "foo");
+  ++iter;
+  ASSERT_TRUE(iter == ioctx.nobjects_end());
+  ASSERT_TRUE(iter == iter_end);
+  ASSERT_TRUE(iter == iter_end2);
+  NObjectIterator iter2 = iter;
+  ASSERT_TRUE(iter2 == ioctx.nobjects_end());
+  ASSERT_TRUE(iter2 == iter_end);
+  ASSERT_TRUE(iter2 == iter_end2);
+}
+
+static void check_listpp(std::set<std::string>& myset, IoCtx& ioctx, const std::string &check_nspace)
+{
+  NObjectIterator iter(ioctx.nobjects_begin());
+  std::set<std::string> orig_set(myset);
+  /**
+   * During splitting, we might see duplicate items.
+   * We assert that every object returned is in myset and that
+   * we don't hit ENOENT until we have hit every item in myset
+   * at least once.
+   */
+  while (iter != ioctx.nobjects_end()) {
+    std::string test_name;
+    if (check_nspace == all_nspaces) {
+      test_name = iter->get_nspace() + ":" + iter->get_oid();
+    } else {
+      ASSERT_TRUE(iter->get_nspace() == check_nspace);
+      test_name = iter->get_oid();
+    }
+    ASSERT_TRUE(orig_set.end() != orig_set.find(test_name));
+    myset.erase(test_name);
+    ++iter;
+  }
+  ASSERT_TRUE(myset.empty());
+}
+
+TEST_F(LibRadosListPP, ListObjectsPPNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  // Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo1", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns1");
+  ASSERT_EQ(0, ioctx.write("foo1", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo2", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo3", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns1");
+  ASSERT_EQ(0, ioctx.write("foo4", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo5", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns2");
+  ASSERT_EQ(0, ioctx.write("foo6", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo7", bl1, sizeof(buf), 0));
+  ASSERT_EQ(std::string("ns2"), ioctx.get_namespace());
+
+  std::set<std::string> def, ns1, ns2, all;
+  def.insert(std::string("foo1"));
+  def.insert(std::string("foo2"));
+  def.insert(std::string("foo3"));
+  ns1.insert(std::string("foo1"));
+  ns1.insert(std::string("foo4"));
+  ns1.insert(std::string("foo5"));
+  ns2.insert(std::string("foo6"));
+  ns2.insert(std::string("foo7"));
+  all.insert(std::string(":foo1"));
+  all.insert(std::string(":foo2"));
+  all.insert(std::string(":foo3"));
+  all.insert(std::string("ns1:foo1"));
+  all.insert(std::string("ns1:foo4"));
+  all.insert(std::string("ns1:foo5"));
+  all.insert(std::string("ns2:foo6"));
+  all.insert(std::string("ns2:foo7"));
+
+  ioctx.set_namespace("");
+  check_listpp(def, ioctx, "");
+
+  ioctx.set_namespace("ns1");
+  check_listpp(ns1, ioctx, "ns1");
+
+  ioctx.set_namespace("ns2");
+  check_listpp(ns2, ioctx, "ns2");
+
+  ioctx.set_namespace(all_nspaces);
+  check_listpp(all, ioctx, all_nspaces);
+}
+
+TEST_F(LibRadosListPP, ListObjectsManyPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  for (int i=0; i<256; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, bl.length(), 0));
+  }
+
+  librados::NObjectIterator it = ioctx.nobjects_begin();
+  std::set<std::string> saw_obj;
+  std::set<int> saw_pg;
+  for (; it != ioctx.nobjects_end(); ++it) {
+    std::cout << it->get_oid()
+	      << " " << it.get_pg_hash_position() << std::endl;
+    saw_obj.insert(it->get_oid());
+    saw_pg.insert(it.get_pg_hash_position());
+  }
+  std::cout << "saw " << saw_pg.size() << " pgs " << std::endl;
+
+  // make sure they are 0..n
+  for (unsigned i = 0; i < saw_pg.size(); ++i)
+    ASSERT_TRUE(saw_pg.count(i));
+}
+
+TEST_F(LibRadosListPP, ListObjectsStartPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  for (int i=0; i<16; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, bl.length(), 0));
+  }
+
+  librados::NObjectIterator it = ioctx.nobjects_begin();
+  std::map<int, std::set<std::string> > pg_to_obj;
+  for (; it != ioctx.nobjects_end(); ++it) {
+    std::cout << it->get_oid() << " " << it.get_pg_hash_position() << std::endl;
+    pg_to_obj[it.get_pg_hash_position()].insert(it->get_oid());
+  }
+
+  std::map<int, std::set<std::string> >::reverse_iterator p =
+    pg_to_obj.rbegin();
+  it = ioctx.nobjects_begin(p->first);
+  while (p != pg_to_obj.rend()) {
+    ASSERT_EQ((uint32_t)p->first, it.seek(p->first));
+    std::cout << "have " << it->get_oid() << " expect one of " << p->second << std::endl;
+    ASSERT_TRUE(p->second.count(it->get_oid()));
+    ++p;
+  }
+}
+
+TEST_F(LibRadosListPP, ListObjectsCursorNSPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  const int max_objs = 16;
+
+  map<string, string> oid_to_ns;
+
+  for (int i=0; i<max_objs; ++i) {
+    stringstream ss;
+    ss << "ns" << i / 4;
+    ioctx.set_namespace(ss.str());
+    string oid = stringify(i);
+    ASSERT_EQ(0, ioctx.write(oid, bl, bl.length(), 0));
+
+    oid_to_ns[oid] = ss.str();
+  }
+
+  ioctx.set_namespace(all_nspaces);
+
+  librados::NObjectIterator it = ioctx.nobjects_begin();
+  std::map<librados::ObjectCursor, string> cursor_to_obj;
+
+  int count = 0;
+
+  librados::ObjectCursor seek_cursor;
+
+  map<string, list<librados::ObjectCursor> > ns_to_cursors;
+
+  for (it = ioctx.nobjects_begin(); it != ioctx.nobjects_end(); ++it) {
+    librados::ObjectCursor cursor = it.get_cursor();
+    string oid = it->get_oid();
+    cout << "> oid=" << oid << " cursor=" << it.get_cursor() << std::endl;
+  }
+
+  vector<string> objs_order;
+
+  for (it = ioctx.nobjects_begin(); it != ioctx.nobjects_end(); ++it, ++count) {
+    librados::ObjectCursor cursor = it.get_cursor();
+    string oid = it->get_oid();
+    std::cout << oid << " " << it.get_pg_hash_position() << std::endl;
+    cout << ": oid=" << oid << " cursor=" << it.get_cursor() << std::endl;
+    cursor_to_obj[cursor] = oid;
+
+    ASSERT_EQ(oid_to_ns[oid], it->get_nspace());
+
+    it.seek(cursor);
+    cout << ": seek to " << cursor << " it.cursor=" << it.get_cursor() << std::endl;
+    ASSERT_EQ(oid, it->get_oid());
+    ASSERT_LT(count, max_objs); /* avoid infinite loops due to bad seek */
+
+    ns_to_cursors[it->get_nspace()].push_back(cursor);
+
+    if (count == max_objs/2) {
+      seek_cursor = cursor;
+    }
+    objs_order.push_back(it->get_oid());
+  }
+
+  ASSERT_EQ(count, max_objs);
+
+  /* check that reading past seek also works */
+  cout << "seek_cursor=" << seek_cursor << std::endl;
+  it.seek(seek_cursor);
+  for (count = max_objs/2; count < max_objs; ++count, ++it) {
+    ASSERT_EQ(objs_order[count], it->get_oid());
+  }
+
+  /* seek to all cursors, check that we get expected obj */
+  for (auto& niter : ns_to_cursors) {
+    const string& ns = niter.first;
+    list<librados::ObjectCursor>& cursors = niter.second;
+
+    for (auto& cursor : cursors) {
+      cout << ": seek to " << cursor << std::endl;
+      it.seek(cursor);
+      ASSERT_EQ(cursor, it.get_cursor());
+      string& expected_oid = cursor_to_obj[cursor];
+      cout << ": it->get_cursor()=" << it.get_cursor() << " expected=" << cursor << std::endl;
+      cout << ": it->get_oid()=" << it->get_oid() << " expected=" << expected_oid << std::endl;
+      cout << ": it->get_nspace()=" << it->get_oid() << " expected=" << ns << std::endl;
+      ASSERT_EQ(expected_oid, it->get_oid());
+      ASSERT_EQ(it->get_nspace(), ns);
+    }
+  }
+}
+
+TEST_F(LibRadosListPP, ListObjectsCursorPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  const int max_objs = 16;
+
+  for (int i=0; i<max_objs; ++i) {
+    stringstream ss;
+    ss << "ns" << i / 4;
+    ioctx.set_namespace(ss.str());
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, bl.length(), 0));
+  }
+
+  ioctx.set_namespace(all_nspaces);
+
+  librados::NObjectIterator it = ioctx.nobjects_begin();
+  std::map<librados::ObjectCursor, string> cursor_to_obj;
+
+  int count = 0;
+
+  for (; it != ioctx.nobjects_end(); ++it, ++count) {
+    librados::ObjectCursor cursor = it.get_cursor();
+    string oid = it->get_oid();
+    std::cout << oid << " " << it.get_pg_hash_position() << std::endl;
+    cout << ": oid=" << oid << " cursor=" << it.get_cursor() << std::endl;
+    cursor_to_obj[cursor] = oid;
+
+    it.seek(cursor);
+    cout << ": seek to " << cursor << std::endl;
+    ASSERT_EQ(oid, it->get_oid());
+    ASSERT_LT(count, max_objs); /* avoid infinite loops due to bad seek */
+  }
+
+  ASSERT_EQ(count, max_objs);
+
+  auto p = cursor_to_obj.rbegin();
+  it = ioctx.nobjects_begin();
+  while (p != cursor_to_obj.rend()) {
+    cout << ": seek to " << p->first << std::endl;
+    it.seek(p->first);
+    ASSERT_EQ(p->first, it.get_cursor());
+    cout << ": it->get_cursor()=" << it.get_cursor() << " expected=" << p->first << std::endl;
+    cout << ": it->get_oid()=" << it->get_oid() << " expected=" << p->second << std::endl;
+    ASSERT_EQ(p->second, it->get_oid());
+
+    librados::NObjectIterator it2 = ioctx.nobjects_begin(it.get_cursor());
+    ASSERT_EQ(it2->get_oid(), it->get_oid());
+
+    ++p;
+  }
+}
+
+TEST_F(LibRadosListECPP, ListObjectsPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  NObjectIterator iter(ioctx.nobjects_begin());
+  bool foundit = false;
+  while (iter != ioctx.nobjects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).get_oid(), "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListECPP, ListObjectsTwicePP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  NObjectIterator iter(ioctx.nobjects_begin());
+  bool foundit = false;
+  while (iter != ioctx.nobjects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).get_oid(), "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+  ++iter;
+  ASSERT_TRUE(iter == ioctx.nobjects_end());
+  foundit = false;
+  iter.seek(0);
+  while (iter != ioctx.nobjects_end()) {
+    foundit = true;
+    ASSERT_EQ((*iter).get_oid(), "foo");
+    ++iter;
+  }
+  ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListECPP, ListObjectsCopyIterPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  // make sure this is still valid after the original iterators are gone
+  NObjectIterator iter3;
+  {
+    NObjectIterator iter(ioctx.nobjects_begin());
+    NObjectIterator iter2(iter);
+    iter3 = iter2;
+    ASSERT_EQ((*iter).get_oid(), "foo");
+    ++iter;
+    ASSERT_TRUE(iter == ioctx.nobjects_end());
+    ++iter;
+    ASSERT_TRUE(iter == ioctx.nobjects_end());
+
+    ASSERT_EQ(iter2->get_oid(), "foo");
+    ASSERT_EQ(iter3->get_oid(), "foo");
+    ++iter2;
+    ASSERT_TRUE(iter2 == ioctx.nobjects_end());
+  }
+
+  ASSERT_EQ(iter3->get_oid(), "foo");
+  iter3 = iter3;
+  ASSERT_EQ(iter3->get_oid(), "foo");
+  ++iter3;
+  ASSERT_TRUE(iter3 == ioctx.nobjects_end());
+}
+
+TEST_F(LibRadosListECPP, ListObjectsEndIter) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  NObjectIterator iter(ioctx.nobjects_begin());
+  NObjectIterator iter_end(ioctx.nobjects_end());
+  NObjectIterator iter_end2 = ioctx.nobjects_end();
+  ASSERT_TRUE(iter_end == iter_end2);
+  ASSERT_TRUE(iter_end == ioctx.nobjects_end());
+  ASSERT_TRUE(iter_end2 == ioctx.nobjects_end());
+
+  ASSERT_EQ(iter->get_oid(), "foo");
+  ++iter;
+  ASSERT_TRUE(iter == ioctx.nobjects_end());
+  ASSERT_TRUE(iter == iter_end);
+  ASSERT_TRUE(iter == iter_end2);
+  NObjectIterator iter2 = iter;
+  ASSERT_TRUE(iter2 == ioctx.nobjects_end());
+  ASSERT_TRUE(iter2 == iter_end);
+  ASSERT_TRUE(iter2 == iter_end2);
+}
+
+TEST_F(LibRadosListECPP, ListObjectsPPNS) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  // Create :foo1, :foo2, :foo3, n1:foo1, ns1:foo4, ns1:foo5, ns2:foo6, n2:foo7
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo1", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns1");
+  ASSERT_EQ(0, ioctx.write("foo1", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo2", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo3", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns1");
+  ASSERT_EQ(0, ioctx.write("foo4", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo5", bl1, sizeof(buf), 0));
+  ioctx.set_namespace("ns2");
+  ASSERT_EQ(0, ioctx.write("foo6", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo7", bl1, sizeof(buf), 0));
+
+  std::set<std::string> def, ns1, ns2;
+  def.insert(std::string("foo1"));
+  def.insert(std::string("foo2"));
+  def.insert(std::string("foo3"));
+  ns1.insert(std::string("foo1"));
+  ns1.insert(std::string("foo4"));
+  ns1.insert(std::string("foo5"));
+  ns2.insert(std::string("foo6"));
+  ns2.insert(std::string("foo7"));
+
+  ioctx.set_namespace("");
+  check_listpp(def, ioctx, "");
+
+  ioctx.set_namespace("ns1");
+  check_listpp(ns1, ioctx, "ns1");
+
+  ioctx.set_namespace("ns2");
+  check_listpp(ns2, ioctx, "ns2");
+}
+
+TEST_F(LibRadosListECPP, ListObjectsManyPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  for (int i=0; i<256; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, bl.length(), 0));
+  }
+
+  librados::NObjectIterator it = ioctx.nobjects_begin();
+  std::set<std::string> saw_obj;
+  std::set<int> saw_pg;
+  for (; it != ioctx.nobjects_end(); ++it) {
+    std::cout << it->get_oid()
+	      << " " << it.get_pg_hash_position() << std::endl;
+    saw_obj.insert(it->get_oid());
+    saw_pg.insert(it.get_pg_hash_position());
+  }
+  std::cout << "saw " << saw_pg.size() << " pgs " << std::endl;
+
+  // make sure they are 0..n
+  for (unsigned i = 0; i < saw_pg.size(); ++i)
+    ASSERT_TRUE(saw_pg.count(i));
+}
+
+TEST_F(LibRadosListECPP, ListObjectsStartPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  for (int i=0; i<16; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, bl.length(), 0));
+  }
+
+  librados::NObjectIterator it = ioctx.nobjects_begin();
+  std::map<int, std::set<std::string> > pg_to_obj;
+  for (; it != ioctx.nobjects_end(); ++it) {
+    std::cout << it->get_oid() << " " << it.get_pg_hash_position() << std::endl;
+    pg_to_obj[it.get_pg_hash_position()].insert(it->get_oid());
+  }
+
+  std::map<int, std::set<std::string> >::reverse_iterator p =
+    pg_to_obj.rbegin();
+  it = ioctx.nobjects_begin(p->first);
+  while (p != pg_to_obj.rend()) {
+    ASSERT_EQ((uint32_t)p->first, it.seek(p->first));
+    std::cout << "have " << it->get_oid() << " expect one of " << p->second << std::endl;
+    ASSERT_TRUE(p->second.count(it->get_oid()));
+    ++p;
+  }
+}
+
+TEST_F(LibRadosListPP, ListObjectsFilterPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist obj_content;
+  obj_content.append(buf, sizeof(buf));
+
+  std::string target_str = "content";
+
+  // Write xattr bare, no ::encod'ing
+  bufferlist target_val;
+  target_val.append(target_str);
+  bufferlist nontarget_val;
+  nontarget_val.append("rhubarb");
+
+  ASSERT_EQ(0, ioctx.write("has_xattr", obj_content, obj_content.length(), 0));
+  ASSERT_EQ(0, ioctx.write("has_wrong_xattr", obj_content, obj_content.length(), 0));
+  ASSERT_EQ(0, ioctx.write("no_xattr", obj_content, obj_content.length(), 0));
+
+  ASSERT_EQ(0, ioctx.setxattr("has_xattr", "theattr", target_val));
+  ASSERT_EQ(0, ioctx.setxattr("has_wrong_xattr", "theattr", nontarget_val));
+
+  bufferlist filter_bl;
+  std::string filter_name = "plain";
+  encode(filter_name, filter_bl);
+  encode("_theattr", filter_bl);
+  encode(target_str, filter_bl);
+
+  NObjectIterator iter(ioctx.nobjects_begin(filter_bl));
+  bool foundit = false;
+  int k = 0;
+  while (iter != ioctx.nobjects_end()) {
+    foundit = true;
+    // We should only see the object that matches the filter
+    ASSERT_EQ((*iter).get_oid(), "has_xattr");
+    // We should only see it once
+    ASSERT_EQ(k, 0);
+    ++iter;
+    ++k;
+  }
+  ASSERT_TRUE(foundit);
+}
+
+TEST_F(LibRadosListPP, EnumerateObjectsPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  const uint32_t n_objects = 16;
+  for (unsigned i=0; i<n_objects; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, sizeof(buf), 0));
+  }
+
+  std::set<std::string> saw_obj;
+  ObjectCursor c = ioctx.object_list_begin();
+  ObjectCursor end = ioctx.object_list_end();
+  while(!ioctx.object_list_is_end(c))
+  {
+    std::vector<ObjectItem> result;
+    int r = ioctx.object_list(c, end, 12, {}, &result, &c);
+    ASSERT_GE(r, 0);
+    ASSERT_EQ(r, (int)result.size());
+    for (int i = 0; i < r; ++i) {
+      auto oid = result[i].oid;
+      if (saw_obj.count(oid)) {
+          std::cerr << "duplicate obj " << oid << std::endl;
+      }
+      ASSERT_FALSE(saw_obj.count(oid));
+      saw_obj.insert(oid);
+    }
+  }
+
+  for (unsigned i=0; i<n_objects; ++i) {
+    if (!saw_obj.count(stringify(i))) {
+        std::cerr << "missing object " << i << std::endl;
+    }
+    ASSERT_TRUE(saw_obj.count(stringify(i)));
+  }
+  ASSERT_EQ(n_objects, saw_obj.size());
+}
+
+TEST_F(LibRadosListPP, EnumerateObjectsSplitPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  const uint32_t n_objects = 16;
+  for (unsigned i=0; i<n_objects; ++i) {
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, sizeof(buf), 0));
+  }
+
+  ObjectCursor begin = ioctx.object_list_begin();
+  ObjectCursor end = ioctx.object_list_end();
+
+  // Step through an odd number of shards
+  unsigned m = 5;
+  std::set<std::string> saw_obj;
+  for (unsigned n = 0; n < m; ++n) {
+      ObjectCursor shard_start;
+      ObjectCursor shard_end;
+
+      ioctx.object_list_slice(
+        begin,
+        end,
+        n,
+        m,
+        &shard_start,
+        &shard_end);
+
+      ObjectCursor c(shard_start);
+      while(c < shard_end)
+      {
+        std::vector<ObjectItem> result;
+        int r = ioctx.object_list(c, shard_end, 12, {}, &result, &c);
+        ASSERT_GE(r, 0);
+
+        for (const auto & i : result) {
+          const auto &oid = i.oid;
+          if (saw_obj.count(oid)) {
+              std::cerr << "duplicate obj " << oid << std::endl;
+          }
+          ASSERT_FALSE(saw_obj.count(oid));
+          saw_obj.insert(oid);
+        }
+      }
+  }
+
+  for (unsigned i=0; i<n_objects; ++i) {
+    if (!saw_obj.count(stringify(i))) {
+        std::cerr << "missing object " << i << std::endl;
+    }
+    ASSERT_TRUE(saw_obj.count(stringify(i)));
+  }
+  ASSERT_EQ(n_objects, saw_obj.size());
+}
+
+
+TEST_F(LibRadosListPP, EnumerateObjectsFilterPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist obj_content;
+  obj_content.append(buf, sizeof(buf));
+
+  std::string target_str = "content";
+
+  // Write xattr bare, no ::encod'ing
+  bufferlist target_val;
+  target_val.append(target_str);
+  bufferlist nontarget_val;
+  nontarget_val.append("rhubarb");
+
+  ASSERT_EQ(0, ioctx.write("has_xattr", obj_content, obj_content.length(), 0));
+  ASSERT_EQ(0, ioctx.write("has_wrong_xattr", obj_content, obj_content.length(), 0));
+  ASSERT_EQ(0, ioctx.write("no_xattr", obj_content, obj_content.length(), 0));
+
+  ASSERT_EQ(0, ioctx.setxattr("has_xattr", "theattr", target_val));
+  ASSERT_EQ(0, ioctx.setxattr("has_wrong_xattr", "theattr", nontarget_val));
+
+  bufferlist filter_bl;
+  std::string filter_name = "plain";
+  encode(filter_name, filter_bl);
+  encode("_theattr", filter_bl);
+  encode(target_str, filter_bl);
+
+  ObjectCursor c = ioctx.object_list_begin();
+  ObjectCursor end = ioctx.object_list_end();
+  bool foundit = false;
+  while(!ioctx.object_list_is_end(c))
+  {
+    std::vector<ObjectItem> result;
+    int r = ioctx.object_list(c, end, 12, filter_bl, &result, &c);
+    ASSERT_GE(r, 0);
+    ASSERT_EQ(r, (int)result.size());
+    for (int i = 0; i < r; ++i) {
+      auto oid = result[i].oid;
+      // We should only see the object that matches the filter
+      ASSERT_EQ(oid, "has_xattr");
+      // We should only see it once
+      ASSERT_FALSE(foundit);
+      foundit = true;
+    }
+  }
+  ASSERT_TRUE(foundit);
+}
diff --git a/src/test/librados/lock.cc b/src/test/librados/lock.cc
new file mode 100644
index 000000000..a6ac36365
--- /dev/null
+++ b/src/test/librados/lock.cc
@@ -0,0 +1,237 @@
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+#include "cls/lock/cls_lock_client.h"
+
+#include <algorithm>
+#include <chrono>
+#include <thread>
+#include <errno.h>
+#include "gtest/gtest.h"
+#include <sys/time.h>
+
+#include "crimson_utils.h"
+
+using namespace std::chrono_literals;
+
+typedef RadosTest LibRadosLock;
+typedef RadosTestEC LibRadosLockEC;
+
+
+TEST_F(LibRadosLock, LockExclusive) {
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock1", "Cookie", "", NULL,  0));
+  ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLock1", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLock, LockShared) {
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock2", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, rados_lock_shared(ioctx, "foo", "TestLock2", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLock, LockExclusiveDur) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  auto lock_exclusive = [this](timeval* tv) {
+    return rados_lock_exclusive(ioctx, "foo", "TestLock3", "Cookie", "", tv, 0);
+  };
+  constexpr int expected = 0;
+  ASSERT_EQ(expected, lock_exclusive(&tv));
+  ASSERT_EQ(expected, wait_until(1.0s, 0.1s, expected, lock_exclusive, nullptr));
+}
+
+TEST_F(LibRadosLock, LockSharedDur) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  auto lock_shared = [this](timeval* tv) {
+    return rados_lock_shared(ioctx, "foo", "TestLock4", "Cookie", "Tag", "", tv, 0);
+  };
+  constexpr int expected = 0;
+  ASSERT_EQ(expected, lock_shared(&tv));
+  ASSERT_EQ(expected, wait_until(1.0s, 0.1s, expected, lock_shared, nullptr));
+}
+
+
+TEST_F(LibRadosLock, LockMayRenew) {
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock5", "Cookie", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLock5", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock5", "Cookie", "", NULL, LOCK_FLAG_MAY_RENEW));
+}
+
+TEST_F(LibRadosLock, Unlock) {
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock6", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLock6", "Cookie"));
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock6", "Cookie", "", NULL,  0));
+}
+
+TEST_F(LibRadosLock, ListLockers) {
+  int exclusive;
+  char tag[1024];
+  char clients[1024];
+  char cookies[1024];
+  char addresses[1024];
+  size_t tag_len = 1024;
+  size_t clients_len = 1024;
+  size_t cookies_len = 1024;
+  size_t addresses_len = 1024;
+  std::stringstream sstm;
+  sstm << "client." << rados_get_instance_id(cluster);
+  std::string me = sstm.str();
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock7", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLock7", "Cookie"));
+  ASSERT_EQ(0, rados_list_lockers(ioctx, "foo", "TestLock7", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLock7", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-34, rados_list_lockers(ioctx, "foo", "TestLock7", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  tag_len = 1024;
+  clients_len = 1024;
+  cookies_len = 1024;
+  addresses_len = 1024;
+  ASSERT_EQ(1, rados_list_lockers(ioctx, "foo", "TestLock7", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(0, exclusive);
+  ASSERT_EQ(0, strcmp(tag, "Tag"));
+  ASSERT_EQ(strlen("Tag") + 1, tag_len);
+  ASSERT_EQ(0, strcmp(me.c_str(), clients));
+  ASSERT_EQ(me.size() + 1, clients_len);
+  ASSERT_EQ(0, strcmp(cookies, "Cookie"));
+  ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
+}
+
+TEST_F(LibRadosLock, BreakLock) {
+  int exclusive;
+  char tag[1024];
+  char clients[1024];
+  char cookies[1024];
+  char addresses[1024];
+  size_t tag_len = 1024;
+  size_t clients_len = 1024;
+  size_t cookies_len = 1024;
+  size_t addresses_len = 1024;
+  std::stringstream sstm;
+  sstm << "client." << rados_get_instance_id(cluster);
+  std::string me = sstm.str();
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLock8", "Cookie", "", NULL, 0));
+  ASSERT_EQ(1, rados_list_lockers(ioctx, "foo", "TestLock8", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(1, exclusive);
+  ASSERT_EQ(0, strcmp(tag, ""));
+  ASSERT_EQ(1U, tag_len);
+  ASSERT_EQ(0, strcmp(me.c_str(), clients));
+  ASSERT_EQ(me.size() + 1, clients_len);
+  ASSERT_EQ(0, strcmp(cookies, "Cookie"));
+  ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
+  ASSERT_EQ(0, rados_break_lock(ioctx, "foo", "TestLock8", clients, "Cookie"));
+}
+
+// EC testing
+TEST_F(LibRadosLockEC, LockExclusive) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLockEC1", "Cookie", "", NULL,  0));
+  ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLockEC1", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockEC, LockShared) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLockEC2", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, rados_lock_shared(ioctx, "foo", "TestLockEC2", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockEC, LockExclusiveDur) {
+  SKIP_IF_CRIMSON();
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  auto lock_exclusive = [this](timeval* tv) {
+    return rados_lock_exclusive(ioctx, "foo", "TestLockEC3", "Cookie", "", tv, 0);
+  };
+  constexpr int expected = 0;
+  ASSERT_EQ(expected, lock_exclusive(&tv));
+  ASSERT_EQ(expected, wait_until(1.0s, 0.1s, expected, lock_exclusive, nullptr));
+}
+
+TEST_F(LibRadosLockEC, LockSharedDur) {
+  SKIP_IF_CRIMSON();
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  auto lock_shared = [this](timeval* tv) {
+    return rados_lock_shared(ioctx, "foo", "TestLockEC4", "Cookie", "Tag", "", tv, 0);
+  };
+  constexpr int expected = 0;
+  ASSERT_EQ(expected, lock_shared(&tv));
+  ASSERT_EQ(expected, wait_until(1.0s, 0.1s, expected, lock_shared, nullptr));
+}
+
+
+TEST_F(LibRadosLockEC, LockMayRenew) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLockEC5", "Cookie", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, rados_lock_exclusive(ioctx, "foo", "TestLockEC5", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLockEC5", "Cookie", "", NULL, LOCK_FLAG_MAY_RENEW));
+}
+
+TEST_F(LibRadosLockEC, Unlock) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLockEC6", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLockEC6", "Cookie"));
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLockEC6", "Cookie", "", NULL,  0));
+}
+
+TEST_F(LibRadosLockEC, ListLockers) {
+  SKIP_IF_CRIMSON();
+  int exclusive;
+  char tag[1024];
+  char clients[1024];
+  char cookies[1024];
+  char addresses[1024];
+  size_t tag_len = 1024;
+  size_t clients_len = 1024;
+  size_t cookies_len = 1024;
+  size_t addresses_len = 1024;
+  std::stringstream sstm;
+  sstm << "client." << rados_get_instance_id(cluster);
+  std::string me = sstm.str();
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLockEC7", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(0, rados_unlock(ioctx, "foo", "TestLockEC7", "Cookie"));
+  ASSERT_EQ(0, rados_list_lockers(ioctx, "foo", "TestLockEC7", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(0, rados_lock_shared(ioctx, "foo", "TestLockEC7", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-34, rados_list_lockers(ioctx, "foo", "TestLockEC7", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  tag_len = 1024;
+  clients_len = 1024;
+  cookies_len = 1024;
+  addresses_len = 1024;
+  ASSERT_EQ(1, rados_list_lockers(ioctx, "foo", "TestLockEC7", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(0, exclusive);
+  ASSERT_EQ(0, strcmp(tag, "Tag"));
+  ASSERT_EQ(strlen("Tag") + 1, tag_len);
+  ASSERT_EQ(0, strcmp(me.c_str(), clients));
+  ASSERT_EQ(me.size() + 1, clients_len);
+  ASSERT_EQ(0, strcmp(cookies, "Cookie"));
+  ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
+}
+
+TEST_F(LibRadosLockEC, BreakLock) {
+  SKIP_IF_CRIMSON();
+  int exclusive;
+  char tag[1024];
+  char clients[1024];
+  char cookies[1024];
+  char addresses[1024];
+  size_t tag_len = 1024;
+  size_t clients_len = 1024;
+  size_t cookies_len = 1024;
+  size_t addresses_len = 1024;
+  std::stringstream sstm;
+  sstm << "client." << rados_get_instance_id(cluster);
+  std::string me = sstm.str();
+  ASSERT_EQ(0, rados_lock_exclusive(ioctx, "foo", "TestLockEC8", "Cookie", "", NULL, 0));
+  ASSERT_EQ(1, rados_list_lockers(ioctx, "foo", "TestLockEC8", &exclusive, tag, &tag_len, clients, &clients_len, cookies, &cookies_len, addresses, &addresses_len ));
+  ASSERT_EQ(1, exclusive);
+  ASSERT_EQ(0, strcmp(tag, ""));
+  ASSERT_EQ(1U, tag_len);
+  ASSERT_EQ(0, strcmp(me.c_str(), clients));
+  ASSERT_EQ(me.size() + 1, clients_len);
+  ASSERT_EQ(0, strcmp(cookies, "Cookie"));
+  ASSERT_EQ(strlen("Cookie") + 1, cookies_len);
+  ASSERT_EQ(0, rados_break_lock(ioctx, "foo", "TestLockEC8", clients, "Cookie"));
+}
+
diff --git a/src/test/librados/lock_cxx.cc b/src/test/librados/lock_cxx.cc
new file mode 100644
index 000000000..0267ea938
--- /dev/null
+++ b/src/test/librados/lock_cxx.cc
@@ -0,0 +1,203 @@
+#include <algorithm>
+#include <chrono>
+#include <thread>
+#include <errno.h>
+#include <sys/time.h>
+#include "gtest/gtest.h"
+
+#include "include/rados/librados.hpp"
+#include "cls/lock/cls_lock_client.h"
+
+#include "test/librados/test_cxx.h"
+#include "test/librados/testcase_cxx.h"
+
+#include "crimson_utils.h"
+
+using namespace std::chrono_literals;
+using namespace librados;
+
+typedef RadosTestPP LibRadosLockPP;
+typedef RadosTestECPP LibRadosLockECPP;
+
+TEST_F(LibRadosLockPP, LockExclusivePP) {
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockPP1", "Cookie", "", NULL,  0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLockPP1", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockPP, LockSharedPP) {
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLockPP2", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_shared("foo", "TestLockPP2", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockPP, LockExclusiveDurPP) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  auto lock_exclusive = [this](timeval* tv) {
+    return ioctx.lock_exclusive("foo", "TestLockPP3", "Cookie", "", tv, 0);
+  };
+  constexpr int expected = 0;
+  ASSERT_EQ(expected, lock_exclusive(&tv));
+  ASSERT_EQ(expected, wait_until(1.0s, 0.1s, expected, lock_exclusive, nullptr));
+}
+
+TEST_F(LibRadosLockPP, LockSharedDurPP) {
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  auto lock_shared = [this](timeval* tv) {
+    return ioctx.lock_shared("foo", "TestLockPP4", "Cookie", "Tag", "", tv, 0);
+  };
+  constexpr int expected = 0;
+  ASSERT_EQ(expected, lock_shared(&tv));
+  ASSERT_EQ(expected, wait_until(1.0s, 0.1s, expected, lock_shared, nullptr));
+}
+
+TEST_F(LibRadosLockPP, LockMayRenewPP) {
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockPP5", "Cookie", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLockPP5", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockPP5", "Cookie", "", NULL, LOCK_FLAG_MAY_RENEW));
+}
+
+TEST_F(LibRadosLockPP, UnlockPP) {
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockPP6", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.unlock("foo", "TestLockPP6", "Cookie"));
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockPP6", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockPP, ListLockersPP) {
+  std::stringstream sstm;
+  sstm << "client." << cluster.get_instance_id();
+  std::string me = sstm.str();
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLockPP7", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.unlock("foo", "TestLockPP7", "Cookie"));
+  {
+    int exclusive;
+    std::string tag;
+    std::list<librados::locker_t> lockers;
+    ASSERT_EQ(0, ioctx.list_lockers("foo", "TestLockPP7", &exclusive, &tag, &lockers));
+  }
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLockPP7", "Cookie", "Tag", "", NULL, 0));
+  {
+    int exclusive;
+    std::string tag;
+    std::list<librados::locker_t> lockers;
+    ASSERT_EQ(1, ioctx.list_lockers("foo", "TestLockPP7", &exclusive, &tag, &lockers));
+    std::list<librados::locker_t>::iterator it = lockers.begin();
+    ASSERT_FALSE(lockers.end() == it);
+    ASSERT_EQ(me, it->client);
+    ASSERT_EQ("Cookie", it->cookie);
+  }
+}
+
+TEST_F(LibRadosLockPP, BreakLockPP) {
+  int exclusive;
+  std::string tag;
+  std::list<librados::locker_t> lockers;
+  std::stringstream sstm;
+  sstm << "client." << cluster.get_instance_id();
+  std::string me = sstm.str();
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockPP8", "Cookie",  "", NULL, 0));
+  ASSERT_EQ(1, ioctx.list_lockers("foo", "TestLockPP8", &exclusive, &tag, &lockers));
+  std::list<librados::locker_t>::iterator it = lockers.begin();
+  ASSERT_FALSE(lockers.end() == it);
+  ASSERT_EQ(me, it->client);
+  ASSERT_EQ("Cookie", it->cookie);
+  ASSERT_EQ(0, ioctx.break_lock("foo", "TestLockPP8", it->client, "Cookie"));
+}
+
+// EC testing
+TEST_F(LibRadosLockECPP, LockExclusivePP) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockECPP1", "Cookie", "", NULL,  0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLockECPP1", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockECPP, LockSharedPP) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLockECPP2", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_shared("foo", "TestLockECPP2", "Cookie", "Tag", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockECPP, LockExclusiveDurPP) {
+  SKIP_IF_CRIMSON();
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  auto lock_exclusive = [this](timeval* tv) {
+    return ioctx.lock_exclusive("foo", "TestLockECPP3", "Cookie", "", tv, 0);
+  };
+  constexpr int expected = 0;
+  ASSERT_EQ(expected, lock_exclusive(&tv));
+  ASSERT_EQ(expected, wait_until(1.0s, 0.1s, expected, lock_exclusive, nullptr));
+}
+
+TEST_F(LibRadosLockECPP, LockSharedDurPP) {
+  SKIP_IF_CRIMSON();
+  struct timeval tv;
+  tv.tv_sec = 1;
+  tv.tv_usec = 0;
+  auto lock_shared = [this](timeval* tv) {
+    return ioctx.lock_shared("foo", "TestLockECPP4", "Cookie", "Tag", "", tv, 0);
+  };
+  const int expected = 0;
+  ASSERT_EQ(expected, lock_shared(&tv));
+  ASSERT_EQ(expected, wait_until(1.0s, 0.1s, expected, lock_shared, nullptr));
+}
+
+TEST_F(LibRadosLockECPP, LockMayRenewPP) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockECPP5", "Cookie", "", NULL, 0));
+  ASSERT_EQ(-EEXIST, ioctx.lock_exclusive("foo", "TestLockECPP5", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockECPP5", "Cookie", "", NULL, LOCK_FLAG_MAY_RENEW));
+}
+
+TEST_F(LibRadosLockECPP, UnlockPP) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockECPP6", "Cookie", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.unlock("foo", "TestLockECPP6", "Cookie"));
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockECPP6", "Cookie", "", NULL, 0));
+}
+
+TEST_F(LibRadosLockECPP, ListLockersPP) {
+  SKIP_IF_CRIMSON();
+  std::stringstream sstm;
+  sstm << "client." << cluster.get_instance_id();
+  std::string me = sstm.str();
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLockECPP7", "Cookie", "Tag", "", NULL, 0));
+  ASSERT_EQ(0, ioctx.unlock("foo", "TestLockECPP7", "Cookie"));
+  {
+    int exclusive;
+    std::string tag;
+    std::list<librados::locker_t> lockers;
+    ASSERT_EQ(0, ioctx.list_lockers("foo", "TestLockECPP7", &exclusive, &tag, &lockers));
+  }
+  ASSERT_EQ(0, ioctx.lock_shared("foo", "TestLockECPP7", "Cookie", "Tag", "", NULL, 0));
+  {
+    int exclusive;
+    std::string tag;
+    std::list<librados::locker_t> lockers;
+    ASSERT_EQ(1, ioctx.list_lockers("foo", "TestLockECPP7", &exclusive, &tag, &lockers));
+    std::list<librados::locker_t>::iterator it = lockers.begin();
+    ASSERT_FALSE(lockers.end() == it);
+    ASSERT_EQ(me, it->client);
+    ASSERT_EQ("Cookie", it->cookie);
+  }
+}
+
+TEST_F(LibRadosLockECPP, BreakLockPP) {
+  SKIP_IF_CRIMSON();
+  int exclusive;
+  std::string tag;
+  std::list<librados::locker_t> lockers;
+  std::stringstream sstm;
+  sstm << "client." << cluster.get_instance_id();
+  std::string me = sstm.str();
+  ASSERT_EQ(0, ioctx.lock_exclusive("foo", "TestLockECPP8", "Cookie",  "", NULL, 0));
+  ASSERT_EQ(1, ioctx.list_lockers("foo", "TestLockECPP8", &exclusive, &tag, &lockers));
+  std::list<librados::locker_t>::iterator it = lockers.begin();
+  ASSERT_FALSE(lockers.end() == it);
+  ASSERT_EQ(me, it->client);
+  ASSERT_EQ("Cookie", it->cookie);
+  ASSERT_EQ(0, ioctx.break_lock("foo", "TestLockECPP8", it->client, "Cookie"));
+}
diff --git a/src/test/librados/misc.cc b/src/test/librados/misc.cc
new file mode 100644
index 000000000..d9cb1c5b8
--- /dev/null
+++ b/src/test/librados/misc.cc
@@ -0,0 +1,358 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "gtest/gtest.h"
+
+#include "mds/mdstypes.h"
+#include "include/err.h"
+#include "include/buffer.h"
+#include "include/rbd_types.h"
+#include "include/rados.h"
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/scope_guard.h"
+#include "include/stringify.h"
+#include "common/Checksummer.h"
+#include "global/global_context.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+#include "gtest/gtest.h"
+#include <sys/time.h>
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+#include <errno.h>
+#include <map>
+#include <sstream>
+#include <string>
+#include <regex>
+
+using namespace std;
+using namespace librados;
+
+typedef RadosTest LibRadosMisc;
+
+TEST(LibRadosMiscVersion, Version) {
+  int major, minor, extra;
+  rados_version(&major, &minor, &extra);
+}
+
+static void test_rados_log_cb(void *arg,
+                              const char *line,
+                              const char *who,
+                              uint64_t sec, uint64_t nsec,
+                              uint64_t seq, const char *level,
+                              const char *msg)
+{
+    std::cerr << "monitor log callback invoked" << std::endl;
+}
+
+TEST(LibRadosMiscConnectFailure, ConnectFailure) {
+  rados_t cluster;
+
+  char *id = getenv("CEPH_CLIENT_ID");
+  if (id)
+    std::cerr << "Client id is: " << id << std::endl;
+
+  ASSERT_EQ(0, rados_create(&cluster, NULL));
+  ASSERT_EQ(0, rados_conf_read_file(cluster, NULL));
+  ASSERT_EQ(0, rados_conf_parse_env(cluster, NULL));
+
+  ASSERT_EQ(-ENOTCONN, rados_monitor_log(cluster, "error",
+                                         test_rados_log_cb, NULL));
+
+  ASSERT_EQ(0, rados_connect(cluster));
+  rados_shutdown(cluster);
+
+  ASSERT_EQ(0, rados_create(&cluster, NULL));
+  ASSERT_EQ(-ENOENT, rados_connect(cluster));
+  rados_shutdown(cluster);
+}
+
+TEST(LibRadosMiscConnectFailure, ConnectTimeout) {
+  rados_t cluster;
+
+  ASSERT_EQ(0, rados_create(&cluster, NULL));
+  ASSERT_EQ(0, rados_conf_set(cluster, "mon_host", "255.0.1.2:3456"));
+  ASSERT_EQ(0, rados_conf_set(cluster, "key",
+                              "AQAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAA=="));
+  ASSERT_EQ(0, rados_conf_set(cluster, "client_mount_timeout", "2s"));
+
+  utime_t start = ceph_clock_now();
+  ASSERT_EQ(-ETIMEDOUT, rados_connect(cluster));
+  utime_t end = ceph_clock_now();
+
+  utime_t dur = end - start;
+  ASSERT_GE(dur, utime_t(2, 0));
+  ASSERT_LT(dur, utime_t(4, 0));
+
+  rados_shutdown(cluster);
+}
+
+TEST(LibRadosMiscPool, PoolCreationRace) {
+  rados_t cluster_a, cluster_b;
+
+  char *id = getenv("CEPH_CLIENT_ID");
+  if (id)
+    std::cerr << "Client id is: " << id << std::endl;
+
+  ASSERT_EQ(0, rados_create(&cluster_a, NULL));
+  ASSERT_EQ(0, rados_conf_read_file(cluster_a, NULL));
+  // kludge: i want to --log-file foo and only get cluster b
+  //ASSERT_EQ(0, rados_conf_parse_env(cluster_a, NULL));
+  ASSERT_EQ(0, rados_conf_set(cluster_a,
+			      "objecter_debug_inject_relock_delay", "true"));
+  ASSERT_EQ(0, rados_connect(cluster_a));
+
+  ASSERT_EQ(0, rados_create(&cluster_b, NULL));
+  ASSERT_EQ(0, rados_conf_read_file(cluster_b, NULL));
+  ASSERT_EQ(0, rados_conf_parse_env(cluster_b, NULL));
+  ASSERT_EQ(0, rados_connect(cluster_b));
+
+  char poolname[80];
+  snprintf(poolname, sizeof(poolname), "poolrace.%d", rand());
+  rados_pool_create(cluster_a, poolname);
+  rados_ioctx_t a;
+  rados_ioctx_create(cluster_a, poolname, &a);
+
+  char pool2name[80];
+  snprintf(pool2name, sizeof(pool2name), "poolrace2.%d", rand());
+  rados_pool_create(cluster_b, pool2name);
+
+  list<rados_completion_t> cls;
+  // this should normally trigger pretty easily, but we need to bound
+  // the requests because if we get too many we'll get stuck by always
+  // sending enough messages that we hit the socket failure injection.
+  int max = 512;
+  while (max--) {
+    char buf[100];
+    rados_completion_t c;
+    rados_aio_create_completion2(nullptr, nullptr, &c);
+    cls.push_back(c);
+    rados_aio_read(a, "PoolCreationRaceObj", c, buf, 100, 0);
+    cout << "started " << (void*)c << std::endl;
+    if (rados_aio_is_complete(cls.front())) {
+      break;
+    }
+  }
+  while (!rados_aio_is_complete(cls.front())) {
+    cout << "waiting 1 sec" << std::endl;
+    sleep(1);
+  }
+
+  cout << " started " << cls.size() << " aios" << std::endl;
+  for (auto c : cls) {
+    cout << "waiting " << (void*)c << std::endl;
+    rados_aio_wait_for_complete_and_cb(c);
+    rados_aio_release(c);
+  }
+  cout << "done." << std::endl;
+
+  rados_ioctx_destroy(a);
+  rados_pool_delete(cluster_a, poolname);
+  rados_pool_delete(cluster_a, pool2name);
+  rados_shutdown(cluster_b);
+  rados_shutdown(cluster_a);
+}
+
+TEST_F(LibRadosMisc, ClusterFSID) {
+  char fsid[37];
+  ASSERT_EQ(-ERANGE, rados_cluster_fsid(cluster, fsid, sizeof(fsid) - 1));
+  ASSERT_EQ(sizeof(fsid) - 1,
+            (size_t)rados_cluster_fsid(cluster, fsid, sizeof(fsid)));
+}
+
+TEST_F(LibRadosMisc, Exec) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  char buf2[512];
+  int res = rados_exec(ioctx, "foo", "rbd", "get_all_features",
+			  NULL, 0, buf2, sizeof(buf2));
+  ASSERT_GT(res, 0);
+  bufferlist bl;
+  bl.append(buf2, res);
+  auto iter = bl.cbegin();
+  uint64_t all_features;
+  decode(all_features, iter);
+  // make sure *some* features are specified; don't care which ones
+  ASSERT_NE(all_features, (unsigned)0);
+}
+
+TEST_F(LibRadosMisc, WriteSame) {
+  char buf[128];
+  char full[128 * 4];
+  char *cmp;
+
+  /* zero the full range before using writesame */
+  memset(full, 0, sizeof(full));
+  ASSERT_EQ(0, rados_write(ioctx, "ws", full, sizeof(full), 0));
+
+  memset(buf, 0xcc, sizeof(buf));
+  /* write the same buf four times */
+  ASSERT_EQ(0, rados_writesame(ioctx, "ws", buf, sizeof(buf), sizeof(full), 0));
+
+  /* read back the full buffer and confirm that it matches */
+  ASSERT_EQ((int)sizeof(full), rados_read(ioctx, "ws", full, sizeof(full), 0));
+
+  for (cmp = full; cmp < full + sizeof(full); cmp += sizeof(buf)) {
+    ASSERT_EQ(0, memcmp(cmp, buf, sizeof(buf)));
+  }
+
+  /* write_len not a multiple of data_len should throw error */
+  ASSERT_EQ(-EINVAL, rados_writesame(ioctx, "ws", buf, sizeof(buf),
+				     (sizeof(buf) * 4) - 1, 0));
+  ASSERT_EQ(-EINVAL,
+	    rados_writesame(ioctx, "ws", buf, sizeof(buf), sizeof(buf) / 2, 0));
+  ASSERT_EQ(-EINVAL,
+	    rados_writesame(ioctx, "ws", buf, 0, sizeof(buf), 0));
+  /* write_len = data_len, i.e. same as rados_write() */
+  ASSERT_EQ(0, rados_writesame(ioctx, "ws", buf, sizeof(buf), sizeof(buf), 0));
+}
+
+TEST_F(LibRadosMisc, CmpExt) {
+  bufferlist cmp_bl, bad_cmp_bl, write_bl;
+  char stored_str[] = "1234567891";
+  char mismatch_str[] = "1234577777";
+
+  ASSERT_EQ(0,
+	    rados_write(ioctx, "cmpextpp", stored_str, sizeof(stored_str), 0));
+
+  ASSERT_EQ(0,
+	    rados_cmpext(ioctx, "cmpextpp", stored_str, sizeof(stored_str), 0));
+
+  ASSERT_EQ(-MAX_ERRNO - 5,
+	    rados_cmpext(ioctx, "cmpextpp", mismatch_str, sizeof(mismatch_str), 0));
+}
+
+TEST_F(LibRadosMisc, Applications) {
+  const char *cmd[] = {"{\"prefix\":\"osd dump\"}", nullptr};
+  char *buf, *st;
+  size_t buflen, stlen;
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf,
+                                 &buflen, &st, &stlen));
+  ASSERT_LT(0u, buflen);
+  string result(buf);
+  rados_buffer_free(buf);
+  rados_buffer_free(st);
+  if (!std::regex_search(result, std::regex("require_osd_release [l-z]"))) {
+    std::cout << "SKIPPING";
+    return;
+  }
+
+  char apps[128];
+  size_t app_len;
+
+  app_len = sizeof(apps);
+  ASSERT_EQ(0, rados_application_list(ioctx, apps, &app_len));
+  ASSERT_EQ(6U, app_len);
+  ASSERT_EQ(0, memcmp("rados\0", apps, app_len));
+
+  ASSERT_EQ(0, rados_application_enable(ioctx, "app1", 1));
+  ASSERT_EQ(-EPERM, rados_application_enable(ioctx, "app2", 0));
+  ASSERT_EQ(0, rados_application_enable(ioctx, "app2", 1));
+
+  ASSERT_EQ(-ERANGE, rados_application_list(ioctx, apps, &app_len));
+  ASSERT_EQ(16U, app_len);
+  ASSERT_EQ(0, rados_application_list(ioctx, apps, &app_len));
+  ASSERT_EQ(16U, app_len);
+  ASSERT_EQ(0, memcmp("app1\0app2\0rados\0", apps, app_len));
+
+  char keys[128];
+  char vals[128];
+  size_t key_len;
+  size_t val_len;
+
+  key_len = sizeof(keys);
+  val_len = sizeof(vals);
+  ASSERT_EQ(-ENOENT, rados_application_metadata_list(ioctx, "dne", keys,
+                                                     &key_len, vals, &val_len));
+  ASSERT_EQ(0, rados_application_metadata_list(ioctx, "app1", keys, &key_len,
+                                               vals, &val_len));
+  ASSERT_EQ(0U, key_len);
+  ASSERT_EQ(0U, val_len);
+
+  ASSERT_EQ(-ENOENT, rados_application_metadata_set(ioctx, "dne", "key",
+                                                    "value"));
+  ASSERT_EQ(0, rados_application_metadata_set(ioctx, "app1", "key1", "value1"));
+  ASSERT_EQ(0, rados_application_metadata_set(ioctx, "app1", "key2", "value2"));
+
+  ASSERT_EQ(-ERANGE, rados_application_metadata_list(ioctx, "app1", keys,
+                                                     &key_len, vals, &val_len));
+  ASSERT_EQ(10U, key_len);
+  ASSERT_EQ(14U, val_len);
+  ASSERT_EQ(0, rados_application_metadata_list(ioctx, "app1", keys, &key_len,
+                                               vals, &val_len));
+  ASSERT_EQ(10U, key_len);
+  ASSERT_EQ(14U, val_len);
+  ASSERT_EQ(0, memcmp("key1\0key2\0", keys, key_len));
+  ASSERT_EQ(0, memcmp("value1\0value2\0", vals, val_len));
+
+  ASSERT_EQ(0, rados_application_metadata_remove(ioctx, "app1", "key1"));
+  ASSERT_EQ(0, rados_application_metadata_list(ioctx, "app1", keys, &key_len,
+                                               vals, &val_len));
+  ASSERT_EQ(5U, key_len);
+  ASSERT_EQ(7U, val_len);
+  ASSERT_EQ(0, memcmp("key2\0", keys, key_len));
+  ASSERT_EQ(0, memcmp("value2\0", vals, val_len));
+}
+
+TEST_F(LibRadosMisc, MinCompatOSD) {
+  int8_t require_osd_release;
+  ASSERT_EQ(0, rados_get_min_compatible_osd(cluster, &require_osd_release));
+  ASSERT_LE(-1, require_osd_release);
+  ASSERT_GT(CEPH_RELEASE_MAX, require_osd_release);
+}
+
+TEST_F(LibRadosMisc, MinCompatClient) {
+  int8_t min_compat_client;
+  int8_t require_min_compat_client;
+  ASSERT_EQ(0, rados_get_min_compatible_client(cluster,
+                                               &min_compat_client,
+                                               &require_min_compat_client));
+  ASSERT_LE(-1, min_compat_client);
+  ASSERT_GT(CEPH_RELEASE_MAX, min_compat_client);
+
+  ASSERT_LE(-1, require_min_compat_client);
+  ASSERT_GT(CEPH_RELEASE_MAX, require_min_compat_client);
+}
+
+static void shutdown_racer_func()
+{
+  const int niter = 32;
+  rados_t rad;
+  int i;
+
+  for (i = 0; i < niter; ++i) {
+    auto r = connect_cluster(&rad);
+    if (getenv("ALLOW_TIMEOUTS")) {
+      ASSERT_TRUE(r == "" || r == "rados_connect failed with error -110");
+    } else {
+      ASSERT_EQ("", r);
+    }
+    rados_shutdown(rad);
+  }
+}
+
+#ifndef _WIN32
+// See trackers #20988 and #42026
+TEST_F(LibRadosMisc, ShutdownRace)
+{
+  const int nthreads = 128;
+  std::thread threads[nthreads];
+
+  // Need a bunch of fd's for this test
+  struct rlimit rold, rnew;
+  ASSERT_EQ(getrlimit(RLIMIT_NOFILE, &rold), 0);
+  rnew = rold;
+  rnew.rlim_cur = rnew.rlim_max;
+  ASSERT_EQ(setrlimit(RLIMIT_NOFILE, &rnew), 0);
+
+  for (int i = 0; i < nthreads; ++i)
+    threads[i] = std::thread(shutdown_racer_func);
+
+  for (int i = 0; i < nthreads; ++i)
+    threads[i].join();
+  ASSERT_EQ(setrlimit(RLIMIT_NOFILE, &rold), 0);
+}
+#endif /* _WIN32 */
diff --git a/src/test/librados/misc_cxx.cc b/src/test/librados/misc_cxx.cc
new file mode 100644
index 000000000..545d5e57b
--- /dev/null
+++ b/src/test/librados/misc_cxx.cc
@@ -0,0 +1,923 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include <errno.h>
+#include <map>
+#include <sstream>
+#include <string>
+#include <regex>
+
+#include "gtest/gtest.h"
+
+#include "include/err.h"
+#include "include/buffer.h"
+#include "include/rbd_types.h"
+#include "include/rados.h"
+#include "include/rados/librados.hpp"
+#include "include/scope_guard.h"
+#include "include/stringify.h"
+#include "common/Checksummer.h"
+#include "mds/mdstypes.h"
+#include "global/global_context.h"
+#include "test/librados/testcase_cxx.h"
+#include "test/librados/test_cxx.h"
+
+#include "crimson_utils.h"
+
+using namespace std;
+using namespace librados;
+
+typedef RadosTestPP LibRadosMiscPP;
+typedef RadosTestECPP LibRadosMiscECPP;
+
+TEST(LibRadosMiscVersion, VersionPP) {
+  int major, minor, extra;
+  Rados::version(&major, &minor, &extra);
+}
+
+TEST_F(LibRadosMiscPP, WaitOSDMapPP) {
+  ASSERT_EQ(0, cluster.wait_for_latest_osdmap());
+}
+
+TEST_F(LibRadosMiscPP, LongNamePP) {
+  bufferlist bl;
+  bl.append("content");
+  int maxlen = g_conf()->osd_max_object_name_len;
+  ASSERT_EQ(0, ioctx.write(string(maxlen/2, 'a').c_str(), bl, bl.length(), 0));
+  ASSERT_EQ(0, ioctx.write(string(maxlen-1, 'a').c_str(), bl, bl.length(), 0));
+  ASSERT_EQ(0, ioctx.write(string(maxlen, 'a').c_str(), bl, bl.length(), 0));
+  ASSERT_EQ(-ENAMETOOLONG, ioctx.write(string(maxlen+1, 'a').c_str(), bl, bl.length(), 0));
+  ASSERT_EQ(-ENAMETOOLONG, ioctx.write(string(maxlen*2, 'a').c_str(), bl, bl.length(), 0));
+}
+
+TEST_F(LibRadosMiscPP, LongLocatorPP) {
+  bufferlist bl;
+  bl.append("content");
+  int maxlen = g_conf()->osd_max_object_name_len;
+  ioctx.locator_set_key(
+    string((maxlen/2), 'a'));
+  ASSERT_EQ(
+    0,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+  ioctx.locator_set_key(
+    string(maxlen - 1, 'a'));
+  ASSERT_EQ(
+    0,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+  ioctx.locator_set_key(
+    string(maxlen, 'a'));
+  ASSERT_EQ(
+    0,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+  ioctx.locator_set_key(
+    string(maxlen+1, 'a'));
+  ASSERT_EQ(
+    -ENAMETOOLONG,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+  ioctx.locator_set_key(
+    string((maxlen*2), 'a'));
+  ASSERT_EQ(
+    -ENAMETOOLONG,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+}
+
+TEST_F(LibRadosMiscPP, LongNSpacePP) {
+  bufferlist bl;
+  bl.append("content");
+  int maxlen = g_conf()->osd_max_object_namespace_len;
+  ioctx.set_namespace(
+    string((maxlen/2), 'a'));
+  ASSERT_EQ(
+    0,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+  ioctx.set_namespace(
+    string(maxlen - 1, 'a'));
+  ASSERT_EQ(
+    0,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+  ioctx.set_namespace(
+    string(maxlen, 'a'));
+  ASSERT_EQ(
+    0,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+  ioctx.set_namespace(
+    string(maxlen+1, 'a'));
+  ASSERT_EQ(
+    -ENAMETOOLONG,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+  ioctx.set_namespace(
+    string((maxlen*2), 'a'));
+  ASSERT_EQ(
+    -ENAMETOOLONG,
+    ioctx.write(
+      string("a").c_str(),
+      bl, bl.length(), 0));
+}
+
+TEST_F(LibRadosMiscPP, LongAttrNamePP) {
+  bufferlist bl;
+  bl.append("content");
+  int maxlen = g_conf()->osd_max_attr_name_len;
+  ASSERT_EQ(0, ioctx.setxattr("bigattrobj", string(maxlen/2, 'a').c_str(), bl));
+  ASSERT_EQ(0, ioctx.setxattr("bigattrobj", string(maxlen-1, 'a').c_str(), bl));
+  ASSERT_EQ(0, ioctx.setxattr("bigattrobj", string(maxlen, 'a').c_str(), bl));
+  ASSERT_EQ(-ENAMETOOLONG, ioctx.setxattr("bigattrobj", string(maxlen+1, 'a').c_str(), bl));
+  ASSERT_EQ(-ENAMETOOLONG, ioctx.setxattr("bigattrobj", string(maxlen*2, 'a').c_str(), bl));
+}
+
+TEST_F(LibRadosMiscPP, ExecPP) {
+  bufferlist bl;
+  ASSERT_EQ(0, ioctx.write("foo", bl, 0, 0));
+  bufferlist bl2, out;
+  int r = ioctx.exec("foo", "rbd", "get_all_features", bl2, out);
+  ASSERT_EQ(0, r);
+  auto iter = out.cbegin();
+  uint64_t all_features;
+  decode(all_features, iter);
+  // make sure *some* features are specified; don't care which ones
+  ASSERT_NE(all_features, (unsigned)0);
+}
+
+void set_completion_complete(rados_completion_t cb, void *arg)
+{
+  bool *my_aio_complete = (bool*)arg;
+  *my_aio_complete = true;
+}
+
+TEST_F(LibRadosMiscPP, BadFlagsPP) {
+  unsigned badflags = CEPH_OSD_FLAG_PARALLELEXEC;
+  {
+    bufferlist bl;
+    bl.append("data");
+    ASSERT_EQ(0, ioctx.write("badfoo", bl, bl.length(), 0));
+  }
+  {
+    ASSERT_EQ(-EINVAL, ioctx.remove("badfoo", badflags));
+  }
+}
+
+TEST_F(LibRadosMiscPP, Operate1PP) {
+  ObjectWriteOperation o;
+  {
+    bufferlist bl;
+    o.write(0, bl);
+  }
+  std::string val1("val1");
+  {
+    bufferlist bl;
+    bl.append(val1.c_str(), val1.size() + 1);
+    o.setxattr("key1", bl);
+    o.omap_clear(); // shouldn't affect attrs!
+  }
+  ASSERT_EQ(0, ioctx.operate("foo", &o));
+
+  ObjectWriteOperation empty;
+  ASSERT_EQ(0, ioctx.operate("foo", &empty));
+
+  {
+    bufferlist bl;
+    ASSERT_GT(ioctx.getxattr("foo", "key1", bl), 0);
+    ASSERT_EQ(0, strcmp(bl.c_str(), val1.c_str()));
+  }
+  ObjectWriteOperation o2;
+  {
+    bufferlist bl;
+    bl.append(val1);
+    o2.cmpxattr("key1", CEPH_OSD_CMPXATTR_OP_EQ, bl);
+    o2.rmxattr("key1");
+  }
+  ASSERT_EQ(-ECANCELED, ioctx.operate("foo", &o2));
+  ObjectWriteOperation o3;
+  {
+    bufferlist bl;
+    bl.append(val1);
+    o3.cmpxattr("key1", CEPH_OSD_CMPXATTR_OP_EQ, bl);
+  }
+  ASSERT_EQ(-ECANCELED, ioctx.operate("foo", &o3));
+}
+
+TEST_F(LibRadosMiscPP, Operate2PP) {
+  ObjectWriteOperation o;
+  {
+    bufferlist bl;
+    bl.append("abcdefg");
+    o.write(0, bl);
+  }
+  std::string val1("val1");
+  {
+    bufferlist bl;
+    bl.append(val1.c_str(), val1.size() + 1);
+    o.setxattr("key1", bl);
+    o.truncate(0);
+  }
+  ASSERT_EQ(0, ioctx.operate("foo", &o));
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(0U, size);
+}
+
+TEST_F(LibRadosMiscPP, BigObjectPP) {
+  bufferlist bl;
+  bl.append("abcdefg");
+  ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+
+  {
+    ObjectWriteOperation o;
+    o.truncate(500000000000ull);
+    ASSERT_EQ(-EFBIG, ioctx.operate("foo", &o));
+  }
+  {
+    ObjectWriteOperation o;
+    o.zero(500000000000ull, 1);
+    ASSERT_EQ(-EFBIG, ioctx.operate("foo", &o));
+  }
+  {
+    ObjectWriteOperation o;
+    o.zero(1, 500000000000ull);
+    ASSERT_EQ(-EFBIG, ioctx.operate("foo", &o));
+  }
+  {
+    ObjectWriteOperation o;
+    o.zero(500000000000ull, 500000000000ull);
+    ASSERT_EQ(-EFBIG, ioctx.operate("foo", &o));
+  }
+
+#ifdef __LP64__
+  // this test only works on 64-bit platforms
+  ASSERT_EQ(-EFBIG, ioctx.write("foo", bl, bl.length(), 500000000000ull));
+#endif
+}
+
+TEST_F(LibRadosMiscPP, AioOperatePP) {
+  bool my_aio_complete = false;
+  AioCompletion *my_completion = cluster.aio_create_completion(
+	  (void*)&my_aio_complete, set_completion_complete);
+  AioCompletion *my_completion_null = NULL;
+  ASSERT_NE(my_completion, my_completion_null);
+
+  ObjectWriteOperation o;
+  {
+    bufferlist bl;
+    o.write(0, bl);
+  }
+  std::string val1("val1");
+  {
+    bufferlist bl;
+    bl.append(val1.c_str(), val1.size() + 1);
+    o.setxattr("key1", bl);
+    bufferlist bl2;
+    char buf2[1024];
+    memset(buf2, 0xdd, sizeof(buf2));
+    bl2.append(buf2, sizeof(buf2));
+    o.append(bl2);
+  }
+  ASSERT_EQ(0, ioctx.aio_operate("foo", my_completion, &o));
+  ASSERT_EQ(0, my_completion->wait_for_complete_and_cb());
+  ASSERT_EQ(my_aio_complete, true);
+  my_completion->release();
+
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(1024U, size);
+}
+
+TEST_F(LibRadosMiscPP, AssertExistsPP) {
+  char buf[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  ObjectWriteOperation op;
+  op.assert_exists();
+  op.write(0, bl);
+  ASSERT_EQ(-ENOENT, ioctx.operate("asdffoo", &op));
+  ASSERT_EQ(0, ioctx.create("asdffoo", true));
+  ASSERT_EQ(0, ioctx.operate("asdffoo", &op));
+  ASSERT_EQ(-EEXIST, ioctx.create("asdffoo", true));
+}
+
+TEST_F(LibRadosMiscPP, AssertVersionPP) {
+  char buf[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  // Create test object...
+  ASSERT_EQ(0, ioctx.create("asdfbar", true));
+  // ...then write it again to guarantee that the
+  // (unsigned) version must be at least 1 (not 0)
+  // since we want to decrement it by 1 later.
+  ASSERT_EQ(0, ioctx.write_full("asdfbar", bl));
+
+  uint64_t v = ioctx.get_last_version();
+  ObjectWriteOperation op1;
+  op1.assert_version(v+1);
+  op1.write(0, bl);
+  ASSERT_EQ(-EOVERFLOW, ioctx.operate("asdfbar", &op1));
+  ObjectWriteOperation op2;
+  op2.assert_version(v-1);
+  op2.write(0, bl);
+  ASSERT_EQ(-ERANGE, ioctx.operate("asdfbar", &op2));
+  ObjectWriteOperation op3;
+  op3.assert_version(v);
+  op3.write(0, bl);
+  ASSERT_EQ(0, ioctx.operate("asdfbar", &op3));
+}
+
+TEST_F(LibRadosMiscPP, BigAttrPP) {
+  char buf[64];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  ASSERT_EQ(0, ioctx.create("foo", true));
+
+  bufferlist got;
+
+  cout << "osd_max_attr_size = " << g_conf()->osd_max_attr_size << std::endl;
+  if (g_conf()->osd_max_attr_size) {
+    bl.clear();
+    got.clear();
+    bl.append(buffer::create(g_conf()->osd_max_attr_size));
+    ASSERT_EQ(0, ioctx.setxattr("foo", "one", bl));
+    ASSERT_EQ((int)bl.length(), ioctx.getxattr("foo", "one", got));
+    ASSERT_TRUE(bl.contents_equal(got));
+
+    bl.clear();
+    bl.append(buffer::create(g_conf()->osd_max_attr_size+1));
+    ASSERT_EQ(-EFBIG, ioctx.setxattr("foo", "one", bl));
+  } else {
+    cout << "osd_max_attr_size == 0; skipping test" << std::endl;
+  }
+
+  for (int i=0; i<1000; i++) {
+    bl.clear();
+    got.clear();
+    bl.append(buffer::create(std::min<uint64_t>(g_conf()->osd_max_attr_size,
+						1024)));
+    char n[10];
+    snprintf(n, sizeof(n), "a%d", i);
+    ASSERT_EQ(0, ioctx.setxattr("foo", n, bl));
+    ASSERT_EQ((int)bl.length(), ioctx.getxattr("foo", n, got));
+    ASSERT_TRUE(bl.contents_equal(got));
+  }
+}
+
+TEST_F(LibRadosMiscPP, CopyPP) {
+  SKIP_IF_CRIMSON();
+  bufferlist bl, x;
+  bl.append("hi there");
+  x.append("bar");
+
+  // small object
+  bufferlist blc = bl;
+  bufferlist xc = x;
+  ASSERT_EQ(0, ioctx.write_full("foo", blc));
+  ASSERT_EQ(0, ioctx.setxattr("foo", "myattr", xc));
+
+  version_t uv = ioctx.get_last_version();
+  {
+    // pass future version
+    ObjectWriteOperation op;
+    op.copy_from("foo", ioctx, uv + 1, LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+    ASSERT_EQ(-EOVERFLOW, ioctx.operate("foo.copy", &op));
+  }
+  {
+    // pass old version
+    ObjectWriteOperation op;
+    op.copy_from("foo", ioctx, uv - 1, LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+    ASSERT_EQ(-ERANGE, ioctx.operate("foo.copy", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.copy_from("foo", ioctx, uv, LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+    ASSERT_EQ(0, ioctx.operate("foo.copy", &op));
+
+    bufferlist bl2, x2;
+    ASSERT_EQ((int)bl.length(), ioctx.read("foo.copy", bl2, 10000, 0));
+    ASSERT_TRUE(bl.contents_equal(bl2));
+    ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy", "myattr", x2));
+    ASSERT_TRUE(x.contents_equal(x2));
+  }
+
+  // small object without a version
+  {
+    ObjectWriteOperation op;
+    op.copy_from("foo", ioctx, 0, LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+    ASSERT_EQ(0, ioctx.operate("foo.copy2", &op));
+
+    bufferlist bl2, x2;
+    ASSERT_EQ((int)bl.length(), ioctx.read("foo.copy2", bl2, 10000, 0));
+    ASSERT_TRUE(bl.contents_equal(bl2));
+    ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy2", "myattr", x2));
+    ASSERT_TRUE(x.contents_equal(x2));
+  }
+
+  // do a big object
+  bl.append(buffer::create(g_conf()->osd_copyfrom_max_chunk * 3));
+  bl.zero();
+  bl.append("tail");
+  blc = bl;
+  xc = x;
+  ASSERT_EQ(0, ioctx.write_full("big", blc));
+  ASSERT_EQ(0, ioctx.setxattr("big", "myattr", xc));
+
+  {
+    ObjectWriteOperation op;
+    op.copy_from("big", ioctx, ioctx.get_last_version(), LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
+    ASSERT_EQ(0, ioctx.operate("big.copy", &op));
+
+    bufferlist bl2, x2;
+    ASSERT_EQ((int)bl.length(), ioctx.read("big.copy", bl2, bl.length(), 0));
+    ASSERT_TRUE(bl.contents_equal(bl2));
+    ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy", "myattr", x2));
+    ASSERT_TRUE(x.contents_equal(x2));
+  }
+
+  {
+    ObjectWriteOperation op;
+    op.copy_from("big", ioctx, 0, LIBRADOS_OP_FLAG_FADVISE_SEQUENTIAL);
+    ASSERT_EQ(0, ioctx.operate("big.copy2", &op));
+
+    bufferlist bl2, x2;
+    ASSERT_EQ((int)bl.length(), ioctx.read("big.copy2", bl2, bl.length(), 0));
+    ASSERT_TRUE(bl.contents_equal(bl2));
+    ASSERT_EQ((int)x.length(), ioctx.getxattr("foo.copy2", "myattr", x2));
+    ASSERT_TRUE(x.contents_equal(x2));
+  }
+}
+
+class LibRadosTwoPoolsECPP : public RadosTestECPP
+{
+public:
+  LibRadosTwoPoolsECPP() {};
+  ~LibRadosTwoPoolsECPP() override {};
+protected:
+  static void SetUpTestCase() {
+    SKIP_IF_CRIMSON();
+    pool_name = get_temp_pool_name();
+    ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
+    src_pool_name = get_temp_pool_name();
+    ASSERT_EQ(0, s_cluster.pool_create(src_pool_name.c_str()));
+
+    librados::IoCtx ioctx;
+    ASSERT_EQ(0, s_cluster.ioctx_create(pool_name.c_str(), ioctx));
+    ioctx.application_enable("rados", true);
+
+    librados::IoCtx src_ioctx;
+    ASSERT_EQ(0, s_cluster.ioctx_create(src_pool_name.c_str(), src_ioctx));
+    src_ioctx.application_enable("rados", true);
+  }
+  static void TearDownTestCase() {
+    SKIP_IF_CRIMSON();
+    ASSERT_EQ(0, s_cluster.pool_delete(src_pool_name.c_str()));
+    ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
+  }
+  static std::string src_pool_name;
+
+  void SetUp() override {
+    SKIP_IF_CRIMSON();
+    RadosTestECPP::SetUp();
+    ASSERT_EQ(0, cluster.ioctx_create(src_pool_name.c_str(), src_ioctx));
+    src_ioctx.set_namespace(nspace);
+  }
+  void TearDown() override {
+    SKIP_IF_CRIMSON();
+    // wait for maps to settle before next test
+    cluster.wait_for_latest_osdmap();
+
+    RadosTestECPP::TearDown();
+
+    cleanup_default_namespace(src_ioctx);
+    cleanup_namespace(src_ioctx, nspace);
+
+    src_ioctx.close();
+  }
+
+  librados::IoCtx src_ioctx;
+};
+std::string LibRadosTwoPoolsECPP::src_pool_name;
+
+//copy_from between ecpool and no-ecpool.
+TEST_F(LibRadosTwoPoolsECPP, CopyFrom) {
+  SKIP_IF_CRIMSON();
+  bufferlist z;
+  z.append_zero(4194304*2);
+  bufferlist b;
+  b.append("copyfrom");
+
+  // create big object w/ omapheader
+  {
+    ASSERT_EQ(0, src_ioctx.write_full("foo", z));
+    ASSERT_EQ(0, src_ioctx.omap_set_header("foo", b));
+    version_t uv = src_ioctx.get_last_version();
+    ObjectWriteOperation op;
+    op.copy_from("foo", src_ioctx, uv, 0);
+    ASSERT_EQ(-EOPNOTSUPP, ioctx.operate("foo.copy", &op));
+  }
+
+  // same with small object
+  {
+    ASSERT_EQ(0, src_ioctx.omap_set_header("bar", b));
+    version_t uv = src_ioctx.get_last_version();
+    ObjectWriteOperation op;
+    op.copy_from("bar", src_ioctx, uv, 0);
+    ASSERT_EQ(-EOPNOTSUPP, ioctx.operate("bar.copy", &op));
+  }
+}
+
+TEST_F(LibRadosMiscPP, CopyScrubPP) {
+  SKIP_IF_CRIMSON();
+  bufferlist inbl, bl, x;
+  for (int i=0; i<100; ++i)
+    x.append("barrrrrrrrrrrrrrrrrrrrrrrrrr");
+  bl.append(buffer::create(g_conf()->osd_copyfrom_max_chunk * 3));
+  bl.zero();
+  bl.append("tail");
+  bufferlist cbl;
+
+  map<string, bufferlist> to_set;
+  for (int i=0; i<1000; ++i)
+    to_set[string("foo") + stringify(i)] = x;
+
+  // small
+  cbl = x;
+  ASSERT_EQ(0, ioctx.write_full("small", cbl));
+  ASSERT_EQ(0, ioctx.setxattr("small", "myattr", x));
+
+  // big
+  cbl = bl;
+  ASSERT_EQ(0, ioctx.write_full("big", cbl));
+
+  // without header
+  cbl = bl;
+  ASSERT_EQ(0, ioctx.write_full("big2", cbl));
+  ASSERT_EQ(0, ioctx.setxattr("big2", "myattr", x));
+  ASSERT_EQ(0, ioctx.setxattr("big2", "myattr2", x));
+  ASSERT_EQ(0, ioctx.omap_set("big2", to_set));
+
+  // with header
+  cbl = bl;
+  ASSERT_EQ(0, ioctx.write_full("big3", cbl));
+  ASSERT_EQ(0, ioctx.omap_set_header("big3", x));
+  ASSERT_EQ(0, ioctx.omap_set("big3", to_set));
+
+  // deep scrub to ensure digests are in place
+  {
+    for (int i=0; i<10; ++i) {
+      ostringstream ss;
+      ss << "{\"prefix\": \"pg deep-scrub\", \"pgid\": \""
+	 << ioctx.get_id() << "." << i
+	 << "\"}";
+      cluster.mon_command(ss.str(), inbl, NULL, NULL);
+    }
+
+    // give it a few seconds to go.  this is sloppy but is usually enough time
+    cout << "waiting for initial deep scrubs..." << std::endl;
+    sleep(30);
+    cout << "done waiting, doing copies" << std::endl;
+  }
+
+  {
+    ObjectWriteOperation op;
+    op.copy_from("small", ioctx, 0, 0);
+    ASSERT_EQ(0, ioctx.operate("small.copy", &op));
+  }
+
+  {
+    ObjectWriteOperation op;
+    op.copy_from("big", ioctx, 0, 0);
+    ASSERT_EQ(0, ioctx.operate("big.copy", &op));
+  }
+
+  {
+    ObjectWriteOperation op;
+    op.copy_from("big2", ioctx, 0, 0);
+    ASSERT_EQ(0, ioctx.operate("big2.copy", &op));
+  }
+
+  {
+    ObjectWriteOperation op;
+    op.copy_from("big3", ioctx, 0, 0);
+    ASSERT_EQ(0, ioctx.operate("big3.copy", &op));
+  }
+
+  // deep scrub to ensure digests are correct
+  {
+    for (int i=0; i<10; ++i) {
+      ostringstream ss;
+      ss << "{\"prefix\": \"pg deep-scrub\", \"pgid\": \""
+	 << ioctx.get_id() << "." << i
+	 << "\"}";
+      cluster.mon_command(ss.str(), inbl, NULL, NULL);
+    }
+
+    // give it a few seconds to go.  this is sloppy but is usually enough time
+    cout << "waiting for final deep scrubs..." << std::endl;
+    sleep(30);
+    cout << "done waiting" << std::endl;
+  }
+}
+
+TEST_F(LibRadosMiscPP, WriteSamePP) {
+  bufferlist bl;
+  char buf[128];
+  bufferlist fl;
+  char full[128 * 4];
+  char *cmp;
+
+  /* zero the full range before using writesame */
+  memset(full, 0, sizeof(full));
+  fl.append(full, sizeof(full));
+  ASSERT_EQ(0, ioctx.write("ws", fl, fl.length(), 0));
+
+  memset(buf, 0xcc, sizeof(buf));
+  bl.clear();
+  bl.append(buf, sizeof(buf));
+  /* write the same buf four times */
+  ASSERT_EQ(0, ioctx.writesame("ws", bl, sizeof(full), 0));
+
+  /* read back the full buffer and confirm that it matches */
+  fl.clear();
+  fl.append(full, sizeof(full));
+  ASSERT_EQ((int)fl.length(), ioctx.read("ws", fl, fl.length(), 0));
+
+  for (cmp = fl.c_str(); cmp < fl.c_str() + fl.length(); cmp += sizeof(buf)) {
+    ASSERT_EQ(0, memcmp(cmp, buf, sizeof(buf)));
+  }
+
+  /* write_len not a multiple of data_len should throw error */
+  bl.clear();
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(-EINVAL, ioctx.writesame("ws", bl, (sizeof(buf) * 4) - 1, 0));
+  ASSERT_EQ(-EINVAL,
+	    ioctx.writesame("ws", bl, bl.length() / 2, 0));
+  /* write_len = data_len, i.e. same as write() */
+  ASSERT_EQ(0, ioctx.writesame("ws", bl, sizeof(buf), 0));
+  bl.clear();
+  ASSERT_EQ(-EINVAL,
+	    ioctx.writesame("ws", bl, sizeof(buf), 0));
+}
+
+template <typename T>
+class LibRadosChecksum : public LibRadosMiscPP {
+public:
+  typedef typename T::alg_t alg_t;
+  typedef typename T::value_t value_t;
+  typedef typename alg_t::init_value_t init_value_t;
+
+  static const rados_checksum_type_t type = T::type;
+
+  bufferlist content_bl;
+
+  using LibRadosMiscPP::SetUpTestCase;
+  using LibRadosMiscPP::TearDownTestCase;
+
+  void SetUp() override {
+    LibRadosMiscPP::SetUp();
+
+    std::string content(4096, '\0');
+    for (size_t i = 0; i < content.length(); ++i) {
+      content[i] = static_cast<char>(rand() % (126 - 33) + 33);
+    }
+    content_bl.append(content);
+    ASSERT_EQ(0, ioctx.write("foo", content_bl, content_bl.length(), 0));
+  }
+};
+
+template <rados_checksum_type_t _type, typename AlgT, typename ValueT>
+class LibRadosChecksumParams {
+public:
+  typedef AlgT alg_t;
+  typedef ValueT value_t;
+  static const rados_checksum_type_t type = _type;
+};
+
+typedef ::testing::Types<
+    LibRadosChecksumParams<LIBRADOS_CHECKSUM_TYPE_XXHASH32,
+			   Checksummer::xxhash32, ceph_le32>,
+    LibRadosChecksumParams<LIBRADOS_CHECKSUM_TYPE_XXHASH64,
+			   Checksummer::xxhash64, ceph_le64>,
+    LibRadosChecksumParams<LIBRADOS_CHECKSUM_TYPE_CRC32C,
+			   Checksummer::crc32c, ceph_le32>
+  > LibRadosChecksumTypes;
+
+TYPED_TEST_SUITE(LibRadosChecksum, LibRadosChecksumTypes);
+
+TYPED_TEST(LibRadosChecksum, Subset) {
+  uint32_t chunk_size = 1024;
+  uint32_t csum_count = this->content_bl.length() / chunk_size;
+
+  typename TestFixture::init_value_t init_value = -1;
+  bufferlist init_value_bl;
+  encode(init_value, init_value_bl);
+
+  std::vector<bufferlist> checksum_bls(csum_count);
+  std::vector<int> checksum_rvals(csum_count);
+
+  // individual checksum ops for each chunk
+  ObjectReadOperation op;
+  for (uint32_t i = 0; i < csum_count; ++i) {
+    op.checksum(TestFixture::type, init_value_bl, i * chunk_size, chunk_size,
+		0, &checksum_bls[i], &checksum_rvals[i]);
+  }
+  ASSERT_EQ(0, this->ioctx.operate("foo", &op, NULL));
+
+  for (uint32_t i = 0; i < csum_count; ++i) {
+    ASSERT_EQ(0, checksum_rvals[i]);
+
+    auto bl_it = checksum_bls[i].cbegin();
+    uint32_t count;
+    decode(count, bl_it);
+    ASSERT_EQ(1U, count);
+
+    typename TestFixture::value_t value;
+    decode(value, bl_it);
+
+    bufferlist content_sub_bl;
+    content_sub_bl.substr_of(this->content_bl, i * chunk_size, chunk_size);
+
+    typename TestFixture::value_t expected_value;
+    bufferptr expected_value_bp = buffer::create_static(
+      sizeof(expected_value), reinterpret_cast<char*>(&expected_value));
+    Checksummer::template calculate<typename TestFixture::alg_t>(
+      init_value, chunk_size, 0, chunk_size, content_sub_bl,
+      &expected_value_bp);
+    ASSERT_EQ(expected_value, value);
+  }
+}
+
+TYPED_TEST(LibRadosChecksum, Chunked) {
+  uint32_t chunk_size = 1024;
+  uint32_t csum_count = this->content_bl.length() / chunk_size;
+
+  typename TestFixture::init_value_t init_value = -1;
+  bufferlist init_value_bl;
+  encode(init_value, init_value_bl);
+
+  bufferlist checksum_bl;
+  int checksum_rval;
+
+  // single op with chunked checksum results
+  ObjectReadOperation op;
+  op.checksum(TestFixture::type, init_value_bl, 0, this->content_bl.length(),
+	      chunk_size, &checksum_bl, &checksum_rval);
+  ASSERT_EQ(0, this->ioctx.operate("foo", &op, NULL));
+  ASSERT_EQ(0, checksum_rval);
+
+  auto bl_it = checksum_bl.cbegin();
+  uint32_t count;
+  decode(count, bl_it);
+  ASSERT_EQ(csum_count, count);
+
+  std::vector<typename TestFixture::value_t> expected_values(csum_count);
+  bufferptr expected_values_bp = buffer::create_static(
+    csum_count * sizeof(typename TestFixture::value_t),
+    reinterpret_cast<char*>(&expected_values[0]));
+
+  Checksummer::template calculate<typename TestFixture::alg_t>(
+    init_value, chunk_size, 0, this->content_bl.length(), this->content_bl,
+    &expected_values_bp);
+
+  for (uint32_t i = 0; i < csum_count; ++i) {
+    typename TestFixture::value_t value;
+    decode(value, bl_it);
+    ASSERT_EQ(expected_values[i], value);
+  }
+}
+
+TEST_F(LibRadosMiscPP, CmpExtPP) {
+  bufferlist cmp_bl, bad_cmp_bl, write_bl;
+  char stored_str[] = "1234567891";
+  char mismatch_str[] = "1234577777";
+
+  write_bl.append(stored_str);
+  ioctx.write("cmpextpp", write_bl, write_bl.length(), 0);
+  cmp_bl.append(stored_str);
+  ASSERT_EQ(0, ioctx.cmpext("cmpextpp", 0, cmp_bl));
+
+  bad_cmp_bl.append(mismatch_str);
+  ASSERT_EQ(-MAX_ERRNO - 5, ioctx.cmpext("cmpextpp", 0, bad_cmp_bl));
+}
+
+TEST_F(LibRadosMiscPP, Applications) {
+  bufferlist inbl, outbl;
+  string outs;
+  ASSERT_EQ(0, cluster.mon_command("{\"prefix\": \"osd dump\"}",
+				   inbl, &outbl, &outs));
+  ASSERT_LT(0u, outbl.length());
+  ASSERT_LE(0u, outs.length());
+  if (!std::regex_search(outbl.to_str(),
+			 std::regex("require_osd_release [l-z]"))) {
+    std::cout << "SKIPPING";
+    return;
+  }
+
+  std::set<std::string> expected_apps = {"rados"};
+  std::set<std::string> apps;
+  ASSERT_EQ(0, ioctx.application_list(&apps));
+  ASSERT_EQ(expected_apps, apps);
+
+  ASSERT_EQ(0, ioctx.application_enable("app1", true));
+  ASSERT_EQ(-EPERM, ioctx.application_enable("app2", false));
+  ASSERT_EQ(0, ioctx.application_enable("app2", true));
+
+  expected_apps = {"app1", "app2", "rados"};
+  ASSERT_EQ(0, ioctx.application_list(&apps));
+  ASSERT_EQ(expected_apps, apps);
+
+  std::map<std::string, std::string> expected_meta;
+  std::map<std::string, std::string> meta;
+  ASSERT_EQ(-ENOENT, ioctx.application_metadata_list("dne", &meta));
+  ASSERT_EQ(0, ioctx.application_metadata_list("app1", &meta));
+  ASSERT_EQ(expected_meta, meta);
+
+  ASSERT_EQ(-ENOENT, ioctx.application_metadata_set("dne", "key1", "value1"));
+  ASSERT_EQ(0, ioctx.application_metadata_set("app1", "key1", "value1"));
+  ASSERT_EQ(0, ioctx.application_metadata_set("app1", "key2", "value2"));
+
+  expected_meta = {{"key1", "value1"}, {"key2", "value2"}};
+  ASSERT_EQ(0, ioctx.application_metadata_list("app1", &meta));
+  ASSERT_EQ(expected_meta, meta);
+
+  ASSERT_EQ(0, ioctx.application_metadata_remove("app1", "key1"));
+
+  expected_meta = {{"key2", "value2"}};
+  ASSERT_EQ(0, ioctx.application_metadata_list("app1", &meta));
+  ASSERT_EQ(expected_meta, meta);
+}
+
+TEST_F(LibRadosMiscECPP, CompareExtentRange) {
+  SKIP_IF_CRIMSON();
+  bufferlist bl1;
+  bl1.append("ceph");
+  ObjectWriteOperation write;
+  write.write(0, bl1);
+  ASSERT_EQ(0, ioctx.operate("foo", &write));
+
+  bufferlist bl2;
+  bl2.append("ph");
+  bl2.append(std::string(2, '\0'));
+  ObjectReadOperation read1;
+  read1.cmpext(2, bl2, nullptr);
+  ASSERT_EQ(0, ioctx.operate("foo", &read1, nullptr));
+
+  bufferlist bl3;
+  bl3.append(std::string(4, '\0'));
+  ObjectReadOperation read2;
+  read2.cmpext(2097152, bl3, nullptr);
+  ASSERT_EQ(0, ioctx.operate("foo", &read2, nullptr));
+}
+
+TEST_F(LibRadosMiscPP, MinCompatOSD) {
+  int8_t require_osd_release;
+  ASSERT_EQ(0, cluster.get_min_compatible_osd(&require_osd_release));
+  ASSERT_LE(-1, require_osd_release);
+  ASSERT_GT(CEPH_RELEASE_MAX, require_osd_release);
+}
+
+TEST_F(LibRadosMiscPP, MinCompatClient) {
+  int8_t min_compat_client;
+  int8_t require_min_compat_client;
+  ASSERT_EQ(0, cluster.get_min_compatible_client(&min_compat_client,
+                                                 &require_min_compat_client));
+  ASSERT_LE(-1, min_compat_client);
+  ASSERT_GT(CEPH_RELEASE_MAX, min_compat_client);
+
+  ASSERT_LE(-1, require_min_compat_client);
+  ASSERT_GT(CEPH_RELEASE_MAX, require_min_compat_client);
+}
+
+TEST_F(LibRadosMiscPP, Conf) {
+  const char* const option = "bluestore_throttle_bytes";
+  size_t new_size = 1 << 20;
+  std::string original;
+  ASSERT_EQ(0, cluster.conf_get(option, original));
+  auto restore_setting = make_scope_guard([&] {
+    cluster.conf_set(option, original.c_str());
+  });
+  std::string expected = std::to_string(new_size);
+  ASSERT_EQ(0, cluster.conf_set(option, expected.c_str()));
+  std::string actual;
+  ASSERT_EQ(0, cluster.conf_get(option, actual));
+  ASSERT_EQ(expected, actual);
+}
diff --git a/src/test/librados/op_speed.cc b/src/test/librados/op_speed.cc
new file mode 100644
index 000000000..849a6566f
--- /dev/null
+++ b/src/test/librados/op_speed.cc
@@ -0,0 +1,24 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
+#include <cstdint>
+
+#include "include/rados/librados.hpp"
+
+constexpr int to_create = 10'000'000;
+
+int main() {
+  for (int i = 0; i < to_create; ++i) {
+    librados::ObjectReadOperation op;
+    bufferlist bl;
+    std::uint64_t sz;
+    struct timespec tm;
+    std::map<std::string, ceph::buffer::list> xattrs;
+    std::map<std::string, ceph::buffer::list> omap;
+    bool more;
+    op.read(0, 0, &bl, nullptr);
+    op.stat2(&sz, &tm, nullptr);
+    op.getxattrs(&xattrs, nullptr);
+    op.omap_get_vals2({}, 1000, &omap, &more, nullptr);
+  }
+}
diff --git a/src/test/librados/pool.cc b/src/test/librados/pool.cc
new file mode 100644
index 000000000..a85d8f3a8
--- /dev/null
+++ b/src/test/librados/pool.cc
@@ -0,0 +1,186 @@
+#include <errno.h>
+#include <vector>
+#include "crimson_utils.h"
+#include "gtest/gtest.h"
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+
+#define POOL_LIST_BUF_SZ 32768
+
+TEST(LibRadosPools, PoolList) {
+  char pool_list_buf[POOL_LIST_BUF_SZ];
+  char *buf = pool_list_buf;
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  ASSERT_LT(rados_pool_list(cluster, buf, POOL_LIST_BUF_SZ), POOL_LIST_BUF_SZ);
+
+  // we can pass a null buffer too.
+  ASSERT_LT(rados_pool_list(cluster, NULL, POOL_LIST_BUF_SZ), POOL_LIST_BUF_SZ);
+
+  bool found_pool = false;
+  int firstlen = 0;
+  while (buf[0] != '\0') {
+    if ((found_pool == false) && (strcmp(buf, pool_name.c_str()) == 0)) {
+      found_pool = true;
+    }
+    if (!firstlen)
+      firstlen = strlen(buf) + 1;
+    buf += strlen(buf) + 1;
+  }
+  ASSERT_EQ(found_pool, true);
+
+  // make sure we honor the buffer size limit
+  buf = pool_list_buf;
+  memset(buf, 0, POOL_LIST_BUF_SZ);
+  ASSERT_LT(rados_pool_list(cluster, buf, firstlen), POOL_LIST_BUF_SZ);
+  ASSERT_NE(0, buf[0]);  // include at least one pool name
+  ASSERT_EQ(0, buf[firstlen]);  // but don't touch the stopping point
+
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+int64_t rados_pool_lookup(rados_t cluster, const char *pool_name);
+
+TEST(LibRadosPools, PoolLookup) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  ASSERT_LT(0, rados_pool_lookup(cluster, pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosPools, PoolLookup2) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  int64_t pool_id = rados_pool_lookup(cluster, pool_name.c_str());
+  ASSERT_GT(pool_id, 0);
+  rados_ioctx_t ioctx;
+  ASSERT_EQ(0, rados_ioctx_create(cluster, pool_name.c_str(), &ioctx));
+  int64_t pool_id2 = rados_ioctx_get_id(ioctx);
+  ASSERT_EQ(pool_id, pool_id2);
+  rados_ioctx_destroy(ioctx);
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosPools, PoolLookupOtherInstance) {
+  rados_t cluster1;
+  ASSERT_EQ("", connect_cluster(&cluster1));
+
+  rados_t cluster2;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster2));
+  int64_t pool_id = rados_pool_lookup(cluster2, pool_name.c_str());
+  ASSERT_GT(pool_id, 0);
+
+  ASSERT_EQ(pool_id, rados_pool_lookup(cluster1, pool_name.c_str()));
+
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster2));
+  rados_shutdown(cluster1);
+}
+
+TEST(LibRadosPools, PoolReverseLookupOtherInstance) {
+  rados_t cluster1;
+  ASSERT_EQ("", connect_cluster(&cluster1));
+
+  rados_t cluster2;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster2));
+  int64_t pool_id = rados_pool_lookup(cluster2, pool_name.c_str());
+  ASSERT_GT(pool_id, 0);
+
+  char buf[100];
+  ASSERT_LT(0, rados_pool_reverse_lookup(cluster1, pool_id, buf, 100));
+  ASSERT_EQ(0, strcmp(buf, pool_name.c_str()));
+
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster2));
+  rados_shutdown(cluster1);
+}
+
+TEST(LibRadosPools, PoolDelete) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  ASSERT_EQ(0, rados_pool_delete(cluster, pool_name.c_str()));
+  ASSERT_GT(0, rados_pool_lookup(cluster, pool_name.c_str()));
+  ASSERT_EQ(0, rados_pool_create(cluster, pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosPools, PoolCreateDelete) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+
+  std::string n = pool_name + "abc123";
+  ASSERT_EQ(0, rados_pool_create(cluster, n.c_str()));
+  ASSERT_EQ(-EEXIST, rados_pool_create(cluster, n.c_str()));
+  ASSERT_EQ(0, rados_pool_delete(cluster, n.c_str()));
+  ASSERT_EQ(-ENOENT, rados_pool_delete(cluster, n.c_str()));
+
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosPools, PoolCreateWithCrushRule) {
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+
+  std::string pool2_name = get_temp_pool_name();
+  ASSERT_EQ(0, rados_pool_create_with_crush_rule(cluster,
+			    pool2_name.c_str(), 0));
+  ASSERT_EQ(0, rados_pool_delete(cluster, pool2_name.c_str()));
+
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
+
+TEST(LibRadosPools, PoolGetBaseTier) {
+  SKIP_IF_CRIMSON();
+  rados_t cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool(pool_name, &cluster));
+  std::string tier_pool_name = pool_name + "-cache";
+  ASSERT_EQ(0, rados_pool_create(cluster, tier_pool_name.c_str()));
+
+  int64_t pool_id = rados_pool_lookup(cluster, pool_name.c_str());
+  ASSERT_GE(pool_id, 0);
+
+  int64_t tier_pool_id = rados_pool_lookup(cluster, tier_pool_name.c_str());
+  ASSERT_GE(tier_pool_id, 0);
+
+
+  int64_t base_tier = 0;
+  EXPECT_EQ(0, rados_pool_get_base_tier(cluster, pool_id, &base_tier));
+  EXPECT_EQ(pool_id, base_tier);
+
+  std::string cmdstr = "{\"prefix\": \"osd tier add\", \"pool\": \"" +
+     pool_name + "\", \"tierpool\":\"" + tier_pool_name + "\", \"force_nonempty\":\"\"}";
+  char *cmd[1];
+  cmd[0] = (char *)cmdstr.c_str();
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+
+  cmdstr = "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" +
+     tier_pool_name + "\", \"mode\":\"readonly\"," +
+    " \"yes_i_really_mean_it\": true}";
+  cmd[0] = (char *)cmdstr.c_str();
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+
+  EXPECT_EQ(0, rados_wait_for_latest_osdmap(cluster));
+
+  EXPECT_EQ(0, rados_pool_get_base_tier(cluster, pool_id, &base_tier));
+  EXPECT_EQ(pool_id, base_tier);
+
+  EXPECT_EQ(0, rados_pool_get_base_tier(cluster, tier_pool_id, &base_tier));
+  EXPECT_EQ(pool_id, base_tier);
+
+  int64_t nonexistent_pool_id = (int64_t)((-1ULL) >> 1);
+  EXPECT_EQ(-ENOENT, rados_pool_get_base_tier(cluster, nonexistent_pool_id, &base_tier));
+
+  cmdstr = "{\"prefix\": \"osd tier remove\", \"pool\": \"" +
+     pool_name + "\", \"tierpool\":\"" + tier_pool_name + "\"}";
+  cmd[0] = (char *)cmdstr.c_str();
+  ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+  ASSERT_EQ(0, rados_pool_delete(cluster, tier_pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_pool(pool_name, &cluster));
+}
diff --git a/src/test/librados/service.cc b/src/test/librados/service.cc
new file mode 100644
index 000000000..6d88f0da9
--- /dev/null
+++ b/src/test/librados/service.cc
@@ -0,0 +1,209 @@
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "common/config_proxy.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+#include <mutex>
+#include <condition_variable>
+#include <algorithm>
+#include <thread>
+#include <errno.h>
+#include "gtest/gtest.h"
+#include "test/unit.cc"
+
+using namespace std;
+using namespace librados;
+
+TEST(LibRadosService, RegisterEarly) {
+  rados_t cluster;
+  ASSERT_EQ(0, rados_create(&cluster, "admin"));
+  ASSERT_EQ(0, rados_conf_read_file(cluster, NULL));
+  ASSERT_EQ(0, rados_conf_parse_env(cluster, NULL));
+
+  string name = string("pid") + stringify(getpid());
+  ASSERT_EQ(0, rados_service_register(cluster, "laundry", name.c_str(),
+                                      "foo\0bar\0this\0that\0"));
+  ASSERT_EQ(-EEXIST, rados_service_register(cluster, "laundry", name.c_str(),
+                                            "foo\0bar\0this\0that\0"));
+
+  ASSERT_EQ(0, rados_connect(cluster));
+  sleep(5);
+  rados_shutdown(cluster);
+}
+
+TEST(LibRadosService, RegisterLate) {
+  rados_t cluster;
+  ASSERT_EQ(0, rados_create(&cluster, "admin"));
+  ASSERT_EQ(0, rados_conf_read_file(cluster, NULL));
+  ASSERT_EQ(0, rados_conf_parse_env(cluster, NULL));
+  ASSERT_EQ(0, rados_connect(cluster));
+
+  string name = string("pid") + stringify(getpid());
+  ASSERT_EQ(0, rados_service_register(cluster, "laundry", name.c_str(),
+                                      "foo\0bar\0this\0that\0"));
+  ASSERT_EQ(-EEXIST, rados_service_register(cluster, "laundry", name.c_str(),
+                                            "foo\0bar\0this\0that\0"));
+  rados_shutdown(cluster);
+}
+
+static void status_format_func(const int i, std::mutex &lock,
+                               std::condition_variable &cond,
+                               int &threads_started, bool &stopped)
+{
+  rados_t cluster;
+  char metadata_buf[4096];
+
+  ASSERT_EQ(0, rados_create(&cluster, "admin"));
+  ASSERT_EQ(0, rados_conf_read_file(cluster, NULL));
+  ASSERT_EQ(0, rados_conf_parse_env(cluster, NULL));
+
+  ASSERT_EQ(0, rados_connect(cluster));
+  if (i == 0) {
+    ASSERT_LT(0, sprintf(metadata_buf, "%s%c%s%c",
+                         "foo", '\0', "bar", '\0'));
+  } else if (i == 1) {
+    ASSERT_LT(0, sprintf(metadata_buf, "%s%c%s%c",
+                         "daemon_type", '\0', "portal", '\0'));
+  } else if (i == 2) {
+    ASSERT_LT(0, sprintf(metadata_buf, "%s%c%s%c",
+                         "daemon_prefix", '\0', "gateway", '\0'));
+  } else {
+    string prefix = string("gw") + stringify(i % 4);
+    string zone = string("z") + stringify(i % 3);
+    ASSERT_LT(0, sprintf(metadata_buf, "%s%c%s%c%s%c%s%c%s%c%s%c%s%c%s%c",
+                         "daemon_type", '\0', "portal", '\0',
+                         "daemon_prefix", '\0', prefix.c_str(), '\0',
+                         "hostname", '\0', prefix.c_str(), '\0',
+                         "zone_id", '\0', zone.c_str(), '\0'));
+  }
+  string name = string("rbd/image") + stringify(i);
+  ASSERT_EQ(0, rados_service_register(cluster, "foo", name.c_str(),
+		                      metadata_buf));
+
+  std::unique_lock<std::mutex> l(lock);
+  threads_started++;
+  cond.notify_all();
+  cond.wait(l, [&stopped] {
+    return stopped;
+  });
+
+  rados_shutdown(cluster);
+}
+
+TEST(LibRadosService, StatusFormat) {
+  const int nthreads = 16;
+  std::thread threads[nthreads];
+  std::mutex lock;
+  std::condition_variable cond;
+  bool stopped = false;
+  int threads_started = 0;
+
+  // no rlimits on Windows
+  #ifndef _WIN32
+  // Need a bunch of fd's for this test
+  struct rlimit rold, rnew;
+  ASSERT_EQ(getrlimit(RLIMIT_NOFILE, &rold), 0);
+  rnew = rold;
+  rnew.rlim_cur = rnew.rlim_max;
+  ASSERT_EQ(setrlimit(RLIMIT_NOFILE, &rnew), 0);
+  #endif
+
+  for (int i = 0; i < nthreads; ++i)
+    threads[i] = std::thread(status_format_func, i, std::ref(lock),
+                             std::ref(cond), std::ref(threads_started),
+                             std::ref(stopped));
+
+  {
+    std::unique_lock<std::mutex> l(lock);
+    cond.wait(l, [&threads_started] {
+      return nthreads == threads_started;
+    });
+  }
+
+  int retry = 60; // mon thrashing may make this take a long time
+  while (retry) {
+    rados_t cluster;
+
+    ASSERT_EQ(0, rados_create(&cluster, "admin"));
+    ASSERT_EQ(0, rados_conf_read_file(cluster, NULL));
+    ASSERT_EQ(0, rados_conf_parse_env(cluster, NULL));
+
+    ASSERT_EQ(0, rados_connect(cluster));
+    JSONFormatter cmd_f;
+    cmd_f.open_object_section("command");
+    cmd_f.dump_string("prefix", "status");
+    cmd_f.close_section();
+    std::ostringstream cmd_stream;
+    cmd_f.flush(cmd_stream);
+    const std::string serialized_cmd = cmd_stream.str();
+    const char *cmd[2];
+    cmd[1] = NULL;
+    cmd[0] = serialized_cmd.c_str();
+    char *outbuf = NULL;
+    size_t outlen = 0;
+    ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0,
+              &outbuf, &outlen, NULL, NULL));
+    std::string out(outbuf, outlen);
+    cout << out << std::endl;
+    bool success = false;
+    auto r1 = out.find("16 portals active (1 hosts, 3 zones)");
+    if (std::string::npos != r1) {
+      success = true;
+    }
+    rados_buffer_free(outbuf);
+    rados_shutdown(cluster);
+
+    if (success || !retry) {
+      break;
+    }
+
+    // wait for 2 seconds to make sure all the
+    // services have been successfully updated
+    // to ceph mon, then retry it.
+    sleep(2);
+    retry--;
+  }
+
+  {
+    std::scoped_lock<std::mutex> l(lock);
+    stopped = true;
+    cond.notify_all();
+  }
+  for (int i = 0; i < nthreads; ++i)
+    threads[i].join();
+
+  ASSERT_NE(0, retry);
+  #ifndef _WIN32
+  ASSERT_EQ(setrlimit(RLIMIT_NOFILE, &rold), 0);
+  #endif
+}
+
+TEST(LibRadosService, Status) {
+  rados_t cluster;
+  ASSERT_EQ(0, rados_create(&cluster, "admin"));
+  ASSERT_EQ(0, rados_conf_read_file(cluster, NULL));
+  ASSERT_EQ(0, rados_conf_parse_env(cluster, NULL));
+
+  ASSERT_EQ(-ENOTCONN, rados_service_update_status(cluster,
+                                                   "testing\0testing\0"));
+
+  ASSERT_EQ(0, rados_connect(cluster));
+  string name = string("pid") + stringify(getpid());
+  ASSERT_EQ(0, rados_service_register(cluster, "laundry", name.c_str(),
+                                      "foo\0bar\0this\0that\0"));
+
+  for (int i=0; i<20; ++i) {
+    char buffer[1024];
+    snprintf(buffer, sizeof(buffer), "%s%c%s%c%s%c%d%c",
+             "testing", '\0', "testing", '\0',
+             "count", '\0', i, '\0');
+    ASSERT_EQ(0, rados_service_update_status(cluster, buffer));
+    sleep(1);
+  }
+  rados_shutdown(cluster);
+}
diff --git a/src/test/librados/service_cxx.cc b/src/test/librados/service_cxx.cc
new file mode 100644
index 000000000..1bf682af8
--- /dev/null
+++ b/src/test/librados/service_cxx.cc
@@ -0,0 +1,105 @@
+#include <algorithm>
+#include <thread>
+#include <errno.h>
+#include "gtest/gtest.h"
+
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "common/config_proxy.h"
+#include "test/librados/test_cxx.h"
+#include "test/librados/testcase_cxx.h"
+#include "test/unit.cc"
+
+using namespace std;
+using namespace librados;
+
+TEST(LibRadosServicePP, RegisterEarly) {
+  Rados cluster;
+  cluster.init("admin");
+  ASSERT_EQ(0, cluster.conf_read_file(NULL));
+  cluster.conf_parse_env(NULL);
+  string name = string("pid") + stringify(getpid());
+  ASSERT_EQ(0, cluster.service_daemon_register(
+	      "laundry", name, {{"foo", "bar"}, {"this", "that"}}));
+  ASSERT_EQ(-EEXIST, cluster.service_daemon_register(
+	      "laundry", name, {{"foo", "bar"}, {"this", "that"}}));
+  ASSERT_EQ(0, cluster.connect());
+  sleep(5);
+  cluster.shutdown();
+}
+
+TEST(LibRadosServicePP, RegisterLate) {
+  Rados cluster;
+  cluster.init("admin");
+  ASSERT_EQ(0, cluster.conf_read_file(NULL));
+  cluster.conf_parse_env(NULL);
+  ASSERT_EQ("", connect_cluster_pp(cluster));
+  string name = string("pid") + stringify(getpid());
+  ASSERT_EQ(0, cluster.service_daemon_register(
+	      "laundry", name, {{"foo", "bar"}, {"this", "that"}}));
+}
+
+TEST(LibRadosServicePP, Status) {
+  Rados cluster;
+  cluster.init("admin");
+  ASSERT_EQ(0, cluster.conf_read_file(NULL));
+  cluster.conf_parse_env(NULL);
+  string name = string("pid") + stringify(getpid());
+  ASSERT_EQ(-ENOTCONN, cluster.service_daemon_update_status(
+	      {{"testing", "starting"}}));
+  ASSERT_EQ(0, cluster.connect());
+  ASSERT_EQ(0, cluster.service_daemon_register(
+	      "laundry", name, {{"foo", "bar"}, {"this", "that"}}));
+  for (int i=0; i<20; ++i) {
+    ASSERT_EQ(0, cluster.service_daemon_update_status({
+	  {"testing", "running"},
+	  {"count", stringify(i)}
+	}));
+    sleep(1);
+  }
+  cluster.shutdown();
+}
+
+TEST(LibRadosServicePP, Close) {
+  int tries = 20;
+  string name = string("close-test-pid") + stringify(getpid());
+  int i;
+  for (i = 0; i < tries; ++i) {
+    cout << "attempt " << i << " of " << tries << std::endl;
+    {
+      Rados cluster;
+      cluster.init("admin");
+      ASSERT_EQ(0, cluster.conf_read_file(NULL));
+      cluster.conf_parse_env(NULL);
+      ASSERT_EQ(0, cluster.connect());
+      ASSERT_EQ(0, cluster.service_daemon_register(
+		  "laundry", name, {{"foo", "bar"}, {"this", "that"}}));
+      sleep(3); // let it register
+      cluster.shutdown();
+    }
+    // mgr updates servicemap every tick
+    //sleep(g_conf().get_val<int64_t>("mgr_tick_period"));
+    std::this_thread::sleep_for(g_conf().get_val<std::chrono::seconds>(
+				  "mgr_tick_period"));
+    // make sure we are deregistered
+    {
+      Rados cluster;
+      cluster.init("admin");
+      ASSERT_EQ(0, cluster.conf_read_file(NULL));
+      cluster.conf_parse_env(NULL);
+      ASSERT_EQ(0, cluster.connect());
+      bufferlist inbl, outbl;
+      ASSERT_EQ(0, cluster.mon_command("{\"prefix\": \"service dump\"}",
+				       inbl, &outbl, NULL));
+      string s = outbl.to_str();
+      cluster.shutdown();
+
+      if (s.find(name) != string::npos) {
+	cout << " failed to deregister:\n" << s << std::endl;
+      } else {
+	break;
+      }
+    }
+  }
+  ASSERT_LT(i, tries);
+}
diff --git a/src/test/librados/snapshots.cc b/src/test/librados/snapshots.cc
new file mode 100644
index 000000000..384447ca2
--- /dev/null
+++ b/src/test/librados/snapshots.cc
@@ -0,0 +1,356 @@
+#include "include/rados.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+#include "crimson_utils.h"
+
+#include <algorithm>
+#include <errno.h>
+#include "gtest/gtest.h"
+#include <string>
+
+using std::string;
+
+typedef RadosTest LibRadosSnapshots;
+typedef RadosTest LibRadosSnapshotsSelfManaged;
+typedef RadosTestEC LibRadosSnapshotsEC;
+typedef RadosTestEC LibRadosSnapshotsSelfManagedEC;
+
+const int bufsize = 128;
+
+TEST_F(LibRadosSnapshots, SnapList) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  rados_snap_t snaps[10];
+  EXPECT_EQ(1, rados_ioctx_snap_list(ioctx, snaps,
+				     sizeof(snaps) / sizeof(snaps[0])));
+  rados_snap_t rid;
+  EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+  EXPECT_EQ(rid, snaps[0]);
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+}
+
+TEST_F(LibRadosSnapshots, SnapRemove) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  rados_snap_t rid;
+  ASSERT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+  ASSERT_EQ(-EEXIST, rados_ioctx_snap_create(ioctx, "snap1"));
+  ASSERT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+  ASSERT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+}
+
+TEST_F(LibRadosSnapshots, Rollback) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  EXPECT_EQ(0, rados_write_full(ioctx, "foo", buf2, sizeof(buf2)));
+  EXPECT_EQ(0, rados_ioctx_snap_rollback(ioctx, "foo", "snap1"));
+  char buf3[sizeof(buf)];
+  EXPECT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  EXPECT_EQ(0, memcmp(buf, buf3, sizeof(buf)));
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+}
+
+TEST_F(LibRadosSnapshots, SnapGetName) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snapfoo"));
+  rados_snap_t rid;
+  EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snapfoo", &rid));
+  EXPECT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snapbar", &rid));
+  char name[128];
+  memset(name, 0, sizeof(name));
+  EXPECT_EQ(0, rados_ioctx_snap_get_name(ioctx, rid, name, sizeof(name)));
+  time_t snaptime;
+  EXPECT_EQ(0, rados_ioctx_snap_get_stamp(ioctx, rid, &snaptime));
+  EXPECT_EQ(0, strcmp(name, "snapfoo"));
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snapfoo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManaged, Snap) {
+  std::vector<uint64_t> my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+
+  my_snaps.push_back(-2);
+  rados_completion_t completion;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr,
+                                           &completion));
+  rados_aio_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back(), completion);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(completion));
+  rados_aio_release(completion);
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+  rados_ioctx_snap_set_read(ioctx, my_snaps[1]-1);
+  char buf3[sizeof(buf)];
+  ASSERT_EQ(-ENOENT, rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+
+  rados_ioctx_snap_set_read(ioctx, my_snaps[1]);
+  ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr,
+                                           &completion));
+  rados_aio_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back(), completion);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(completion));
+  rados_aio_release(completion);
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManaged, Rollback) {
+  std::vector<uint64_t> my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  // First write
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  // Second write
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+  // Rollback to my_snaps[1] - Object is expeceted to conatin the first write
+  rados_ioctx_selfmanaged_snap_rollback(ioctx, "foo", my_snaps[1]);
+  char buf3[sizeof(buf)];
+  ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, sizeof(buf)));
+
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManaged, FutureSnapRollback) {
+  std::vector<uint64_t> my_snaps;
+  // Snapshot 1
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  // First write
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+
+  // Snapshot 2
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  // Second write
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+  // Snapshot 3
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+
+  // Rollback to the last snap id - Object is expected to conatin
+  // latest write (head object)
+  rados_ioctx_selfmanaged_snap_rollback(ioctx, "foo", my_snaps[2]);
+  char buf3[sizeof(buf)];
+  ASSERT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  ASSERT_EQ(0, memcmp(buf3, buf2, sizeof(buf)));
+
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+}
+
+
+
+// EC testing
+TEST_F(LibRadosSnapshotsEC, SnapList) {
+  SKIP_IF_CRIMSON();
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  rados_snap_t snaps[10];
+  EXPECT_EQ(1, rados_ioctx_snap_list(ioctx, snaps,
+				     sizeof(snaps) / sizeof(snaps[0])));
+  rados_snap_t rid;
+  EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+  EXPECT_EQ(rid, snaps[0]);
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+}
+
+TEST_F(LibRadosSnapshotsEC, SnapRemove) {
+  SKIP_IF_CRIMSON();
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  rados_snap_t rid;
+  ASSERT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+  ASSERT_EQ(-EEXIST, rados_ioctx_snap_create(ioctx, "snap1"));
+  ASSERT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+  ASSERT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snap1", &rid));
+}
+
+TEST_F(LibRadosSnapshotsEC, Rollback) {
+  SKIP_IF_CRIMSON();
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snap1"));
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  EXPECT_EQ(0, rados_write_full(ioctx, "foo", buf2, sizeof(buf2)));
+  EXPECT_EQ(0, rados_ioctx_snap_rollback(ioctx, "foo", "snap1"));
+  char buf3[sizeof(buf)];
+  EXPECT_EQ((int)sizeof(buf3), rados_read(ioctx, "foo", buf3, sizeof(buf3), 0));
+  EXPECT_EQ(0, memcmp(buf, buf3, sizeof(buf)));
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snap1"));
+}
+
+TEST_F(LibRadosSnapshotsEC, SnapGetName) {
+  SKIP_IF_CRIMSON();
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_ioctx_snap_create(ioctx, "snapfoo"));
+  rados_snap_t rid;
+  EXPECT_EQ(0, rados_ioctx_snap_lookup(ioctx, "snapfoo", &rid));
+  EXPECT_EQ(-ENOENT, rados_ioctx_snap_lookup(ioctx, "snapbar", &rid));
+  char name[128];
+  memset(name, 0, sizeof(name));
+  EXPECT_EQ(0, rados_ioctx_snap_get_name(ioctx, rid, name, sizeof(name)));
+  time_t snaptime;
+  EXPECT_EQ(0, rados_ioctx_snap_get_stamp(ioctx, rid, &snaptime));
+  EXPECT_EQ(0, strcmp(name, "snapfoo"));
+  EXPECT_EQ(0, rados_ioctx_snap_remove(ioctx, "snapfoo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedEC, Snap) {
+  SKIP_IF_CRIMSON();
+  std::vector<uint64_t> my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, bsize, 0));
+
+  my_snaps.push_back(-2);
+  rados_completion_t completion;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr,
+                                           &completion));
+  rados_aio_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back(), completion);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(completion));
+  rados_aio_release(completion);
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char *buf2 = (char *)new char[bsize];
+  memset(buf2, 0xdd, bsize);
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, bsize, bsize));
+  rados_ioctx_snap_set_read(ioctx, my_snaps[1]-1);
+  char *buf3 = (char *)new char[bsize*2];
+  ASSERT_EQ(-ENOENT, rados_read(ioctx, "foo", buf3, bsize*2, 0));
+
+  rados_ioctx_snap_set_read(ioctx, my_snaps[1]);
+  ASSERT_EQ(bsize, rados_read(ioctx, "foo", buf3, bsize*2, 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, bsize));
+
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr,
+                                           &completion));
+  rados_aio_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back(), completion);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(completion));
+  rados_aio_release(completion);
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+  delete[] buf;
+  delete[] buf2;
+  delete[] buf3;
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedEC, Rollback) {
+  SKIP_IF_CRIMSON();
+  std::vector<uint64_t> my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, bsize, 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+					&my_snaps[0], my_snaps.size()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char *buf2 = (char *)new char[bsize];
+  memset(buf2, 0xdd, bsize);
+
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, bsize, bsize));
+  rados_ioctx_selfmanaged_snap_rollback(ioctx, "foo", my_snaps[1]);
+  char *buf3 = (char *)new char[bsize*2];
+  ASSERT_EQ(bsize, rados_read(ioctx, "foo", buf3, bsize*2, 0));
+  ASSERT_EQ(0, memcmp(buf3, buf, bsize));
+
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, rados_remove(ioctx, "foo"));
+  delete[] buf;
+  delete[] buf2;
+  delete[] buf3;
+}
diff --git a/src/test/librados/snapshots_cxx.cc b/src/test/librados/snapshots_cxx.cc
new file mode 100644
index 000000000..8098b2cb7
--- /dev/null
+++ b/src/test/librados/snapshots_cxx.cc
@@ -0,0 +1,790 @@
+#include <algorithm>
+#include <errno.h>
+#include <string>
+
+#include "gtest/gtest.h"
+
+#include "include/rados.h"
+#include "include/rados/librados.hpp"
+#include "test/librados/test_cxx.h"
+#include "test/librados/testcase_cxx.h"
+#include "crimson_utils.h"
+
+using namespace librados;
+
+typedef RadosTestPP LibRadosSnapshotsPP;
+typedef RadosTestPP LibRadosSnapshotsSelfManagedPP;
+typedef RadosTestECPP LibRadosSnapshotsECPP;
+typedef RadosTestECPP LibRadosSnapshotsSelfManagedECPP;
+
+const int bufsize = 128;
+
+TEST_F(LibRadosSnapshotsPP, SnapListPP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_FALSE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  ASSERT_FALSE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  std::vector<snap_t> snaps;
+  EXPECT_EQ(0, ioctx.snap_list(&snaps));
+  EXPECT_EQ(1U, snaps.size());
+  snap_t rid;
+  EXPECT_EQ(0, ioctx.snap_lookup("snap1", &rid));
+  EXPECT_EQ(rid, snaps[0]);
+  EXPECT_EQ(0, ioctx.snap_remove("snap1"));
+  ASSERT_FALSE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+}
+
+TEST_F(LibRadosSnapshotsPP, SnapRemovePP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  rados_snap_t rid;
+  ASSERT_EQ(0, ioctx.snap_lookup("snap1", &rid));
+  ASSERT_EQ(0, ioctx.snap_remove("snap1"));
+  ASSERT_EQ(-ENOENT, ioctx.snap_lookup("snap1", &rid));
+}
+
+TEST_F(LibRadosSnapshotsPP, RollbackPP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  EXPECT_EQ(0, ioctx.write_full("foo", bl2));
+  EXPECT_EQ(0, ioctx.snap_rollback("foo", "snap1"));
+  bufferlist bl3;
+  EXPECT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  EXPECT_EQ(0, memcmp(buf, bl3.c_str(), sizeof(buf)));
+  EXPECT_EQ(0, ioctx.snap_remove("snap1"));
+}
+
+TEST_F(LibRadosSnapshotsPP, SnapGetNamePP) {
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snapfoo"));
+  rados_snap_t rid;
+  EXPECT_EQ(0, ioctx.snap_lookup("snapfoo", &rid));
+  EXPECT_EQ(-ENOENT, ioctx.snap_lookup("snapbar", &rid));
+  std::string name;
+  EXPECT_EQ(0, ioctx.snap_get_name(rid, &name));
+  time_t snaptime;
+  EXPECT_EQ(0, ioctx.snap_get_stamp(rid, &snaptime));
+  EXPECT_EQ(0, strcmp(name.c_str(), "snapfoo"));
+  EXPECT_EQ(0, ioctx.snap_remove("snapfoo"));
+}
+
+TEST_F(LibRadosSnapshotsPP, SnapCreateRemovePP) {
+  // reproduces http://tracker.ceph.com/issues/10262
+  bufferlist bl;
+  bl.append("foo");
+  ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snapfoo"));
+  ASSERT_EQ(0, ioctx.remove("foo"));
+  ASSERT_EQ(0, ioctx.snap_create("snapbar"));
+
+  std::unique_ptr<librados::ObjectWriteOperation> op(new librados::ObjectWriteOperation());
+  op->create(false);
+  op->remove();
+  ASSERT_EQ(0, ioctx.operate("foo", op.get()));
+
+  EXPECT_EQ(0, ioctx.snap_remove("snapfoo"));
+  EXPECT_EQ(0, ioctx.snap_remove("snapbar"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, SnapPP) {
+  std::vector<uint64_t> my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_FALSE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ASSERT_TRUE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ::std::reverse(my_snaps.begin(), my_snaps.end()); 
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  my_snaps.push_back(-2);
+  librados::AioCompletion *completion = cluster.aio_create_completion();
+  ioctx.aio_selfmanaged_snap_create(&my_snaps.back(), completion);
+  ASSERT_EQ(0, completion->wait_for_complete());
+  completion->release();
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), 0));
+
+  ioctx.snap_set_read(my_snaps[1]);
+  bufferlist bl3;
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+
+  completion = cluster.aio_create_completion();
+  ioctx.aio_selfmanaged_snap_remove(my_snaps.back(), completion);
+  ASSERT_EQ(0, completion->wait_for_complete());
+  completion->release();
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
+  ASSERT_TRUE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ASSERT_EQ(0, ioctx.remove("foo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, RollbackPP) {
+  SKIP_IF_CRIMSON();
+  std::vector<uint64_t> my_snaps;
+  IoCtx readioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), readioctx));
+  readioctx.set_namespace(nspace);
+  readioctx.snap_set_read(LIBRADOS_SNAP_DIR);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  //Write 3 consecutive buffers
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*2));
+
+  snap_set_t ss;
+
+  snap_t head = SNAP_HEAD;
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(1u, ss.clones.size());
+  ASSERT_EQ(head, ss.clones[0].cloneid);
+  ASSERT_EQ(0u, ss.clones[0].snaps.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap.size());
+  ASSERT_EQ(384u, ss.clones[0].size);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  //Change the middle buffer
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize));
+  //Add another after
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*3));
+
+  ASSERT_EQ(-EINVAL, ioctx.list_snaps("foo", &ss));
+  ObjectReadOperation o;
+  o.list_snaps(&ss, NULL);
+  ASSERT_EQ(-EINVAL, ioctx.operate("foo", &o, NULL));
+
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(2u, ss.clones.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  ASSERT_EQ(1u, ss.clones[0].snaps.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  ASSERT_EQ(2u, ss.clones[0].overlap.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap[0].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[0].second);
+  ASSERT_EQ(256u, ss.clones[0].overlap[1].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[1].second);
+  ASSERT_EQ(384u, ss.clones[0].size);
+  ASSERT_EQ(head, ss.clones[1].cloneid);
+  ASSERT_EQ(0u, ss.clones[1].snaps.size());
+  ASSERT_EQ(0u, ss.clones[1].overlap.size());
+  ASSERT_EQ(512u, ss.clones[1].size);
+
+  ioctx.selfmanaged_snap_rollback("foo", my_snaps[1]);
+
+  bufferlist bl3;
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), bufsize));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), bufsize*2));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, sizeof(buf)));
+  ASSERT_EQ((int)0, ioctx.read("foo", bl3, sizeof(buf), bufsize*3));
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  readioctx.close();
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, SnapOverlapPP) {
+  // WIP https://tracker.ceph.com/issues/58263
+  SKIP_IF_CRIMSON();
+  std::vector<uint64_t> my_snaps;
+  IoCtx readioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), readioctx));
+  readioctx.set_namespace(nspace);
+  readioctx.snap_set_read(LIBRADOS_SNAP_DIR);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*2));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*4));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*6));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), bufsize*8));
+
+  snap_set_t ss;
+  snap_t head = SNAP_HEAD;
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(1u, ss.clones.size());
+  ASSERT_EQ(head, ss.clones[0].cloneid);
+  ASSERT_EQ(0u, ss.clones[0].snaps.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap.size());
+  ASSERT_EQ(1152u, ss.clones[0].size);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*1));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*3));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*5));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*7));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize*9));
+
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(2u, ss.clones.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  ASSERT_EQ(1u, ss.clones[0].snaps.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  ASSERT_EQ(5u, ss.clones[0].overlap.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap[0].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[0].second);
+  ASSERT_EQ(256u, ss.clones[0].overlap[1].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[1].second);
+  ASSERT_EQ(512u, ss.clones[0].overlap[2].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[2].second);
+  ASSERT_EQ(768u, ss.clones[0].overlap[3].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[3].second);
+  ASSERT_EQ(1024u, ss.clones[0].overlap[4].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[4].second);
+  ASSERT_EQ(1152u, ss.clones[0].size);
+  ASSERT_EQ(head, ss.clones[1].cloneid);
+  ASSERT_EQ(0u, ss.clones[1].snaps.size());
+  ASSERT_EQ(0u, ss.clones[1].overlap.size());
+  ASSERT_EQ(1280u, ss.clones[1].size);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+
+  char buf3[sizeof(buf)];
+  memset(buf3, 0xee, sizeof(buf3));
+  bufferlist bl4;
+  bl4.append(buf3, sizeof(buf3));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf3), bufsize*1));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf3), bufsize*4));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf3), bufsize*5));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf3), bufsize*8));
+
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(3u, ss.clones.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  ASSERT_EQ(1u, ss.clones[0].snaps.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  ASSERT_EQ(5u, ss.clones[0].overlap.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap[0].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[0].second);
+  ASSERT_EQ(256u, ss.clones[0].overlap[1].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[1].second);
+  ASSERT_EQ(512u, ss.clones[0].overlap[2].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[2].second);
+  ASSERT_EQ(768u, ss.clones[0].overlap[3].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[3].second);
+  ASSERT_EQ(1024u, ss.clones[0].overlap[4].first);
+  ASSERT_EQ(128u, ss.clones[0].overlap[4].second);
+  ASSERT_EQ(1152u, ss.clones[0].size);
+
+  ASSERT_EQ(my_snaps[2], ss.clones[1].cloneid);
+  ASSERT_EQ(1u, ss.clones[1].snaps.size());
+  ASSERT_EQ(my_snaps[2], ss.clones[1].snaps[0]);
+  ASSERT_EQ(4u, ss.clones[1].overlap.size());
+  ASSERT_EQ(0u, ss.clones[1].overlap[0].first);
+  ASSERT_EQ(128u, ss.clones[1].overlap[0].second);
+  ASSERT_EQ(256u, ss.clones[1].overlap[1].first);
+  ASSERT_EQ(256u, ss.clones[1].overlap[1].second);
+  ASSERT_EQ(768u, ss.clones[1].overlap[2].first);
+  ASSERT_EQ(256u, ss.clones[1].overlap[2].second);
+  ASSERT_EQ(1152u, ss.clones[1].overlap[3].first);
+  ASSERT_EQ(128u, ss.clones[1].overlap[3].second);
+  ASSERT_EQ(1280u, ss.clones[1].size);
+
+  ASSERT_EQ(head, ss.clones[2].cloneid);
+  ASSERT_EQ(0u, ss.clones[2].snaps.size());
+  ASSERT_EQ(0u, ss.clones[2].overlap.size());
+  ASSERT_EQ(1280u, ss.clones[2].size);
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  readioctx.close();
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, Bug11677) {
+  std::vector<uint64_t> my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+
+  int bsize = 1<<20;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  bufferlist bl1;
+  bl1.append(buf, bsize);
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+
+  std::unique_ptr<librados::ObjectWriteOperation> op(new librados::ObjectWriteOperation());
+  op->assert_exists();
+  op->remove();
+  ASSERT_EQ(0, ioctx.operate("foo", op.get()));
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
+  delete[] buf;
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, OrderSnap) {
+  std::vector<uint64_t> my_snaps;
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  int flags = librados::OPERATION_ORDERSNAP;
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ObjectWriteOperation op1;
+  op1.write(0, bl);
+  librados::AioCompletion *comp1 = cluster.aio_create_completion();
+  ASSERT_EQ(0, ioctx.aio_operate("foo", comp1, &op1, flags));
+  ASSERT_EQ(0, comp1->wait_for_complete());
+  ASSERT_EQ(0, comp1->get_return_value());
+  comp1->release();
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ObjectWriteOperation op2;
+  op2.write(0, bl);
+  librados::AioCompletion *comp2 = cluster.aio_create_completion();
+  ASSERT_EQ(0, ioctx.aio_operate("foo", comp2, &op2, flags));
+  ASSERT_EQ(0, comp2->wait_for_complete());
+  ASSERT_EQ(0, comp2->get_return_value());
+  comp2->release();
+
+  my_snaps.pop_back();
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ObjectWriteOperation op3;
+  op3.write(0, bl);
+  librados::AioCompletion *comp3 = cluster.aio_create_completion();
+  ASSERT_EQ(0, ioctx.aio_operate("foo", comp3, &op3, flags));
+  ASSERT_EQ(0, comp3->wait_for_complete());
+  ASSERT_EQ(-EOLDSNAPC, comp3->get_return_value());
+  comp3->release();
+
+  ObjectWriteOperation op4;
+  op4.write(0, bl);
+  librados::AioCompletion *comp4 = cluster.aio_create_completion();
+  ASSERT_EQ(0, ioctx.aio_operate("foo", comp4, &op4, 0));
+  ASSERT_EQ(0, comp4->wait_for_complete());
+  ASSERT_EQ(0, comp4->get_return_value());
+  comp4->release();
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, WriteRollback) {
+  // https://tracker.ceph.com/issues/59114
+  GTEST_SKIP();
+  uint64_t snapid = 5;
+
+  // buf1
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+
+  // buf2
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+
+  // First write
+  ObjectWriteOperation op_write1;
+  op_write1.write(0, bl);
+  // Operate
+  librados::AioCompletion *comp_write = cluster.aio_create_completion();
+  ASSERT_EQ(0, ioctx.aio_operate("foo", comp_write, &op_write1, 0));
+  ASSERT_EQ(0, comp_write->wait_for_complete());
+  ASSERT_EQ(0, comp_write->get_return_value());
+  comp_write->release();
+
+  // Take Snapshot
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&snapid));
+
+  // Rollback + Second write in the same op
+  ObjectWriteOperation op_write2_snap_rollback;
+  op_write2_snap_rollback.write(0, bl2);
+  op_write2_snap_rollback.selfmanaged_snap_rollback(snapid);
+  // Operate
+  librados::AioCompletion *comp_write2 = cluster.aio_create_completion();
+  ASSERT_EQ(0, ioctx.aio_operate("foo", comp_write2, &op_write2_snap_rollback, 0));
+  ASSERT_EQ(0, comp_write2->wait_for_complete());
+  ASSERT_EQ(0, comp_write2->get_return_value());
+  comp_write2->release();
+
+  // Resolved should be first write
+  bufferlist bl3;
+  EXPECT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  EXPECT_EQ(0, memcmp(buf, bl3.c_str(), sizeof(buf)));
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedPP, ReusePurgedSnap) {
+  std::vector<uint64_t> my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ASSERT_TRUE(cluster.get_pool_is_selfmanaged_snaps_mode(pool_name));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  my_snaps.push_back(-2);
+  librados::AioCompletion *completion = cluster.aio_create_completion();
+  ioctx.aio_selfmanaged_snap_create(&my_snaps.back(), completion);
+  ASSERT_EQ(0, completion->wait_for_complete());
+  completion->release();
+
+  std::cout << "deleting snap " << my_snaps.back() << " in pool "
+	    << ioctx.get_pool_name() << std::endl;
+  completion = cluster.aio_create_completion();
+  ioctx.aio_selfmanaged_snap_remove(my_snaps.back(), completion);
+  ASSERT_EQ(0, completion->wait_for_complete());
+  completion->release();
+
+  std::cout << "waiting for snaps to purge" << std::endl;
+  sleep(15);
+
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), 0));
+
+  // scrub it out?
+  //sleep(600);
+}
+
+// EC testing
+TEST_F(LibRadosSnapshotsECPP, SnapListPP) {
+  SKIP_IF_CRIMSON();
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  std::vector<snap_t> snaps;
+  EXPECT_EQ(0, ioctx.snap_list(&snaps));
+  EXPECT_EQ(1U, snaps.size());
+  snap_t rid;
+  EXPECT_EQ(0, ioctx.snap_lookup("snap1", &rid));
+  EXPECT_EQ(rid, snaps[0]);
+  EXPECT_EQ(0, ioctx.snap_remove("snap1"));
+}
+
+TEST_F(LibRadosSnapshotsECPP, SnapRemovePP) {
+  SKIP_IF_CRIMSON();
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  rados_snap_t rid;
+  ASSERT_EQ(0, ioctx.snap_lookup("snap1", &rid));
+  ASSERT_EQ(0, ioctx.snap_remove("snap1"));
+  ASSERT_EQ(-ENOENT, ioctx.snap_lookup("snap1", &rid));
+}
+
+TEST_F(LibRadosSnapshotsECPP, RollbackPP) {
+  SKIP_IF_CRIMSON();
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snap1"));
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  EXPECT_EQ(0, ioctx.write_full("foo", bl2));
+  EXPECT_EQ(0, ioctx.snap_rollback("foo", "snap1"));
+  bufferlist bl3;
+  EXPECT_EQ((int)sizeof(buf), ioctx.read("foo", bl3, sizeof(buf), 0));
+  EXPECT_EQ(0, memcmp(buf, bl3.c_str(), sizeof(buf)));
+  EXPECT_EQ(0, ioctx.snap_remove("snap1"));
+}
+
+TEST_F(LibRadosSnapshotsECPP, SnapGetNamePP) {
+  SKIP_IF_CRIMSON();
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.snap_create("snapfoo"));
+  rados_snap_t rid;
+  EXPECT_EQ(0, ioctx.snap_lookup("snapfoo", &rid));
+  EXPECT_EQ(-ENOENT, ioctx.snap_lookup("snapbar", &rid));
+  std::string name;
+  EXPECT_EQ(0, ioctx.snap_get_name(rid, &name));
+  time_t snaptime;
+  EXPECT_EQ(0, ioctx.snap_get_stamp(rid, &snaptime));
+  EXPECT_EQ(0, strcmp(name.c_str(), "snapfoo"));
+  EXPECT_EQ(0, ioctx.snap_remove("snapfoo"));
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedECPP, SnapPP) {
+  SKIP_IF_CRIMSON();
+  std::vector<uint64_t> my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  bufferlist bl1;
+  bl1.append(buf, bsize);
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, 0));
+
+  my_snaps.push_back(-2);
+  librados::AioCompletion *completion = cluster.aio_create_completion();
+  ioctx.aio_selfmanaged_snap_create(&my_snaps.back(), completion);
+  ASSERT_EQ(0, completion->wait_for_complete());
+  completion->release();
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char *buf2 = (char *)new char[bsize];
+  memset(buf2, 0xdd, bsize);
+  bufferlist bl2;
+  bl2.append(buf2, bsize);
+  // Add another aligned buffer
+  ASSERT_EQ(0, ioctx.write("foo", bl2, bsize, bsize));
+
+  ioctx.snap_set_read(my_snaps[1]);
+  bufferlist bl3;
+  ASSERT_EQ(bsize, ioctx.read("foo", bl3, bsize*3, 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+
+  completion = cluster.aio_create_completion();
+  ioctx.aio_selfmanaged_snap_remove(my_snaps.back(), completion);
+  ASSERT_EQ(0, completion->wait_for_complete());
+  completion->release();
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
+  ASSERT_EQ(0, ioctx.remove("foo"));
+  delete[] buf;
+  delete[] buf2;
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedECPP, RollbackPP) {
+  SKIP_IF_CRIMSON();
+  std::vector<uint64_t> my_snaps;
+  IoCtx readioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), readioctx));
+  readioctx.set_namespace(nspace);
+  readioctx.snap_set_read(LIBRADOS_SNAP_DIR);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  bufferlist bl1;
+  bl1.append(buf, bsize);
+  //Write 3 consecutive buffers
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, 0));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, bsize));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, bsize*2));
+
+  snap_set_t ss;
+
+  snap_t head = SNAP_HEAD;
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(1u, ss.clones.size());
+  ASSERT_EQ(head, ss.clones[0].cloneid);
+  ASSERT_EQ(0u, ss.clones[0].snaps.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap.size());
+  ASSERT_EQ((unsigned)(bsize*3), ss.clones[0].size);
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  char *buf2 = (char *)new char[bsize];
+  memset(buf2, 0xdd, bsize);
+  bufferlist bl2;
+  bl2.append(buf2, bsize);
+  //Change the middle buffer
+  //ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), bufsize));
+  //Add another after
+  ASSERT_EQ(0, ioctx.write("foo", bl2, bsize, bsize*3));
+
+  ASSERT_EQ(-EINVAL, ioctx.list_snaps("foo", &ss));
+  ObjectReadOperation o;
+  o.list_snaps(&ss, NULL);
+  ASSERT_EQ(-EINVAL, ioctx.operate("foo", &o, NULL));
+
+  ASSERT_EQ(0, readioctx.list_snaps("foo", &ss));
+  ASSERT_EQ(2u, ss.clones.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].cloneid);
+  ASSERT_EQ(1u, ss.clones[0].snaps.size());
+  ASSERT_EQ(my_snaps[1], ss.clones[0].snaps[0]);
+  ASSERT_EQ(1u, ss.clones[0].overlap.size());
+  ASSERT_EQ(0u, ss.clones[0].overlap[0].first);
+  ASSERT_EQ((unsigned)bsize*3, ss.clones[0].overlap[0].second);
+  ASSERT_EQ((unsigned)bsize*3, ss.clones[0].size);
+  ASSERT_EQ(head, ss.clones[1].cloneid);
+  ASSERT_EQ(0u, ss.clones[1].snaps.size());
+  ASSERT_EQ(0u, ss.clones[1].overlap.size());
+  ASSERT_EQ((unsigned)bsize*4, ss.clones[1].size);
+
+  ioctx.selfmanaged_snap_rollback("foo", my_snaps[1]);
+
+  bufferlist bl3;
+  ASSERT_EQ(bsize, ioctx.read("foo", bl3, bsize, 0));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+  ASSERT_EQ(bsize, ioctx.read("foo", bl3, bsize, bsize));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+  ASSERT_EQ(bsize, ioctx.read("foo", bl3, bsize, bsize*2));
+  ASSERT_EQ(0, memcmp(bl3.c_str(), buf, bsize));
+  ASSERT_EQ(0, ioctx.read("foo", bl3, bsize, bsize*3));
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  readioctx.close();
+
+  delete[] buf;
+  delete[] buf2;
+}
+
+TEST_F(LibRadosSnapshotsSelfManagedECPP, Bug11677) {
+  SKIP_IF_CRIMSON();
+  std::vector<uint64_t> my_snaps;
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  bufferlist bl1;
+  bl1.append(buf, bsize);
+  ASSERT_EQ(0, ioctx.write("foo", bl1, bsize, 0));
+
+  my_snaps.push_back(-2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps.back()));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+  ::std::reverse(my_snaps.begin(), my_snaps.end());
+
+  std::unique_ptr<librados::ObjectWriteOperation> op(new librados::ObjectWriteOperation());
+  op->assert_exists();
+  op->remove();
+  ASSERT_EQ(0, ioctx.operate("foo", op.get()));
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps.back()));
+  my_snaps.pop_back();
+  ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
+  delete[] buf;
+}
diff --git a/src/test/librados/snapshots_stats.cc b/src/test/librados/snapshots_stats.cc
new file mode 100644
index 000000000..143299ab1
--- /dev/null
+++ b/src/test/librados/snapshots_stats.cc
@@ -0,0 +1,332 @@
+#include "include/rados.h"
+#include "json_spirit/json_spirit.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+#include <algorithm>
+#include <errno.h>
+#include "gtest/gtest.h"
+#include <string>
+#include <vector>
+
+using std::string;
+
+class LibRadosSnapshotStatsSelfManaged : public RadosTest {
+public:
+  LibRadosSnapshotStatsSelfManaged() {};
+  ~LibRadosSnapshotStatsSelfManaged() override {};
+protected:
+  void SetUp() override {
+    // disable pg autoscaler for the tests
+    string c =
+      "{"
+        "\"prefix\": \"config set\", "
+        "\"who\": \"global\", "
+        "\"name\": \"osd_pool_default_pg_autoscale_mode\", "
+        "\"value\": \"off\""
+      "}";
+    char *cmd[1];
+    cmd[0] = (char *)c.c_str();
+    std::cout << "Setting pg_autoscaler to 'off'" << std::endl;
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL,
+      0, NULL, 0));
+
+    // disable scrubs for the test
+    c = string("{\"prefix\": \"osd set\",\"key\":\"noscrub\"}");
+    cmd[0] = (char *)c.c_str();
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+    c = string("{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}");
+    cmd[0] = (char *)c.c_str();
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+
+    RadosTest::SetUp();
+  }
+
+  void TearDown() override {
+    // re-enable pg autoscaler
+    string c =
+      "{"
+        "\"prefix\": \"config set\", "
+        "\"who\": \"global\", "
+        "\"name\": \"osd_pool_default_pg_autoscale_mode\", "
+        "\"value\": \"on\""
+      "}";
+    char *cmd[1];
+    cmd[0] = (char *)c.c_str();
+    std::cout << "Setting pg_autoscaler to 'on'" << std::endl;
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL,
+      0, NULL, 0));
+
+    // re-enable scrubs
+    c = string("{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}");
+    cmd[0] = (char *)c.c_str();
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+    c = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}");
+    cmd[0] = (char *)c.c_str();
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+
+    RadosTest::TearDown();
+  }
+};
+
+class LibRadosSnapshotStatsSelfManagedEC : public RadosTestEC {
+public:
+  LibRadosSnapshotStatsSelfManagedEC() {};
+  ~LibRadosSnapshotStatsSelfManagedEC() override {};
+protected:
+  void SetUp() override {
+    // disable pg autoscaler for the tests
+    string c =
+      "{"
+        "\"prefix\": \"config set\", "
+        "\"who\": \"global\", "
+        "\"name\": \"osd_pool_default_pg_autoscale_mode\", "
+        "\"value\": \"off\""
+      "}";
+    char *cmd[1];
+    cmd[0] = (char *)c.c_str();
+    std::cout << "Setting pg_autoscaler to 'off'" << std::endl;
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL,
+      0, NULL, 0));
+
+    // disable scrubs for the test
+    c = string("{\"prefix\": \"osd set\",\"key\":\"noscrub\"}");
+    cmd[0] = (char *)c.c_str();
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+    c = string("{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}");
+    cmd[0] = (char *)c.c_str();
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+
+    RadosTestEC::SetUp();
+  }
+
+  void TearDown() override {
+    // re-enable pg autoscaler
+    string c =
+      "{"
+        "\"prefix\": \"config set\", "
+        "\"who\": \"global\", "
+        "\"name\": \"osd_pool_default_pg_autoscale_mode\", "
+        "\"value\": \"on\""
+      "}";
+    char *cmd[1];
+    cmd[0] = (char *)c.c_str();
+    std::cout << "Setting pg_autoscaler to 'on'" << std::endl;
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL,
+      0, NULL, 0));
+
+    // re-enable scrubs
+    c = string("{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}");
+    cmd[0] = (char *)c.c_str();
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+    c = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}");
+    cmd[0] = (char *)c.c_str();
+    ASSERT_EQ(0, rados_mon_command(s_cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0));
+
+    RadosTestEC::TearDown();
+  }
+};
+
+void get_snaptrim_stats(json_spirit::Object& pg_dump,
+                        int *objs_trimmed,
+                        double *trim_duration) {
+  // pg_map
+  json_spirit::Object pgmap;
+  for (json_spirit::Object::size_type i = 0; i < pg_dump.size(); ++i) {
+    json_spirit::Pair& p = pg_dump[i];
+    if (p.name_ == "pg_map") {
+      pgmap = p.value_.get_obj();
+      break;
+    }
+  }
+
+  // pg_stats array
+  json_spirit::Array pgs;
+  for (json_spirit::Object::size_type i = 0; i < pgmap.size(); ++i) {
+    json_spirit::Pair& p = pgmap[i];
+    if (p.name_ == "pg_stats") {
+      pgs = p.value_.get_array();
+      break;
+    }
+  }
+
+  // snaptrim stats
+  for (json_spirit::Object::size_type j = 0; j < pgs.size(); ++j) {
+    json_spirit::Object& pg_stat = pgs[j].get_obj();
+    for(json_spirit::Object::size_type k = 0; k < pg_stat.size(); ++k) {
+      json_spirit::Pair& stats = pg_stat[k];
+      if (stats.name_ == "objects_trimmed") {
+        *objs_trimmed += stats.value_.get_int();
+      }
+      if (stats.name_ == "snaptrim_duration") {
+        *trim_duration += stats.value_.get_real();
+      }
+    }
+  }
+}
+
+const int bufsize = 128;
+
+TEST_F(LibRadosSnapshotStatsSelfManaged, SnaptrimStats) {
+  int num_objs = 10;
+
+  // create objects
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  for (int i = 0; i < num_objs; ++i) {
+    string obj = string("foo") + std::to_string(i);
+    ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf, sizeof(buf), 0));
+  }
+
+  std::vector<uint64_t> my_snaps;
+  for (int snap = 0; snap < 1; ++snap) {
+    // create a snapshot, clone
+    std::vector<uint64_t> ns(1);
+    ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
+    my_snaps.swap(ns);
+    ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps[0]));
+    ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+                                        &my_snaps[0], my_snaps.size()));
+    char buf2[sizeof(buf)];
+    memset(buf2, 0xdd, sizeof(buf2));
+    for (int i = 0; i < num_objs; ++i) {
+      string obj = string("foo") + std::to_string(i);
+      ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf2, sizeof(buf2), 0));
+    }
+  }
+
+  // wait for maps to settle
+  ASSERT_EQ(0, rados_wait_for_latest_osdmap(cluster));
+
+  // remove snaps - should trigger snaptrim
+  rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
+  for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
+    ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps[snap]));
+  }
+
+  // sleep for few secs for the trim stats to populate
+  std::cout << "Waiting for snaptrim stats to be generated" << std::endl;
+  sleep(30);
+
+  // Dump pg stats and determine if snaptrim stats are getting set
+  int objects_trimmed = 0;
+  double snaptrim_duration = 0.0;
+  int tries = 0;
+  do {
+    char *buf, *st;
+    size_t buflen, stlen;
+    string c = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}");
+    const char *cmd = c.c_str();
+    ASSERT_EQ(0, rados_mon_command(cluster, (const char **)&cmd, 1, "", 0,
+      &buf, &buflen, &st, &stlen));
+    string outstr(buf, buflen);
+    json_spirit::Value v;
+    ASSERT_NE(0, json_spirit::read(outstr, v)) << "unable to parse json."
+      << '\n' << outstr;
+
+    // pg dump object
+    json_spirit::Object& obj = v.get_obj();
+    get_snaptrim_stats(obj, &objects_trimmed, &snaptrim_duration);
+    if (objects_trimmed < num_objs) {
+      tries++;
+      objects_trimmed = 0;
+      std::cout << "Still waiting for all objects to be trimmed... " <<std::endl;
+      sleep(30);
+    }
+  } while(objects_trimmed < num_objs && tries < 5);
+
+  // final check for objects trimmed
+  ASSERT_EQ(objects_trimmed, num_objs);
+  std::cout << "Snaptrim duration: " << snaptrim_duration << std::endl;
+  ASSERT_GT(snaptrim_duration, 0.0);
+
+  // clean-up remaining objects
+  for (int i = 0; i < num_objs; ++i) {
+    string obj = string("foo") + std::to_string(i);
+    ASSERT_EQ(0, rados_remove(ioctx, obj.c_str()));
+  }
+}
+
+// EC testing
+TEST_F(LibRadosSnapshotStatsSelfManagedEC, SnaptrimStats) {
+  int num_objs = 10;
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  // create objects
+  for (int i = 0; i < num_objs; ++i) {
+    string obj = string("foo") + std::to_string(i);
+    ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf, bsize, 0));
+  }
+
+  std::vector<uint64_t> my_snaps;
+  for (int snap = 0; snap < 1; ++snap) {
+    // create a snapshot, clone
+    std::vector<uint64_t> ns(1);
+    ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
+    my_snaps.swap(ns);
+    ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_create(ioctx, &my_snaps[0]));
+    ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_set_write_ctx(ioctx, my_snaps[0],
+                                        &my_snaps[0], my_snaps.size()));
+    char *buf2 = (char *)new char[bsize];
+    memset(buf2, 0xdd, bsize);
+    for (int i = 0; i < num_objs; ++i) {
+      string obj = string("foo") + std::to_string(i);
+      ASSERT_EQ(0, rados_write(ioctx, obj.c_str(), buf2, bsize, bsize));
+    }
+    delete[] buf2;
+  }
+
+  // wait for maps to settle
+  ASSERT_EQ(0, rados_wait_for_latest_osdmap(cluster));
+
+  // remove snaps - should trigger snaptrim
+  rados_ioctx_snap_set_read(ioctx, LIBRADOS_SNAP_HEAD);
+  for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
+    ASSERT_EQ(0, rados_ioctx_selfmanaged_snap_remove(ioctx, my_snaps[snap]));
+  }
+
+  // sleep for few secs for the trim stats to populate
+  std::cout << "Waiting for snaptrim stats to be generated" << std::endl;
+  sleep(30);
+
+  // Dump pg stats and determine if snaptrim stats are getting set
+  int objects_trimmed = 0;
+  double snaptrim_duration = 0.0;
+  int tries = 0;
+  do {
+    char *buf, *st;
+    size_t buflen, stlen;
+    string c = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}");
+    const char *cmd = c.c_str();
+    ASSERT_EQ(0, rados_mon_command(cluster, (const char **)&cmd, 1, 0, 0,
+      &buf, &buflen, &st, &stlen));
+    string outstr(buf, buflen);
+    json_spirit::Value v;
+    ASSERT_NE(0, json_spirit::read(outstr, v)) << "Unable tp parse json."
+      << '\n' << outstr;
+
+    // pg dump object
+    json_spirit::Object& obj = v.get_obj();
+    get_snaptrim_stats(obj, &objects_trimmed, &snaptrim_duration);
+    if (objects_trimmed != num_objs) {
+      tries++;
+      objects_trimmed = 0;
+      std::cout << "Still waiting for all objects to be trimmed... " <<std::endl;
+      sleep(30);
+    }
+  } while (objects_trimmed != num_objs && tries < 5);
+
+  // final check for objects trimmed
+  ASSERT_EQ(objects_trimmed, num_objs);
+  std::cout << "Snaptrim duration: " << snaptrim_duration << std::endl;
+  ASSERT_GT(snaptrim_duration, 0.0);
+
+  // clean-up remaining objects
+  for (int i = 0; i < num_objs; ++i) {
+    string obj = string("foo") + std::to_string(i);
+    ASSERT_EQ(0, rados_remove(ioctx, obj.c_str()));
+  }
+
+  delete[] buf;
+}
diff --git a/src/test/librados/snapshots_stats_cxx.cc b/src/test/librados/snapshots_stats_cxx.cc
new file mode 100644
index 000000000..f6be3a915
--- /dev/null
+++ b/src/test/librados/snapshots_stats_cxx.cc
@@ -0,0 +1,324 @@
+#include <algorithm>
+#include <errno.h>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "include/rados.h"
+#include "include/rados/librados.hpp"
+#include "json_spirit/json_spirit.h"
+#include "test/librados/test_cxx.h"
+#include "test/librados/testcase_cxx.h"
+
+using namespace librados;
+
+using std::string;
+
+class LibRadosSnapshotStatsSelfManagedPP : public RadosTestPP {
+public:
+  LibRadosSnapshotStatsSelfManagedPP() {};
+  ~LibRadosSnapshotStatsSelfManagedPP() override {};
+protected:
+  void SetUp() override {
+    // disable pg autoscaler for the tests
+    string cmd =
+      "{"
+        "\"prefix\": \"config set\", "
+        "\"who\": \"global\", "
+        "\"name\": \"osd_pool_default_pg_autoscale_mode\", "
+        "\"value\": \"off\""
+      "}";
+    std::cout << "Setting pg_autoscaler to 'off'" << std::endl;
+    bufferlist inbl;
+    bufferlist outbl;
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+
+    // disable scrubs for the test
+    cmd = "{\"prefix\": \"osd set\",\"key\":\"noscrub\"}";
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+    cmd = "{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}";
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+
+    RadosTestPP::SetUp();
+  }
+
+  void TearDown() override {
+    // re-enable pg autoscaler
+    string cmd =
+      "{"
+        "\"prefix\": \"config set\", "
+        "\"who\": \"global\", "
+        "\"name\": \"osd_pool_default_pg_autoscale_mode\", "
+        "\"value\": \"on\""
+      "}";
+    std::cout << "Setting pg_autoscaler to 'on'" << std::endl;
+    bufferlist inbl;
+    bufferlist outbl;
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+
+    // re-enable scrubs
+    cmd = "{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}";
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+    cmd = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}");
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+
+    RadosTestPP::TearDown();
+  }
+};
+
+class LibRadosSnapshotStatsSelfManagedECPP : public RadosTestECPP {
+public:
+  LibRadosSnapshotStatsSelfManagedECPP() {};
+  ~LibRadosSnapshotStatsSelfManagedECPP() override {};
+protected:
+  void SetUp() override {
+    // disable pg autoscaler for the tests
+    string cmd =
+      "{"
+        "\"prefix\": \"config set\", "
+        "\"who\": \"global\", "
+        "\"name\": \"osd_pool_default_pg_autoscale_mode\", "
+        "\"value\": \"off\""
+      "}";
+    std::cout << "Setting pg_autoscaler to 'off'" << std::endl;
+    bufferlist inbl;
+    bufferlist outbl;
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+
+    // disable scrubs for the test
+    cmd = string("{\"prefix\": \"osd set\",\"key\":\"noscrub\"}");
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+    cmd = string("{\"prefix\": \"osd set\",\"key\":\"nodeep-scrub\"}");
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+
+    RadosTestECPP::SetUp();
+  }
+
+  void TearDown() override {
+    // re-enable pg autoscaler
+    string cmd =
+      "{"
+        "\"prefix\": \"config set\", "
+        "\"who\": \"global\", "
+        "\"name\": \"osd_pool_default_pg_autoscale_mode\", "
+        "\"value\": \"on\""
+      "}";
+    std::cout << "Setting pg_autoscaler to 'on'" << std::endl;
+    bufferlist inbl;
+    bufferlist outbl;
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+
+    // re-enable scrubs
+    cmd = string("{\"prefix\": \"osd unset\",\"key\":\"noscrub\"}");
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+    cmd = string("{\"prefix\": \"osd unset\",\"key\":\"nodeep-scrub\"}");
+    ASSERT_EQ(0, s_cluster.mon_command(cmd, inbl, &outbl, NULL));
+
+    RadosTestECPP::TearDown();
+  }
+};
+
+void get_snaptrim_stats(json_spirit::Object& pg_dump,
+                        int *objs_trimmed,
+                        double *trim_duration) {
+  // pg_map
+  json_spirit::Object pgmap;
+  for (json_spirit::Object::size_type i = 0; i < pg_dump.size(); ++i) {
+    json_spirit::Pair& p = pg_dump[i];
+    if (p.name_ == "pg_map") {
+      pgmap = p.value_.get_obj();
+      break;
+    }
+  }
+
+  // pg_stats array
+  json_spirit::Array pgs;
+  for (json_spirit::Object::size_type i = 0; i < pgmap.size(); ++i) {
+    json_spirit::Pair& p = pgmap[i];
+    if (p.name_ == "pg_stats") {
+      pgs = p.value_.get_array();
+      break;
+    }
+  }
+
+  // snaptrim stats
+  for (json_spirit::Object::size_type j = 0; j < pgs.size(); ++j) {
+    json_spirit::Object& pg_stat = pgs[j].get_obj();
+    for(json_spirit::Object::size_type k = 0; k < pg_stat.size(); ++k) {
+      json_spirit::Pair& stats = pg_stat[k];
+      if (stats.name_ == "objects_trimmed") {
+        *objs_trimmed += stats.value_.get_int();
+      }
+      if (stats.name_ == "snaptrim_duration") {
+        *trim_duration += stats.value_.get_real();
+      }
+    }
+  }
+}
+const int bufsize = 128;
+
+TEST_F(LibRadosSnapshotStatsSelfManagedPP, SnaptrimStatsPP) {
+  int num_objs = 10;
+
+  // create objects
+  char buf[bufsize];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  for (int i = 0; i < num_objs; ++i) {
+   string obj = string("foo") + std::to_string(i);
+    ASSERT_EQ(0, ioctx.write(obj, bl, sizeof(buf), 0));
+  }
+
+  std::vector<uint64_t> my_snaps;
+  char buf2[sizeof(buf)];
+  memset(buf2, 0xdd, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  for (int snap = 0; snap < 1; ++snap) {
+    // create a snapshot, clone
+    std::vector<uint64_t> ns(1);
+    ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
+    my_snaps.swap(ns);
+    ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+    ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+    for (int i = 0; i < num_objs; ++i) {
+      string obj = string("foo") + std::to_string(i);
+      ASSERT_EQ(0, ioctx.write(obj, bl2, sizeof(buf2), 0));
+    }
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // remove snaps - should trigger snaptrim
+  for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
+    ioctx.selfmanaged_snap_remove(my_snaps[snap]);
+  }
+
+  // sleep for few secs for the trim stats to populate
+  std::cout << "Waiting for snaptrim stats to be generated" << std::endl;
+  sleep(30);
+
+  // Dump pg stats and determine if snaptrim stats are getting set
+  int objects_trimmed = 0;
+  double snaptrim_duration = 0.0;
+  int tries = 0;
+  do {
+    string cmd = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}");
+    bufferlist inbl;
+    bufferlist outbl;
+    ASSERT_EQ(0, cluster.mon_command(cmd, inbl, &outbl, NULL));
+    string outstr(outbl.c_str(), outbl.length());
+    json_spirit::Value v;
+    ASSERT_NE(0, json_spirit::read(outstr, v)) << "unable to parse json." << '\n' << outstr;
+
+    // pg_map
+    json_spirit::Object& obj = v.get_obj();
+    get_snaptrim_stats(obj, &objects_trimmed, &snaptrim_duration);
+    if (objects_trimmed < num_objs) {
+      tries++;
+      objects_trimmed = 0;
+      std::cout << "Still waiting for all objects to be trimmed... " <<std::endl;
+      sleep(30);
+    }
+  } while(objects_trimmed < num_objs && tries < 5);
+
+  // final check for objects trimmed
+  ASSERT_EQ(objects_trimmed, num_objs);
+  std::cout << "Snaptrim duration: " << snaptrim_duration << std::endl;
+  ASSERT_GT(snaptrim_duration, 0.0);
+
+  // clean-up remaining objects
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  for (int i = 0; i < num_objs; ++i) {
+    string obj = string("foo") + std::to_string(i);
+    ASSERT_EQ(0, ioctx.remove(obj));
+  }
+}
+
+// EC testing
+TEST_F(LibRadosSnapshotStatsSelfManagedECPP, SnaptrimStatsECPP) {
+  int num_objs = 10;
+  int bsize = alignment;
+
+  // create objects
+  char *buf = (char *)new char[bsize];
+  memset(buf, 0xcc, bsize);
+  bufferlist bl;
+  bl.append(buf, bsize);
+  for (int i = 0; i < num_objs; ++i) {
+   string obj = string("foo") + std::to_string(i);
+    ASSERT_EQ(0, ioctx.write(obj, bl, bsize, 0));
+  }
+
+  std::vector<uint64_t> my_snaps;
+  char *buf2 = (char *)new char[bsize];
+  memset(buf2, 0xdd, bsize);
+  bufferlist bl2;
+  bl2.append(buf2, bsize);
+  for (int snap = 0; snap < 1; ++snap) {
+    // create a snapshot, clone
+    std::vector<uint64_t> ns(1);
+    ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
+    my_snaps.swap(ns);
+    ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+    ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0], my_snaps));
+    for (int i = 0; i < num_objs; ++i) {
+      string obj = string("foo") + std::to_string(i);
+      ASSERT_EQ(0, ioctx.write(obj, bl2, bsize, bsize));
+    }
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // remove snaps - should trigger snaptrim
+  for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
+    ioctx.selfmanaged_snap_remove(my_snaps[snap]);
+  }
+
+  // sleep for few secs for the trim stats to populate
+  std::cout << "Waiting for snaptrim stats to be generated" << std::endl;
+  sleep(30);
+
+  // Dump pg stats and determine if snaptrim stats are getting set
+  int objects_trimmed = 0;
+  double snaptrim_duration = 0.0;
+  int tries = 0;
+  do {
+    string cmd = string("{\"prefix\": \"pg dump\",\"format\":\"json\"}");
+    bufferlist inbl;
+    bufferlist outbl;
+    ASSERT_EQ(0, cluster.mon_command(cmd, inbl, &outbl, NULL));
+    string outstr(outbl.c_str(), outbl.length());
+    json_spirit::Value v;
+    ASSERT_NE(0, json_spirit::read(outstr, v)) << "unable to parse json." << '\n' << outstr;
+
+    // pg_map
+    json_spirit::Object& obj = v.get_obj();
+    get_snaptrim_stats(obj, &objects_trimmed, &snaptrim_duration);
+    if (objects_trimmed < num_objs) {
+      tries++;
+      objects_trimmed = 0;
+      std::cout << "Still waiting for all objects to be trimmed... " <<std::endl;
+      sleep(30);
+    }
+  } while(objects_trimmed < num_objs && tries < 5);
+
+  // final check for objects trimmed
+  ASSERT_EQ(objects_trimmed, num_objs);
+  std::cout << "Snaptrim duration: " << snaptrim_duration << std::endl;
+  ASSERT_GT(snaptrim_duration, 0.0);
+
+  // clean-up remaining objects
+  ioctx.snap_set_read(LIBRADOS_SNAP_HEAD);
+  for (int i = 0; i < num_objs; ++i) {
+    string obj = string("foo") + std::to_string(i);
+    ASSERT_EQ(0, ioctx.remove(obj));
+  }
+
+  delete[] buf;
+  delete[] buf2;
+}
diff --git a/src/test/librados/stat.cc b/src/test/librados/stat.cc
new file mode 100644
index 000000000..6ae14395e
--- /dev/null
+++ b/src/test/librados/stat.cc
@@ -0,0 +1,153 @@
+#include "include/rados/librados.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+
+#include "common/ceph_time.h"
+
+#include <algorithm>
+#include <errno.h>
+#include "gtest/gtest.h"
+#include "crimson_utils.h"
+
+typedef RadosTest LibRadosStat;
+typedef RadosTestEC LibRadosStatEC;
+
+TEST_F(LibRadosStat, Stat) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  uint64_t size = 0;
+  time_t mtime = 0;
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+}
+
+TEST_F(LibRadosStat, Stat2) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  rados_write_op_t op = rados_create_write_op();
+  rados_write_op_write(op, buf, sizeof(buf), 0);
+  struct timespec ts;
+  ts.tv_sec = 1457129052;
+  ts.tv_nsec = 123456789;
+  ASSERT_EQ(0, rados_write_op_operate2(op, ioctx, "foo", &ts, 0));
+  rados_release_write_op(op);
+
+  uint64_t size = 0;
+  time_t mtime = 0;
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(mtime, ts.tv_sec);
+
+  struct timespec ts2 = {};
+  ASSERT_EQ(0, rados_stat2(ioctx, "foo", &size, &ts2));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(ts2.tv_sec, ts.tv_sec);
+  ASSERT_EQ(ts2.tv_nsec, ts.tv_nsec);
+
+  ASSERT_EQ(-ENOENT, rados_stat2(ioctx, "nonexistent", &size, &ts2));
+}
+
+TEST_F(LibRadosStat, StatNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo2", buf, sizeof(buf), 0));
+
+  char buf2[64];
+  memset(buf2, 0xcc, sizeof(buf2));
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+
+  uint64_t size = 0;
+  time_t mtime = 0;
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf2), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "foo2", &size, &mtime));
+}
+
+TEST_F(LibRadosStat, ClusterStat) {
+  struct rados_cluster_stat_t result;
+  ASSERT_EQ(0, rados_cluster_stat(cluster, &result));
+}
+
+TEST_F(LibRadosStat, PoolStat) {
+  char buf[128];
+  char actual_pool_name[80];
+  unsigned l = rados_ioctx_get_pool_name(ioctx, actual_pool_name, sizeof(actual_pool_name));
+  ASSERT_EQ(strlen(actual_pool_name), l);
+  ASSERT_EQ(0, strcmp(actual_pool_name, pool_name.c_str()));
+  memset(buf, 0xff, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  struct rados_pool_stat_t stats;
+  memset(&stats, 0, sizeof(stats));
+  ASSERT_EQ(0, rados_ioctx_pool_stat(ioctx, &stats));
+}
+
+TEST_F(LibRadosStatEC, Stat) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  uint64_t size = 0;
+  time_t mtime = 0;
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+}
+
+TEST_F(LibRadosStatEC, StatNS) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  ASSERT_EQ(0, rados_write(ioctx, "foo2", buf, sizeof(buf), 0));
+
+  char buf2[64];
+  memset(buf2, 0xcc, sizeof(buf2));
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf2, sizeof(buf2), 0));
+
+  uint64_t size = 0;
+  time_t mtime = 0;
+  rados_ioctx_set_namespace(ioctx, "");
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+
+  rados_ioctx_set_namespace(ioctx, "nspace");
+  ASSERT_EQ(0, rados_stat(ioctx, "foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf2), size);
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "nonexistent", &size, &mtime));
+  ASSERT_EQ(-ENOENT, rados_stat(ioctx, "foo2", &size, &mtime));
+}
+
+TEST_F(LibRadosStatEC, ClusterStat) {
+  SKIP_IF_CRIMSON();
+  struct rados_cluster_stat_t result;
+  ASSERT_EQ(0, rados_cluster_stat(cluster, &result));
+}
+
+TEST_F(LibRadosStatEC, PoolStat) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  char actual_pool_name[80];
+  unsigned l = rados_ioctx_get_pool_name(ioctx, actual_pool_name, sizeof(actual_pool_name));
+  ASSERT_EQ(strlen(actual_pool_name), l);
+  ASSERT_EQ(0, strcmp(actual_pool_name, pool_name.c_str()));
+  memset(buf, 0xff, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  struct rados_pool_stat_t stats;
+  memset(&stats, 0, sizeof(stats));
+  ASSERT_EQ(0, rados_ioctx_pool_stat(ioctx, &stats));
+}
diff --git a/src/test/librados/stat_cxx.cc b/src/test/librados/stat_cxx.cc
new file mode 100644
index 000000000..07b480ad0
--- /dev/null
+++ b/src/test/librados/stat_cxx.cc
@@ -0,0 +1,168 @@
+#include "gtest/gtest.h"
+
+#include "include/rados/librados.hpp"
+
+#include "test/librados/test_cxx.h"
+#include "test/librados/testcase_cxx.h"
+#include "crimson_utils.h"
+
+using namespace librados;
+
+typedef RadosTestPP LibRadosStatPP;
+typedef RadosTestECPP LibRadosStatECPP;
+
+TEST_F(LibRadosStatPP, StatPP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+}
+
+TEST_F(LibRadosStatPP, Stat2Mtime2PP) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  librados::ObjectWriteOperation op;
+  struct timespec ts;
+  ts.tv_sec = 1457129052;
+  ts.tv_nsec = 123456789;
+  op.mtime2(&ts);
+  op.write(0, bl);
+  ASSERT_EQ(0, ioctx.operate("foo", &op));
+
+  /* XXX time comparison asserts could spuriously fail */
+
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(mtime, ts.tv_sec);
+
+  struct timespec ts2;
+  ASSERT_EQ(0, ioctx.stat2("foo", &size, &ts2));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(ts2.tv_sec, ts.tv_sec);
+  ASSERT_EQ(ts2.tv_nsec, ts.tv_nsec);
+
+  ASSERT_EQ(-ENOENT, ioctx.stat2("nonexistent", &size, &ts2));
+}
+
+TEST_F(LibRadosStatPP, ClusterStatPP) {
+  cluster_stat_t cstat;
+  ASSERT_EQ(0, cluster.cluster_stat(cstat));
+}
+
+TEST_F(LibRadosStatPP, PoolStatPP) {
+  std::string n = ioctx.get_pool_name();
+  ASSERT_EQ(n, pool_name);
+  char buf[128];
+  memset(buf, 0xff, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  std::list<std::string> v;
+  std::map<std::string,stats_map> stats;
+  ASSERT_EQ(0, cluster.get_pool_stats(v, stats));
+}
+
+TEST_F(LibRadosStatECPP, StatPP) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  uint64_t size;
+  time_t mtime;
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+}
+
+TEST_F(LibRadosStatECPP, ClusterStatPP) {
+  SKIP_IF_CRIMSON();
+  cluster_stat_t cstat;
+  ASSERT_EQ(0, cluster.cluster_stat(cstat));
+}
+
+TEST_F(LibRadosStatECPP, PoolStatPP) {
+  SKIP_IF_CRIMSON();
+  std::string n = ioctx.get_pool_name();
+  ASSERT_EQ(n, pool_name);
+  char buf[128];
+  memset(buf, 0xff, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  std::list<std::string> v;
+  std::map<std::string,stats_map> stats;
+  ASSERT_EQ(0, cluster.get_pool_stats(v, stats));
+}
+
+TEST_F(LibRadosStatPP, StatPPNS) {
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo2", bl, sizeof(buf), 0));
+
+  char buf2[64];
+  memset(buf2, 0xbb, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), 0));
+
+  uint64_t size;
+  time_t mtime;
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf2), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+  ASSERT_EQ(-ENOENT, ioctx.stat("foo2", &size, &mtime));
+}
+
+TEST_F(LibRadosStatECPP, StatPPNS) {
+  SKIP_IF_CRIMSON();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl;
+  bl.append(buf, sizeof(buf));
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.write("foo", bl, sizeof(buf), 0));
+  ASSERT_EQ(0, ioctx.write("foo2", bl, sizeof(buf), 0));
+
+  char buf2[64];
+  memset(buf2, 0xbb, sizeof(buf2));
+  bufferlist bl2;
+  bl2.append(buf2, sizeof(buf2));
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.write("foo", bl2, sizeof(buf2), 0));
+
+  uint64_t size;
+  time_t mtime;
+  ioctx.set_namespace("");
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+
+  ioctx.set_namespace("nspace");
+  ASSERT_EQ(0, ioctx.stat("foo", &size, &mtime));
+  ASSERT_EQ(sizeof(buf2), size);
+  ASSERT_EQ(-ENOENT, ioctx.stat("nonexistent", &size, &mtime));
+  ASSERT_EQ(-ENOENT, ioctx.stat("foo2", &size, &mtime));
+}
diff --git a/src/test/librados/test.cc b/src/test/librados/test.cc
new file mode 100644
index 000000000..be1b4faf8
--- /dev/null
+++ b/src/test/librados/test.cc
@@ -0,0 +1,198 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
+#include "include/rados/librados.h"
+#include "include/rados/librados.hpp"
+#include "test/librados/test.h"
+
+#include "include/stringify.h"
+#include "common/ceph_context.h"
+#include "common/config.h"
+
+#include <errno.h>
+#include <sstream>
+#include <stdlib.h>
+#include <string>
+#include <time.h>
+#include <unistd.h>
+#include <iostream>
+#include "gtest/gtest.h"
+
+std::string create_one_pool(
+    const std::string &pool_name, rados_t *cluster, uint32_t pg_num)
+{
+  std::string err_str = connect_cluster(cluster);
+  if (err_str.length())
+    return err_str;
+
+  int ret = rados_pool_create(*cluster, pool_name.c_str());
+  if (ret) {
+    rados_shutdown(*cluster);
+    std::ostringstream oss;
+    oss << "create_one_pool(" << pool_name << ") failed with error " << ret;
+    return oss.str();
+  }
+
+  rados_ioctx_t ioctx;
+  ret = rados_ioctx_create(*cluster, pool_name.c_str(), &ioctx);
+  if (ret < 0) {
+    rados_shutdown(*cluster);
+    std::ostringstream oss;
+    oss << "rados_ioctx_create(" << pool_name << ") failed with error " << ret;
+    return oss.str();
+  }
+
+  rados_application_enable(ioctx, "rados", 1);
+  rados_ioctx_destroy(ioctx);
+  return "";
+}
+
+int destroy_ec_profile(rados_t *cluster,
+		       const std::string& pool_name,
+		       std::ostream &oss)
+{
+  char buf[1000];
+  snprintf(buf, sizeof(buf),
+	   "{\"prefix\": \"osd erasure-code-profile rm\", \"name\": \"testprofile-%s\"}",
+	   pool_name.c_str());
+  char *cmd[2];
+  cmd[0] = buf;
+  cmd[1] = NULL;
+  int ret = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0, NULL,
+			      0, NULL, 0);
+  if (ret)
+    oss << "rados_mon_command: erasure-code-profile rm testprofile-"
+	<< pool_name << " failed with error " << ret;
+  return ret;
+}
+
+int destroy_rule(rados_t *cluster,
+		 const std::string &rule,
+		 std::ostream &oss)
+{
+  char *cmd[2];
+  std::string tmp = ("{\"prefix\": \"osd crush rule rm\", \"name\":\"" +
+                     rule + "\"}");
+  cmd[0] = (char*)tmp.c_str();
+  cmd[1] = NULL;
+  int ret = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0);
+  if (ret)
+    oss << "rados_mon_command: osd crush rule rm " + rule + " failed with error " << ret;
+  return ret;
+}
+
+int destroy_ec_profile_and_rule(rados_t *cluster,
+				const std::string &rule,
+				std::ostream &oss)
+{
+  int ret;
+  ret = destroy_ec_profile(cluster, rule, oss);
+  if (ret)
+    return ret;
+  return destroy_rule(cluster, rule, oss);
+}
+
+
+std::string create_one_ec_pool(const std::string &pool_name, rados_t *cluster)
+{
+  std::string err = connect_cluster(cluster);
+  if (err.length())
+    return err;
+
+  std::ostringstream oss;
+  int ret = destroy_ec_profile_and_rule(cluster, pool_name, oss);
+  if (ret) {
+    rados_shutdown(*cluster);
+    return oss.str();
+  }
+    
+  char *cmd[2];
+  cmd[1] = NULL;
+
+  std::string profile_create = "{\"prefix\": \"osd erasure-code-profile set\", \"name\": \"testprofile-" + pool_name + "\", \"profile\": [ \"k=2\", \"m=1\", \"crush-failure-domain=osd\"]}";
+  cmd[0] = (char *)profile_create.c_str();
+  ret = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0);
+  if (ret) {
+    rados_shutdown(*cluster);
+    oss << "rados_mon_command erasure-code-profile set name:testprofile-" << pool_name << " failed with error " << ret;
+    return oss.str();
+  }
+
+  std::string cmdstr = "{\"prefix\": \"osd pool create\", \"pool\": \"" +
+     pool_name + "\", \"pool_type\":\"erasure\", \"pg_num\":8, \"pgp_num\":8, \"erasure_code_profile\":\"testprofile-" + pool_name + "\"}";
+  cmd[0] = (char *)cmdstr.c_str();
+  ret = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0, NULL, 0, NULL, 0);
+  if (ret) {
+    destroy_ec_profile(cluster, pool_name, oss);
+    rados_shutdown(*cluster);
+    oss << "rados_mon_command osd pool create failed with error " << ret;
+    return oss.str();
+  }
+
+  rados_wait_for_latest_osdmap(*cluster);
+  return "";
+}
+
+std::string connect_cluster(rados_t *cluster)
+{
+  char *id = getenv("CEPH_CLIENT_ID");
+  if (id) std::cerr << "Client id is: " << id << std::endl;
+
+  int ret;
+  ret = rados_create(cluster, NULL);
+  if (ret) {
+    std::ostringstream oss;
+    oss << "rados_create failed with error " << ret;
+    return oss.str();
+  }
+  ret = rados_conf_read_file(*cluster, NULL);
+  if (ret) {
+    rados_shutdown(*cluster);
+    std::ostringstream oss;
+    oss << "rados_conf_read_file failed with error " << ret;
+    return oss.str();
+  }
+  rados_conf_parse_env(*cluster, NULL);
+  ret = rados_connect(*cluster);
+  if (ret) {
+    rados_shutdown(*cluster);
+    std::ostringstream oss;
+    oss << "rados_connect failed with error " << ret;
+    return oss.str();
+  }
+  return "";
+}
+
+int destroy_one_pool(const std::string &pool_name, rados_t *cluster)
+{
+  int ret = rados_pool_delete(*cluster, pool_name.c_str());
+  if (ret) {
+    rados_shutdown(*cluster);
+    return ret;
+  }
+  rados_shutdown(*cluster);
+  return 0;
+}
+
+int destroy_one_ec_pool(const std::string &pool_name, rados_t *cluster)
+{
+  int ret = rados_pool_delete(*cluster, pool_name.c_str());
+  if (ret) {
+    rados_shutdown(*cluster);
+    return ret;
+  }
+
+  CephContext *cct = static_cast<CephContext*>(rados_cct(*cluster));
+  if (!cct->_conf->mon_fake_pool_delete) { // hope this is in [global]
+    std::ostringstream oss;
+    ret = destroy_ec_profile_and_rule(cluster, pool_name, oss);
+    if (ret) {
+      rados_shutdown(*cluster);
+      return ret;
+    }
+  }
+
+  rados_wait_for_latest_osdmap(*cluster);
+  rados_shutdown(*cluster);
+  return ret;
+}
diff --git a/src/test/librados/test.h b/src/test/librados/test.h
new file mode 100644
index 000000000..b3e0115fb
--- /dev/null
+++ b/src/test/librados/test.h
@@ -0,0 +1,32 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_TEST_RADOS_API_TEST_H
+#define CEPH_TEST_RADOS_API_TEST_H
+
+#include "include/rados/librados.h"
+#include "test/librados/test_shared.h"
+
+#include <map>
+#include <string>
+#include <unistd.h>
+
+std::string create_one_pool(const std::string &pool_name, rados_t *cluster,
+    uint32_t pg_num=0);
+std::string create_one_ec_pool(const std::string &pool_name, rados_t *cluster);
+std::string connect_cluster(rados_t *cluster);
+int destroy_one_pool(const std::string &pool_name, rados_t *cluster);
+int destroy_one_ec_pool(const std::string &pool_name, rados_t *cluster);
+
+#endif
diff --git a/src/test/librados/test_common.cc b/src/test/librados/test_common.cc
new file mode 100644
index 000000000..647a9ff48
--- /dev/null
+++ b/src/test/librados/test_common.cc
@@ -0,0 +1,167 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/Formatter.h"
+#include "include/stringify.h"
+#include "json_spirit/json_spirit.h"
+#include "test_common.h"
+
+using namespace std;
+
+namespace {
+
+using namespace ceph;
+
+int wait_for_healthy(rados_t *cluster)
+{
+  bool healthy = false;
+  // This timeout is very long because the tests are sometimes
+  // run on a thrashing cluster
+  int timeout = 3600;
+  int slept = 0;
+
+  while(!healthy) {
+    JSONFormatter cmd_f;
+    cmd_f.open_object_section("command");
+    cmd_f.dump_string("prefix", "status");
+    cmd_f.dump_string("format", "json");
+    cmd_f.close_section();
+    std::ostringstream cmd_stream;
+    cmd_f.flush(cmd_stream);
+    const std::string serialized_cmd = cmd_stream.str();
+
+    const char *cmd[2];
+    cmd[1] = NULL;
+    cmd[0] = serialized_cmd.c_str();
+
+    char *outbuf = NULL;
+    size_t outlen = 0;
+    int ret = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0,
+        &outbuf, &outlen, NULL, NULL);
+    if (ret) {
+      return ret;
+    }
+
+    std::string out(outbuf, outlen);
+    rados_buffer_free(outbuf);
+
+    json_spirit::mValue root;
+    [[maybe_unused]] bool json_parse_success = json_spirit::read(out, root);
+    ceph_assert(json_parse_success);
+    json_spirit::mObject root_obj = root.get_obj();
+    json_spirit::mObject pgmap = root_obj["pgmap"].get_obj();
+    json_spirit::mArray pgs_by_state = pgmap["pgs_by_state"].get_array();
+
+    if (pgs_by_state.size() == 1) {
+      json_spirit::mObject state = pgs_by_state[0].get_obj();
+      std::string state_name = state["state_name"].get_str();
+      if (state_name != std::string("active+clean")) {
+        healthy = false;
+      } else {
+        healthy = true;
+      }
+    } else {
+      healthy = false;
+    }
+
+    if (slept >= timeout) {
+      return -ETIMEDOUT;
+    };
+
+    if (!healthy) {
+      sleep(1);
+      slept += 1;
+    }
+  }
+
+  return 0;
+}
+
+int rados_pool_set(
+    rados_t *cluster,
+    const std::string &pool_name,
+    const std::string &var,
+    const std::string &val)
+{
+  JSONFormatter cmd_f;
+  cmd_f.open_object_section("command");
+  cmd_f.dump_string("prefix", "osd pool set");
+  cmd_f.dump_string("pool", pool_name);
+  cmd_f.dump_string("var", var);
+  cmd_f.dump_string("val", val);
+  cmd_f.close_section();
+
+  std::ostringstream cmd_stream;
+  cmd_f.flush(cmd_stream);
+
+  const std::string serialized_cmd = cmd_stream.str();
+
+  const char *cmd[2];
+  cmd[1] = NULL;
+  cmd[0] = serialized_cmd.c_str();
+  int ret = rados_mon_command(*cluster, (const char **)cmd, 1, "", 0, NULL,
+      NULL, NULL, NULL);
+  return ret;
+}
+
+struct pool_op_error : std::exception {
+  string msg;
+  pool_op_error(const std::string& pool_name,
+		const std::string& func_name,
+		int err) {
+    std::ostringstream oss;
+    oss << func_name << "(" << pool_name << ") failed with error " << err;
+    msg = oss.str();
+  }
+  const char* what() const noexcept override {
+    return msg.c_str();
+  }
+};
+
+template<typename Func>
+std::string with_healthy_cluster(rados_t* cluster,
+				 const std::string& pool_name,
+				 Func&& func)
+{
+  try {
+    // Wait for 'creating/backfilling' to clear
+    if (int r = wait_for_healthy(cluster); r != 0) {
+      throw pool_op_error{pool_name, "wait_for_healthy", r};
+    }
+    func();
+    // Wait for 'creating/backfilling' to clear
+    if (int r = wait_for_healthy(cluster); r != 0) {
+      throw pool_op_error{pool_name, "wait_for_healthy", r};
+    }
+  } catch (const pool_op_error& e) {
+    return e.what();
+  }
+  return "";
+}
+}
+
+std::string set_pg_num(
+    rados_t *cluster, const std::string &pool_name, uint32_t pg_num)
+{
+  return with_healthy_cluster(cluster, pool_name, [&] {
+    // Adjust pg_num
+      if (int r = rados_pool_set(cluster, pool_name, "pg_num",
+				 stringify(pg_num));
+	  r != 0) {
+	throw pool_op_error{pool_name, "set_pg_num", r};
+      }
+  });
+}
+
+std::string set_pgp_num(
+    rados_t *cluster, const std::string &pool_name, uint32_t pgp_num)
+{
+  return with_healthy_cluster(cluster, pool_name, [&] {
+    // Adjust pgp_num
+    if (int r = rados_pool_set(cluster, pool_name, "pgp_num",
+			       stringify(pgp_num));
+	r != 0) {
+      throw pool_op_error{pool_name, "set_pgp_num", r};
+    }
+  });
+}
diff --git a/src/test/librados/test_common.h b/src/test/librados/test_common.h
new file mode 100644
index 000000000..71ef9de2c
--- /dev/null
+++ b/src/test/librados/test_common.h
@@ -0,0 +1,9 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
+#include "include/rados/librados.h"
+
+std::string set_pg_num(
+    rados_t *cluster, const std::string &pool_name, uint32_t pg_num);
+std::string set_pgp_num(
+    rados_t *cluster, const std::string &pool_name, uint32_t pgp_num);
diff --git a/src/test/librados/test_cxx.cc b/src/test/librados/test_cxx.cc
new file mode 100644
index 000000000..6c7e353e4
--- /dev/null
+++ b/src/test/librados/test_cxx.cc
@@ -0,0 +1,203 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
+#include "test_cxx.h"
+
+#include "include/stringify.h"
+#include "common/ceph_context.h"
+#include "common/config.h"
+
+#include <errno.h>
+#include <sstream>
+#include <stdlib.h>
+#include <string>
+#include <time.h>
+#include <unistd.h>
+#include <iostream>
+#include "gtest/gtest.h"
+
+using namespace librados;
+
+std::string create_one_pool_pp(const std::string &pool_name, Rados &cluster)
+{
+    return create_one_pool_pp(pool_name, cluster, {});
+}
+std::string create_one_pool_pp(const std::string &pool_name, Rados &cluster,
+                               const std::map<std::string, std::string> &config)
+{
+  std::string err = connect_cluster_pp(cluster, config);
+  if (err.length())
+    return err;
+  int ret = cluster.pool_create(pool_name.c_str());
+  if (ret) {
+    cluster.shutdown();
+    std::ostringstream oss;
+    oss << "cluster.pool_create(" << pool_name << ") failed with error " << ret;
+    return oss.str();
+  }
+
+  IoCtx ioctx;
+  ret = cluster.ioctx_create(pool_name.c_str(), ioctx);
+  if (ret < 0) {
+    cluster.shutdown();
+    std::ostringstream oss;
+    oss << "cluster.ioctx_create(" << pool_name << ") failed with error "
+        << ret;
+    return oss.str();
+  }
+  ioctx.application_enable("rados", true);
+  return "";
+}
+
+int destroy_rule_pp(Rados &cluster,
+                       const std::string &rule,
+                       std::ostream &oss)
+{
+  bufferlist inbl;
+  int ret = cluster.mon_command("{\"prefix\": \"osd crush rule rm\", \"name\":\"" +
+                                rule + "\"}", inbl, NULL, NULL);
+  if (ret)
+    oss << "mon_command: osd crush rule rm " + rule + " failed with error " << ret << std::endl;
+  return ret;
+}
+
+int destroy_ec_profile_pp(Rados &cluster, const std::string& pool_name,
+			  std::ostream &oss)
+{
+  bufferlist inbl;
+  int ret = cluster.mon_command("{\"prefix\": \"osd erasure-code-profile rm\", \"name\": \"testprofile-" + pool_name + "\"}",
+                                inbl, NULL, NULL);
+  if (ret)
+    oss << "mon_command: osd erasure-code-profile rm testprofile-" << pool_name << " failed with error " << ret << std::endl;
+  return ret;
+}
+
+int destroy_ec_profile_and_rule_pp(Rados &cluster,
+                                      const std::string &rule,
+                                      std::ostream &oss)
+{
+  int ret;
+  ret = destroy_ec_profile_pp(cluster, rule, oss);
+  if (ret)
+    return ret;
+  return destroy_rule_pp(cluster, rule, oss);
+}
+
+std::string create_one_ec_pool_pp(const std::string &pool_name, Rados &cluster)
+{
+  std::string err = connect_cluster_pp(cluster);
+  if (err.length())
+    return err;
+
+  std::ostringstream oss;
+  int ret = destroy_ec_profile_and_rule_pp(cluster, pool_name, oss);
+  if (ret) {
+    cluster.shutdown();
+    return oss.str();
+  }
+
+  bufferlist inbl;
+  ret = cluster.mon_command(
+    "{\"prefix\": \"osd erasure-code-profile set\", \"name\": \"testprofile-" + pool_name + "\", \"profile\": [ \"k=2\", \"m=1\", \"crush-failure-domain=osd\"]}",
+    inbl, NULL, NULL);
+  if (ret) {
+    cluster.shutdown();
+    oss << "mon_command erasure-code-profile set name:testprofile-" << pool_name << " failed with error " << ret;
+    return oss.str();
+  }
+    
+  ret = cluster.mon_command(
+    "{\"prefix\": \"osd pool create\", \"pool\": \"" + pool_name + "\", \"pool_type\":\"erasure\", \"pg_num\":8, \"pgp_num\":8, \"erasure_code_profile\":\"testprofile-" + pool_name + "\"}",
+    inbl, NULL, NULL);
+  if (ret) {
+    bufferlist inbl;
+    destroy_ec_profile_pp(cluster, pool_name, oss);
+    cluster.shutdown();
+    oss << "mon_command osd pool create pool:" << pool_name << " pool_type:erasure failed with error " << ret;
+    return oss.str();
+  }
+
+  cluster.wait_for_latest_osdmap();
+  return "";
+}
+
+std::string connect_cluster_pp(librados::Rados &cluster)
+{
+  return connect_cluster_pp(cluster, {});
+}
+
+std::string connect_cluster_pp(librados::Rados &cluster,
+                               const std::map<std::string, std::string> &config)
+{
+  char *id = getenv("CEPH_CLIENT_ID");
+  if (id) std::cerr << "Client id is: " << id << std::endl;
+
+  int ret;
+  ret = cluster.init(id);
+  if (ret) {
+    std::ostringstream oss;
+    oss << "cluster.init failed with error " << ret;
+    return oss.str();
+  }
+  ret = cluster.conf_read_file(NULL);
+  if (ret) {
+    cluster.shutdown();
+    std::ostringstream oss;
+    oss << "cluster.conf_read_file failed with error " << ret;
+    return oss.str();
+  }
+  cluster.conf_parse_env(NULL);
+
+  for (auto &setting : config) {
+    ret = cluster.conf_set(setting.first.c_str(), setting.second.c_str());
+    if (ret) {
+      std::ostringstream oss;
+      oss << "failed to set config value " << setting.first << " to '"
+          << setting.second << "': " << strerror(-ret);
+      return oss.str();
+    }
+  }
+
+  ret = cluster.connect();
+  if (ret) {
+    cluster.shutdown();
+    std::ostringstream oss;
+    oss << "cluster.connect failed with error " << ret;
+    return oss.str();
+  }
+  return "";
+}
+
+int destroy_one_pool_pp(const std::string &pool_name, Rados &cluster)
+{
+  int ret = cluster.pool_delete(pool_name.c_str());
+  if (ret) {
+    cluster.shutdown();
+    return ret;
+  }
+  cluster.shutdown();
+  return 0;
+}
+
+int destroy_one_ec_pool_pp(const std::string &pool_name, Rados &cluster)
+{
+  int ret = cluster.pool_delete(pool_name.c_str());
+  if (ret) {
+    cluster.shutdown();
+    return ret;
+  }
+
+  CephContext *cct = static_cast<CephContext*>(cluster.cct());
+  if (!cct->_conf->mon_fake_pool_delete) { // hope this is in [global]
+    std::ostringstream oss;
+    ret = destroy_ec_profile_and_rule_pp(cluster, pool_name, oss);
+    if (ret) {
+      cluster.shutdown();
+      return ret;
+    }
+  }
+
+  cluster.wait_for_latest_osdmap();
+  cluster.shutdown();
+  return ret;
+}
diff --git a/src/test/librados/test_cxx.h b/src/test/librados/test_cxx.h
new file mode 100644
index 000000000..1d11d6923
--- /dev/null
+++ b/src/test/librados/test_cxx.h
@@ -0,0 +1,19 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#pragma once
+
+#include "include/rados/librados.hpp"
+#include "test/librados/test_shared.h"
+
+std::string create_one_pool_pp(const std::string &pool_name,
+			    librados::Rados &cluster);
+std::string create_one_pool_pp(const std::string &pool_name,
+			       librados::Rados &cluster,
+			       const std::map<std::string, std::string> &config);
+std::string create_one_ec_pool_pp(const std::string &pool_name,
+			    librados::Rados &cluster);
+std::string connect_cluster_pp(librados::Rados &cluster);
+std::string connect_cluster_pp(librados::Rados &cluster,
+			       const std::map<std::string, std::string> &config);
+int destroy_one_pool_pp(const std::string &pool_name, librados::Rados &cluster);
+int destroy_one_ec_pool_pp(const std::string &pool_name, librados::Rados &cluster);
diff --git a/src/test/librados/test_shared.cc b/src/test/librados/test_shared.cc
new file mode 100644
index 000000000..8b50d112e
--- /dev/null
+++ b/src/test/librados/test_shared.cc
@@ -0,0 +1,44 @@
+#include "test_shared.h"
+
+#include <cstring>
+#include "gtest/gtest.h"
+#include "include/buffer.h"
+
+using namespace ceph;
+
+std::string get_temp_pool_name(const std::string &prefix)
+{
+  char hostname[80];
+  char out[160];
+  memset(hostname, 0, sizeof(hostname));
+  memset(out, 0, sizeof(out));
+  gethostname(hostname, sizeof(hostname)-1);
+  static int num = 1;
+  snprintf(out, sizeof(out), "%s-%d-%d", hostname, getpid(), num);
+  num++;
+  return prefix + out;
+}
+
+void assert_eq_sparse(bufferlist& expected,
+                      const std::map<uint64_t, uint64_t>& extents,
+                      bufferlist& actual) {
+  auto i = expected.begin();
+  auto p = actual.begin();
+  uint64_t pos = 0;
+  for (auto extent : extents) {
+    const uint64_t start = extent.first;
+    const uint64_t end = start + extent.second;
+    for (; pos < end; ++i, ++pos) {
+      ASSERT_FALSE(i.end());
+      if (pos < start) {
+        // check the hole
+        ASSERT_EQ('\0', *i);
+      } else {
+        // then the extent
+        ASSERT_EQ(*i, *p);
+        ++p;
+      }
+    }
+  }
+  ASSERT_EQ(expected.length(), pos);
+}
diff --git a/src/test/librados/test_shared.h b/src/test/librados/test_shared.h
new file mode 100644
index 000000000..3a18e916e
--- /dev/null
+++ b/src/test/librados/test_shared.h
@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include <unistd.h>
+#include <chrono>
+#include <map>
+#include <string>
+#include <thread>
+
+#include "include/buffer_fwd.h"
+
+// helpers shared by librados tests
+std::string get_temp_pool_name(const std::string &prefix = "test-rados-api-");
+void assert_eq_sparse(ceph::bufferlist& expected,
+                      const std::map<uint64_t, uint64_t>& extents,
+                      ceph::bufferlist& actual);
+class TestAlarm
+{
+public:
+  #ifndef _WIN32
+  TestAlarm() {
+    alarm(2400);
+  }
+  ~TestAlarm() {
+    alarm(0);
+  }
+  #else
+  // TODO: add a timeout mechanism for Windows as well, possibly by using
+  // CreateTimerQueueTimer.
+  TestAlarm() {
+  }
+  ~TestAlarm() {
+  }
+  #endif
+};
+
+template<class Rep, class Period, typename Func, typename... Args,
+         typename Return = std::result_of_t<Func&&(Args&&...)>>
+Return wait_until(const std::chrono::duration<Rep, Period>& rel_time,
+                const std::chrono::duration<Rep, Period>& step,
+                const Return& expected,
+                Func&& func, Args&&... args)
+{
+  std::this_thread::sleep_for(rel_time - step);
+  for (auto& s : {step, step}) {
+    if (!s.count()) {
+      break;
+    }
+    auto ret = func(std::forward<Args>(args)...);
+    if (ret == expected) {
+      return ret;
+    }
+    std::this_thread::sleep_for(s);
+  }
+  return func(std::forward<Args>(args)...);
+}
diff --git a/src/test/librados/testcase_cxx.cc b/src/test/librados/testcase_cxx.cc
new file mode 100644
index 000000000..407c59b55
--- /dev/null
+++ b/src/test/librados/testcase_cxx.cc
@@ -0,0 +1,407 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "testcase_cxx.h"
+
+#include <errno.h>
+#include <fmt/format.h>
+#include "test_cxx.h"
+#include "test_shared.h"
+#include "crimson_utils.h"
+#include "include/scope_guard.h"
+
+using namespace librados;
+
+namespace {
+
+void init_rand() {
+  static bool seeded = false;
+  if (!seeded) {
+    seeded = true;
+    int seed = getpid();
+    std::cout << "seed " << seed << std::endl;
+    srand(seed);
+  }
+}
+
+} // anonymous namespace
+
+std::string RadosTestPPNS::pool_name;
+Rados RadosTestPPNS::s_cluster;
+
+void RadosTestPPNS::SetUpTestCase()
+{
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestPPNS::TearDownTestCase()
+{
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestPPNS::SetUp()
+{
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+  bool req;
+  ASSERT_EQ(0, ioctx.pool_requires_alignment2(&req));
+  ASSERT_FALSE(req);
+}
+
+void RadosTestPPNS::TearDown()
+{
+  if (cleanup)
+    cleanup_all_objects(ioctx);
+  ioctx.close();
+}
+
+void RadosTestPPNS::cleanup_all_objects(librados::IoCtx ioctx)
+{
+  // remove all objects to avoid polluting other tests
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  ioctx.set_namespace(all_nspaces);
+  for (NObjectIterator it = ioctx.nobjects_begin();
+       it != ioctx.nobjects_end(); ++it) {
+    ioctx.locator_set_key(it->get_locator());
+    ioctx.set_namespace(it->get_nspace());
+    ASSERT_EQ(0, ioctx.remove(it->get_oid()));
+  }
+}
+
+std::string RadosTestParamPPNS::pool_name;
+std::string RadosTestParamPPNS::cache_pool_name;
+Rados RadosTestParamPPNS::s_cluster;
+
+void RadosTestParamPPNS::SetUpTestCase()
+{
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestParamPPNS::TearDownTestCase()
+{
+  if (cache_pool_name.length()) {
+    // tear down tiers
+    bufferlist inbl;
+    ASSERT_EQ(0, s_cluster.mon_command(
+      "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+      "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, s_cluster.mon_command(
+      "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, s_cluster.mon_command(
+      "{\"prefix\": \"osd pool delete\", \"pool\": \"" + cache_pool_name +
+      "\", \"pool2\": \"" + cache_pool_name + "\", \"yes_i_really_really_mean_it\": true}",
+      inbl, NULL, NULL));
+    cache_pool_name = "";
+  }
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestParamPPNS::SetUp()
+{
+  if (!is_crimson_cluster() && strcmp(GetParam(), "cache") == 0 &&
+      cache_pool_name.empty()) {
+    auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+    cache_pool_name = get_temp_pool_name();
+    bufferlist inbl;
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd pool create\", \"pool\": \"" + cache_pool_name +
+      "\", \"pg_num\": 4}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name +
+      "\", \"force_nonempty\": \"--force-nonempty\" }",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+      "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+      "\", \"mode\": \"writeback\"}",
+      inbl, NULL, NULL));
+    cluster.wait_for_latest_osdmap();
+  }
+
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+  bool req;
+  ASSERT_EQ(0, ioctx.pool_requires_alignment2(&req));
+  ASSERT_FALSE(req);
+}
+
+void RadosTestParamPPNS::TearDown()
+{
+  if (cleanup)
+    cleanup_all_objects(ioctx);
+  ioctx.close();
+}
+
+void RadosTestParamPPNS::cleanup_all_objects(librados::IoCtx ioctx)
+{
+  // remove all objects to avoid polluting other tests
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  ioctx.set_namespace(all_nspaces);
+  for (NObjectIterator it = ioctx.nobjects_begin();
+       it != ioctx.nobjects_end(); ++it) {
+    ioctx.locator_set_key(it->get_locator());
+    ioctx.set_namespace(it->get_nspace());
+    ASSERT_EQ(0, ioctx.remove(it->get_oid()));
+  }
+}
+
+std::string RadosTestECPPNS::pool_name;
+Rados RadosTestECPPNS::s_cluster;
+
+void RadosTestECPPNS::SetUpTestCase()
+{
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestECPPNS::TearDownTestCase()
+{
+  ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestECPPNS::SetUp()
+{
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+  bool req;
+  ASSERT_EQ(0, ioctx.pool_requires_alignment2(&req));
+  ASSERT_TRUE(req);
+  ASSERT_EQ(0, ioctx.pool_required_alignment2(&alignment));
+  ASSERT_NE(0U, alignment);
+}
+
+void RadosTestECPPNS::TearDown()
+{
+  if (cleanup)
+    cleanup_all_objects(ioctx);
+  ioctx.close();
+}
+
+std::string RadosTestPP::pool_name;
+Rados RadosTestPP::s_cluster;
+
+void RadosTestPP::SetUpTestCase()
+{
+  init_rand();
+
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestPP::TearDownTestCase()
+{
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestPP::SetUp()
+{
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+  nspace = get_temp_pool_name();
+  ioctx.set_namespace(nspace);
+  bool req;
+  ASSERT_EQ(0, ioctx.pool_requires_alignment2(&req));
+  ASSERT_FALSE(req);
+}
+
+void RadosTestPP::TearDown()
+{
+  if (cleanup) {
+    cleanup_default_namespace(ioctx);
+    cleanup_namespace(ioctx, nspace);
+  }
+  ioctx.close();
+}
+
+void RadosTestPP::cleanup_default_namespace(librados::IoCtx ioctx)
+{
+  // remove all objects from the default namespace to avoid polluting
+  // other tests
+  cleanup_namespace(ioctx, "");
+}
+
+void RadosTestPP::cleanup_namespace(librados::IoCtx ioctx, std::string ns)
+{
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  ioctx.set_namespace(ns);
+  int tries = 20;
+  while (--tries) {
+    int got_enoent = 0;
+    for (NObjectIterator it = ioctx.nobjects_begin();
+	 it != ioctx.nobjects_end(); ++it) {
+      ioctx.locator_set_key(it->get_locator());
+      ObjectWriteOperation op;
+      op.remove();
+      librados::AioCompletion *completion = s_cluster.aio_create_completion();
+      auto sg = make_scope_guard([&] { completion->release(); });
+      ASSERT_EQ(0, ioctx.aio_operate(it->get_oid(), completion, &op,
+				     librados::OPERATION_IGNORE_CACHE));
+      completion->wait_for_complete();
+      if (completion->get_return_value() == -ENOENT) {
+	++got_enoent;
+	std::cout << " got ENOENT removing " << it->get_oid()
+		  << " in ns " << ns << std::endl;
+      } else {
+	ASSERT_EQ(0, completion->get_return_value());
+      }
+    }
+    if (!got_enoent) {
+      break;
+    }
+    std::cout << " got ENOENT on " << got_enoent
+	      << " objects, waiting a bit for snap"
+	      << " trimming before retrying " << tries << " more times..."
+	      << std::endl;
+    sleep(1);
+  }
+  if (tries == 0) {
+    std::cout << "failed to clean up; probably need to scrub purged_snaps."
+	      << std::endl;
+  }
+}
+
+std::string RadosTestParamPP::pool_name;
+std::string RadosTestParamPP::cache_pool_name;
+Rados RadosTestParamPP::s_cluster;
+
+void RadosTestParamPP::SetUpTestCase()
+{
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestParamPP::TearDownTestCase()
+{
+  if (cache_pool_name.length()) {
+    // tear down tiers
+    bufferlist inbl;
+    ASSERT_EQ(0, s_cluster.mon_command(
+      "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+      "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, s_cluster.mon_command(
+      "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, s_cluster.mon_command(
+      "{\"prefix\": \"osd pool delete\", \"pool\": \"" + cache_pool_name +
+      "\", \"pool2\": \"" + cache_pool_name + "\", \"yes_i_really_really_mean_it\": true}",
+      inbl, NULL, NULL));
+    cache_pool_name = "";
+  }
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestParamPP::SetUp()
+{
+  if (!is_crimson_cluster() && strcmp(GetParam(), "cache") == 0 &&
+      cache_pool_name.empty()) {
+    auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+    cache_pool_name = get_temp_pool_name();
+    bufferlist inbl;
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd pool create\", \"pool\": \"" + cache_pool_name +
+      "\", \"pg_num\": 4}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name +
+      "\", \"force_nonempty\": \"--force-nonempty\" }",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+      "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+      "\", \"mode\": \"writeback\"}",
+      inbl, NULL, NULL));
+    cluster.wait_for_latest_osdmap();
+  }
+
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+  nspace = get_temp_pool_name();
+  ioctx.set_namespace(nspace);
+  bool req;
+  ASSERT_EQ(0, ioctx.pool_requires_alignment2(&req));
+  ASSERT_FALSE(req);
+}
+
+void RadosTestParamPP::TearDown()
+{
+  if (cleanup) {
+    cleanup_default_namespace(ioctx);
+    cleanup_namespace(ioctx, nspace);
+  }
+  ioctx.close();
+}
+
+void RadosTestParamPP::cleanup_default_namespace(librados::IoCtx ioctx)
+{
+  // remove all objects from the default namespace to avoid polluting
+  // other tests
+  cleanup_namespace(ioctx, "");
+}
+
+void RadosTestParamPP::cleanup_namespace(librados::IoCtx ioctx, std::string ns)
+{
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  ioctx.set_namespace(ns);
+  for (NObjectIterator it = ioctx.nobjects_begin();
+       it != ioctx.nobjects_end(); ++it) {
+    ioctx.locator_set_key(it->get_locator());
+    ASSERT_EQ(0, ioctx.remove(it->get_oid()));
+  }
+}
+
+std::string RadosTestECPP::pool_name;
+Rados RadosTestECPP::s_cluster;
+
+void RadosTestECPP::SetUpTestCase()
+{
+  SKIP_IF_CRIMSON();
+  auto pool_prefix = fmt::format("{}_", ::testing::UnitTest::GetInstance()->current_test_case()->name());
+  pool_name = get_temp_pool_name(pool_prefix);
+  ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestECPP::TearDownTestCase()
+{
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
+}
+
+void RadosTestECPP::SetUp()
+{
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+  nspace = get_temp_pool_name();
+  ioctx.set_namespace(nspace);
+  bool req;
+  ASSERT_EQ(0, ioctx.pool_requires_alignment2(&req));
+  ASSERT_TRUE(req);
+  ASSERT_EQ(0, ioctx.pool_required_alignment2(&alignment));
+  ASSERT_NE(0U, alignment);
+}
+
+void RadosTestECPP::TearDown()
+{
+  SKIP_IF_CRIMSON();
+  if (cleanup) {
+    cleanup_default_namespace(ioctx);
+    cleanup_namespace(ioctx, nspace);
+  }
+  ioctx.close();
+}
+
diff --git a/src/test/librados/testcase_cxx.h b/src/test/librados/testcase_cxx.h
new file mode 100644
index 000000000..637ec11ee
--- /dev/null
+++ b/src/test/librados/testcase_cxx.h
@@ -0,0 +1,130 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#pragma once
+
+#include "gtest/gtest.h"
+#include "include/rados/librados.hpp"
+
+class RadosTestPPNS : public ::testing::Test {
+public:
+  RadosTestPPNS(bool c=false) : cluster(s_cluster), cleanup(c) {}
+  ~RadosTestPPNS() override {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static void cleanup_all_objects(librados::IoCtx ioctx);
+  static librados::Rados s_cluster;
+  static std::string pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  librados::Rados &cluster;
+  librados::IoCtx ioctx;
+  bool cleanup;
+};
+
+struct RadosTestPPNSCleanup : public RadosTestPPNS {
+  RadosTestPPNSCleanup() : RadosTestPPNS(true) {}
+};
+
+class RadosTestParamPPNS : public ::testing::TestWithParam<const char*> {
+public:
+  RadosTestParamPPNS(bool c=false) : cluster(s_cluster), cleanup(c) {}
+  ~RadosTestParamPPNS() override {}
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+protected:
+  static void cleanup_all_objects(librados::IoCtx ioctx);
+  static librados::Rados s_cluster;
+  static std::string pool_name;
+  static std::string cache_pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  librados::Rados &cluster;
+  librados::IoCtx ioctx;
+  bool cleanup;
+};
+
+class RadosTestECPPNS : public RadosTestPPNS {
+public:
+  RadosTestECPPNS(bool c=false) : cluster(s_cluster), cleanup(c) {}
+  ~RadosTestECPPNS() override {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static librados::Rados s_cluster;
+  static std::string pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  librados::Rados &cluster;
+  librados::IoCtx ioctx;
+  uint64_t alignment = 0;
+  bool cleanup;
+};
+
+struct RadosTestECPPNSCleanup : public RadosTestECPPNS {
+  RadosTestECPPNSCleanup() : RadosTestECPPNS(true) {}
+};
+
+class RadosTestPP : public ::testing::Test {
+public:
+  RadosTestPP(bool c=false) : cluster(s_cluster), cleanup(c) {}
+  ~RadosTestPP() override {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static void cleanup_default_namespace(librados::IoCtx ioctx);
+  static void cleanup_namespace(librados::IoCtx ioctx, std::string ns);
+  static librados::Rados s_cluster;
+  static std::string pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  librados::Rados &cluster;
+  librados::IoCtx ioctx;
+  bool cleanup;
+  std::string nspace;
+};
+
+class RadosTestParamPP : public ::testing::TestWithParam<const char*> {
+public:
+  RadosTestParamPP(bool c=false) : cluster(s_cluster), cleanup(c) {}
+  ~RadosTestParamPP() override {}
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+protected:
+  static void cleanup_default_namespace(librados::IoCtx ioctx);
+  static void cleanup_namespace(librados::IoCtx ioctx, std::string ns);
+  static librados::Rados s_cluster;
+  static std::string pool_name;
+  static std::string cache_pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  librados::Rados &cluster;
+  librados::IoCtx ioctx;
+  bool cleanup;
+  std::string nspace;
+};
+
+class RadosTestECPP : public RadosTestPP {
+public:
+  RadosTestECPP(bool c=false) : cluster(s_cluster), cleanup(c) {}
+  ~RadosTestECPP() override {}
+protected:
+  static void SetUpTestCase();
+  static void TearDownTestCase();
+  static librados::Rados s_cluster;
+  static std::string pool_name;
+
+  void SetUp() override;
+  void TearDown() override;
+  librados::Rados &cluster;
+  librados::IoCtx ioctx;
+  bool cleanup;
+  std::string nspace;
+  uint64_t alignment = 0;
+};
diff --git a/src/test/librados/tier_cxx.cc b/src/test/librados/tier_cxx.cc
new file mode 100644
index 000000000..be1d411a9
--- /dev/null
+++ b/src/test/librados/tier_cxx.cc
@@ -0,0 +1,9304 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#include "gtest/gtest.h"
+
+#include "mds/mdstypes.h"
+#include "include/buffer.h"
+#include "include/rbd_types.h"
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "include/types.h"
+#include "global/global_context.h"
+#include "common/Cond.h"
+#include "common/ceph_crypto.h"
+#include "test/librados/test_cxx.h"
+#include "test/librados/testcase_cxx.h"
+#include "json_spirit/json_spirit.h"
+#include "cls/cas/cls_cas_ops.h"
+#include "cls/cas/cls_cas_internal.h"
+
+#include "osd/HitSet.h"
+
+#include <errno.h>
+#include <map>
+#include <sstream>
+#include <string>
+
+#include "cls/cas/cls_cas_client.h"
+#include "cls/cas/cls_cas_internal.h"
+#include "crimson_utils.h"
+
+using namespace std;
+using namespace librados;
+using ceph::crypto::SHA1;
+
+typedef RadosTestPP LibRadosTierPP;
+typedef RadosTestECPP LibRadosTierECPP;
+
+void flush_evict_all(librados::Rados& cluster, librados::IoCtx& cache_ioctx)
+{
+  bufferlist inbl;
+  cache_ioctx.set_namespace(all_nspaces);
+  for (NObjectIterator it = cache_ioctx.nobjects_begin();
+       it != cache_ioctx.nobjects_end(); ++it) {
+    cache_ioctx.locator_set_key(it->get_locator());
+    cache_ioctx.set_namespace(it->get_nspace());
+    {
+      ObjectReadOperation op;
+      op.cache_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      cache_ioctx.aio_operate(
+        it->get_oid(), completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL);
+      completion->wait_for_complete();
+      completion->get_return_value();
+      completion->release();
+    }
+    {
+      ObjectReadOperation op;
+      op.cache_evict();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      cache_ioctx.aio_operate(
+        it->get_oid(), completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL);
+      completion->wait_for_complete();
+      completion->get_return_value();
+      completion->release();
+    }
+  }
+}
+
+static string _get_required_osd_release(Rados& cluster)
+{
+  bufferlist inbl;
+  string cmd = string("{\"prefix\": \"osd dump\",\"format\":\"json\"}");
+  bufferlist outbl;
+  int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
+  ceph_assert(r >= 0);
+  string outstr(outbl.c_str(), outbl.length());
+  json_spirit::Value v;
+  if (!json_spirit::read(outstr, v)) {
+    cerr <<" unable to parse json " << outstr << std::endl;
+    return "";
+  }
+
+  json_spirit::Object& o = v.get_obj();
+  for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
+    json_spirit::Pair& p = o[i];
+    if (p.name_ == "require_osd_release") {
+      cout << "require_osd_release = " << p.value_.get_str() << std::endl;
+      return p.value_.get_str();
+    }
+  }
+  cerr << "didn't find require_osd_release in " << outstr << std::endl;
+  return "";
+}
+
+void manifest_set_chunk(Rados& cluster, librados::IoCtx& src_ioctx, 
+			librados::IoCtx& tgt_ioctx,
+			uint64_t src_offset, uint64_t length, 
+			std::string src_oid, std::string tgt_oid)
+{
+  ObjectReadOperation op;
+  op.set_chunk(src_offset, length, src_ioctx, src_oid, 0,
+      CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+  librados::AioCompletion *completion = cluster.aio_create_completion();
+  ASSERT_EQ(0, tgt_ioctx.aio_operate(tgt_oid, completion, &op,
+	    librados::OPERATION_IGNORE_CACHE, NULL));
+  completion->wait_for_complete();
+  ASSERT_EQ(0, completion->get_return_value());
+  completion->release();
+}
+
+static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  int i;
+  str[0] = '\0';
+  for (i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+void check_fp_oid_refcount(librados::IoCtx& ioctx, std::string foid, uint64_t count,
+			   std::string fp_algo = NULL)
+{
+  bufferlist t;
+  int size = foid.length();
+  if (fp_algo == "sha1") {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    SHA1 sha1_gen;
+    sha1_gen.Update((const unsigned char *)foid.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
+  } else if (fp_algo.empty()) {
+    ioctx.getxattr(foid, CHUNK_REFCOUNT_ATTR, t);
+  } else if (!fp_algo.empty()) {
+    ceph_assert(0 == "unrecognized fingerprint algorithm");
+  }
+
+  chunk_refs_t refs;
+  try {
+    auto iter = t.cbegin();
+    decode(refs, iter);
+  } catch (buffer::error& err) {
+    ASSERT_TRUE(0);
+  }
+  ASSERT_LE(count, refs.count());
+}
+
+string get_fp_oid(string oid, std::string fp_algo = NULL)
+{
+  if (fp_algo == "sha1") {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    SHA1 sha1_gen;
+    int size = oid.length();
+    sha1_gen.Update((const unsigned char *)oid.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    return string(p_str);
+  } 
+ 
+  return string();
+}
+
+void is_intended_refcount_state(librados::IoCtx& src_ioctx,
+				std::string src_oid,
+				librados::IoCtx& dst_ioctx,
+				std::string dst_oid,
+				int expected_refcount)
+{
+  int src_refcount = 0, dst_refcount = 0;
+  bufferlist t;
+  int r = dst_ioctx.getxattr(dst_oid, CHUNK_REFCOUNT_ATTR, t);
+  if (r == -ENOENT) {
+    dst_refcount = 0;
+  } else {
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ceph_assert(0);
+    }
+    dst_refcount = refs.count();
+  }
+  int tries = 0;
+  for (; tries < 30; ++tries) {
+    r = cls_cas_references_chunk(src_ioctx, src_oid, dst_oid);
+    if (r == -ENOENT || r == -ENOLINK) {
+      src_refcount = 0;
+    } else if (r == -EBUSY) {
+      sleep(20);
+      continue;
+    } else {
+      src_refcount = r;
+    }
+    break;
+  }
+  ASSERT_TRUE(tries < 30);
+  ASSERT_TRUE(src_refcount >= 0);
+  ASSERT_TRUE(src_refcount == expected_refcount);
+  ASSERT_TRUE(src_refcount <= dst_refcount);
+}
+
+class LibRadosTwoPoolsPP : public RadosTestPP
+{
+public:
+  LibRadosTwoPoolsPP() {};
+  ~LibRadosTwoPoolsPP() override {};
+protected:
+  static void SetUpTestCase() {
+    pool_name = get_temp_pool_name();
+    ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
+  }
+  static void TearDownTestCase() {
+    ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
+  }
+  static std::string cache_pool_name;
+
+  void SetUp() override {
+    SKIP_IF_CRIMSON();
+    cache_pool_name = get_temp_pool_name();
+    ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
+    RadosTestPP::SetUp();
+
+    ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+    cache_ioctx.application_enable("rados", true);
+    cache_ioctx.set_namespace(nspace);
+  }
+  void TearDown() override {
+    SKIP_IF_CRIMSON();
+    // flush + evict cache
+    flush_evict_all(cluster, cache_ioctx);
+
+    bufferlist inbl;
+    // tear down tiers
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+      "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+    // wait for maps to settle before next test
+    cluster.wait_for_latest_osdmap();
+
+    RadosTestPP::TearDown();
+
+    cleanup_default_namespace(cache_ioctx);
+    cleanup_namespace(cache_ioctx, nspace);
+
+    cache_ioctx.close();
+    ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
+  }
+  librados::IoCtx cache_ioctx;
+};
+
+class Completions
+{
+public:
+  Completions() = default;
+  librados::AioCompletion* getCompletion() {
+    librados::AioCompletion* comp = librados::Rados::aio_create_completion();
+    m_completions.push_back(comp);
+    return comp;
+  }
+
+  ~Completions() {
+    for (auto& comp : m_completions) {
+      comp->release();
+    }
+  }
+
+private:
+  vector<librados::AioCompletion *> m_completions;
+};
+
+Completions completions;
+
+std::string LibRadosTwoPoolsPP::cache_pool_name;
+
+TEST_F(LibRadosTierPP, Dirty) {
+  SKIP_IF_CRIMSON();
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
+  }
+  {
+    ObjectWriteOperation op;
+    op.create(true);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));  // still 0 if already clean
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+  {
+    ObjectWriteOperation op;
+    op.truncate(0);  // still a write even tho it is a no-op
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, Overlay) {
+  SKIP_IF_CRIMSON();
+  // create objects
+  {
+    bufferlist bl;
+    bl.append("base");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("cache");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // by default, the overlay sends us to cache pool
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // unless we say otherwise
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(0, 1, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+	"foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+    ASSERT_EQ('b', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, Promote) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+  }
+
+  // read, trigger a whiteout
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, PromoteSnap) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  ioctx.snap_set_read(my_snaps[0]);
+
+  // read foo snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // read bar snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // read baz snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+
+  // read foo
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // read bar
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // read baz
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
+  }
+
+  // cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+}
+
+TEST_F(LibRadosTwoPoolsPP, PromoteSnapScrub) {
+  SKIP_IF_CRIMSON();
+  int num = 100;
+
+  // create objects
+  for (int i=0; i<num; ++i) {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
+  }
+
+  vector<uint64_t> my_snaps;
+  for (int snap=0; snap<4; ++snap) {
+    // create a snapshot, clone
+    vector<uint64_t> ns(1);
+    ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
+    my_snaps.swap(ns);
+    ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+    cout << "my_snaps " << my_snaps << std::endl;
+    ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+						      my_snaps));
+    for (int i=0; i<num; ++i) {
+      bufferlist bl;
+      bl.append(string("ciao! snap") + stringify(snap));
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
+    }
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on _some_ heads to make sure we handle cases
+  // where snaps are present and where they are not.
+  cout << "promoting some heads" << std::endl;
+  for (int i=0; i<num; ++i) {
+    if (i % 5 == 0 || i > num - 3) {
+      bufferlist bl;
+      ASSERT_EQ(1, ioctx.read(string("foo") + stringify(i), bl, 1, 0));
+      ASSERT_EQ('c', bl[0]);
+    }
+  }
+
+  for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
+    cout << "promoting from clones for snap " << my_snaps[snap] << std::endl;
+    ioctx.snap_set_read(my_snaps[snap]);
+
+    // read some snaps, semi-randomly
+    for (int i=0; i<50; ++i) {
+      bufferlist bl;
+      string o = string("foo") + stringify((snap * i * 137) % 80);
+      //cout << o << std::endl;
+      ASSERT_EQ(1, ioctx.read(o, bl, 1, 0));
+    }
+  }
+
+  // ok, stop and scrub this pool (to make sure scrub can handle
+  // missing clones in the cache tier).
+  {
+    IoCtx cache_ioctx;
+    ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+    for (int i=0; i<10; ++i) {
+      do {
+	ostringstream ss;
+	ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
+	   << cache_ioctx.get_id() << "." << i
+	   << "\"}";
+	int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
+	if (r == -ENOENT ||  // in case mgr osdmap is stale
+	    r == -EAGAIN) {
+	  sleep(5);
+	  continue;
+	}
+      } while (false);
+    }
+
+    // give it a few seconds to go.  this is sloppy but is usually enough time
+    cout << "waiting for scrubs..." << std::endl;
+    sleep(30);
+    cout << "done waiting" << std::endl;
+  }
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+
+  //cleanup
+  for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
+    ioctx.selfmanaged_snap_remove(my_snaps[snap]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, PromoteSnapTrimRace) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // delete the snap
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
+
+  ioctx.snap_set_read(my_snaps[0]);
+
+  // read foo snap.  the OSD may or may not realize that this snap has
+  // been logically deleted; either response is valid.
+  {
+    bufferlist bl;
+    int r = ioctx.read("foo", bl, 1, 0);
+    ASSERT_TRUE(r == 1 || r == -ENOENT);
+  }
+
+  // cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+}
+
+TEST_F(LibRadosTwoPoolsPP, Whiteout) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create some whiteouts, verify they behave
+  {
+    ObjectWriteOperation op;
+    op.assert_exists();
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  {
+    ObjectWriteOperation op;
+    op.assert_exists();
+    op.remove();
+    ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.assert_exists();
+    op.remove();
+    ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
+  }
+
+  // verify the whiteouts are there in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // delete a whiteout and verify it goes away
+  ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
+				   librados::OPERATION_IGNORE_CACHE));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // recreate an object and verify we can read it
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, WhiteoutDeleteCreate) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create an object
+  {
+    bufferlist bl;
+    bl.append("foo");
+    ASSERT_EQ(0, ioctx.write_full("foo", bl));
+  }
+
+  // do delete + create operation
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    bufferlist bl;
+    bl.append("bar");
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify it still "exists" (w/ new content)
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('b', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, Evict) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+  }
+
+  // read, trigger a whiteout, and a dirty object
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // pin
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict the pinned object with -EPERM
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+					 librados::OPERATION_IGNORE_CACHE,
+					 NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EPERM, completion->get_return_value());
+    completion->release();
+  }
+
+  // unpin
+  {
+    ObjectWriteOperation op;
+    op.cache_unpin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+					 librados::OPERATION_IGNORE_CACHE,
+					 NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // evict bam
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bam", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bam", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+
+  // read foo snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // evict foo snap
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // snap is gone...
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+  // head is still there...
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // promote head + snap of bar
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // evict bar head (fail)
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict bar snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // ...and then head
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+}
+
+// this test case reproduces http://tracker.ceph.com/issues/8629
+TEST_F(LibRadosTwoPoolsPP, EvictSnap2) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify the snapdir is not present in the cache pool
+  {
+    ObjectReadOperation op;
+    librados::snap_set_t snapset;
+    op.list_snaps(&snapset, NULL);
+    ioctx.snap_set_read(librados::SNAP_DIR);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op,
+				   librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+}
+
+//This test case reproduces http://tracker.ceph.com/issues/17445
+TEST_F(LibRadosTwoPoolsPP, ListSnap){
+  SKIP_IF_CRIMSON();
+  // Create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // Create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // Configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // Wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // Read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // Read foo snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // Evict foo snap
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // Snap is gone...
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+
+  // Do list-snaps
+  ioctx.snap_set_read(CEPH_SNAPDIR);
+  {
+    snap_set_t snap_set;
+    int snap_ret;
+    ObjectReadOperation op;
+    op.list_snaps(&snap_set, &snap_ret);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      0, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, snap_ret);
+    ASSERT_LT(0u, snap_set.clones.size());
+    for (vector<librados::clone_info_t>::const_iterator r = snap_set.clones.begin();
+	r != snap_set.clones.end();
+	++r) {
+      if (r->cloneid != librados::SNAP_HEAD) {
+	ASSERT_LT(0u, r->snaps.size());
+      }
+    }
+  }
+
+  // Cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+}
+
+// This test case reproduces https://tracker.ceph.com/issues/49409
+TEST_F(LibRadosTwoPoolsPP, EvictSnapRollbackReadRace) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    int len = string("hi there").length() * 2;
+    // append more chrunk data make sure the second promote
+    // op coming before the first promote op finished
+    for (int i=0; i<4*1024*1024/len; ++i)
+      bl.append("hi therehi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // create two snapshot, a clone
+  vector<uint64_t> my_snaps(2);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[1]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+                                                         my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // try more times
+  int retries = 50;
+  for (int i=0; i<retries; ++i)
+  {
+    {
+      librados::AioCompletion * completion = cluster.aio_create_completion();
+      librados::AioCompletion * completion1 = cluster.aio_create_completion();
+
+      // send a snap rollback op and a snap read op parallel
+      // trigger two promote(copy) to the same snap clone obj
+      // the second snap read op is read-ordered make sure
+      // op not wait for objects_blocked_on_snap_promotion
+      ObjectWriteOperation op;
+      op.selfmanaged_snap_rollback(my_snaps[0]);
+      ASSERT_EQ(0, ioctx.aio_operate(
+        "foo", completion, &op));
+
+      ioctx.snap_set_read(my_snaps[1]);
+      std::map<uint64_t, uint64_t> extents;
+      bufferlist read_bl;
+      int rval = -1;
+      ObjectReadOperation op1;
+      op1.sparse_read(0, 8, &extents, &read_bl, &rval);
+      ASSERT_EQ(0, ioctx.aio_operate("foo", completion1, &op1, &read_bl));
+      ioctx.snap_set_read(librados::SNAP_HEAD);
+
+      completion->wait_for_complete();
+      ASSERT_EQ(0, completion->get_return_value());
+      completion->release();
+
+      completion1->wait_for_complete();
+      ASSERT_EQ(0, completion1->get_return_value());
+      completion1->release();
+    }
+
+    // evict foo snap
+    ioctx.snap_set_read(my_snaps[0]);
+    {
+      ObjectReadOperation op;
+      op.cache_evict();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, ioctx.aio_operate(
+        "foo", completion, &op,
+        librados::OPERATION_IGNORE_CACHE, NULL));
+      completion->wait_for_complete();
+      ASSERT_EQ(0, completion->get_return_value());
+      completion->release();
+    }
+    ioctx.snap_set_read(librados::SNAP_HEAD);
+  }
+
+  // cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+  ioctx.selfmanaged_snap_remove(my_snaps[1]);
+}
+
+TEST_F(LibRadosTwoPoolsPP, TryFlush) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // verify dirty
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // pin
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush the pinned object with -EPERM
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EPERM, completion->get_return_value());
+    completion->release();
+  }
+
+  // unpin
+  {
+    ObjectWriteOperation op;
+    op.cache_unpin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // verify in base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it != ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // evict it
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, Flush) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  uint64_t user_version = 0;
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // verify dirty
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+    user_version = cache_ioctx.get_last_version();
+  }
+
+  // pin
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush the pinned object with -EPERM
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EPERM, completion->get_return_value());
+    completion->release();
+  }
+
+  // unpin
+  {
+    ObjectWriteOperation op;
+    op.cache_unpin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // verify in base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it != ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // evict it
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // read it again and verify the version is consistent
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ(user_version, cache_ioctx.get_last_version());
+  }
+
+  // erase it
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush whiteout
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+  // or base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("a");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("b");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // and another
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("c");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // flush on head (should fail)
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+  // flush on recent snap (should fail)
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+  // flush on oldest snap
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush on next oldest snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush on head
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify i can read the snaps from the cache pool
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('b', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('a', bl[0]);
+  }
+
+  // remove overlay
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // verify i can read the snaps from the base pool
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('b', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('a', bl[0]);
+  }
+
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+}
+
+TEST_F(LibRadosTierPP, FlushWriteRaces) {
+  SKIP_IF_CRIMSON();
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  std::string cache_pool_name = pool_name + "-cache";
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+  IoCtx cache_ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+  cache_ioctx.application_enable("rados", true);
+  IoCtx ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  bufferlist bl;
+  bl.append("hi there");
+  {
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + write
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectWriteOperation op2;
+    op2.write_full(bl);
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion2, &op2, 0));
+
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  int tries = 1000;
+  do {
+    // create/dirty object
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, ioctx.operate("foo", &op));
+    }
+
+    // try-flush + write
+    {
+      ObjectReadOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY |
+	librados::OPERATION_SKIPRWLOCKS, NULL));
+
+      ObjectWriteOperation op2;
+      op2.write_full(bl);
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
+
+      completion->wait_for_complete();
+      completion2->wait_for_complete();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+	break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+
+  ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + try-flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_try_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  // create/dirty object
+  int tries = 1000;
+  do {
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, ioctx.operate("foo", &op));
+    }
+
+    // try-flush + flush
+    //  (flush will not piggyback on try-flush)
+    {
+      ObjectReadOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY |
+	librados::OPERATION_SKIPRWLOCKS, NULL));
+
+      ObjectReadOperation op2;
+      op2.cache_flush();
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion2, &op2,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+      completion->wait_for_complete();
+      completion2->wait_for_complete();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+	break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // try-flush + try-flush
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_try_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+}
+
+
+IoCtx *read_ioctx = 0;
+ceph::mutex test_lock = ceph::make_mutex("FlushReadRaces::lock");
+ceph::condition_variable cond;
+int max_reads = 100;
+int num_reads = 0; // in progress
+
+void flush_read_race_cb(completion_t cb, void *arg);
+
+void start_flush_read()
+{
+  //cout << " starting read" << std::endl;
+  ObjectReadOperation op;
+  op.stat(NULL, NULL, NULL);
+  librados::AioCompletion *completion = completions.getCompletion();
+  completion->set_complete_callback(0, flush_read_race_cb);
+  read_ioctx->aio_operate("foo", completion, &op, NULL);
+}
+
+void flush_read_race_cb(completion_t cb, void *arg)
+{
+  //cout << " finished read" << std::endl;
+  std::lock_guard l{test_lock};
+  if (num_reads > max_reads) {
+    num_reads--;
+    cond.notify_all();
+  } else {
+    start_flush_read();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    bufferptr bp(4000000);  // make it big!
+    bp.zero();
+    bl.append(bp);
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // start a continuous stream of reads
+  read_ioctx = &ioctx;
+  test_lock.lock();
+  for (int i = 0; i < max_reads; ++i) {
+    start_flush_read();
+    num_reads++;
+  }
+  test_lock.unlock();
+
+  // try-flush
+  ObjectReadOperation op;
+  op.cache_try_flush();
+  librados::AioCompletion *completion = cluster.aio_create_completion();
+  ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+  completion->wait_for_complete();
+  ASSERT_EQ(0, completion->get_return_value());
+  completion->release();
+
+  // stop reads
+  std::unique_lock locker{test_lock};
+  max_reads = 0;  
+  cond.wait(locker, [] { return num_reads == 0;});
+}
+
+TEST_F(LibRadosTierPP, HitSetNone) {
+  SKIP_IF_CRIMSON();
+  {
+    list< pair<time_t,time_t> > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
+    c->wait_for_complete();
+    ASSERT_EQ(0, c->get_return_value());
+    ASSERT_TRUE(ls.empty());
+    c->release();
+  }
+  {
+    bufferlist bl;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
+    c->wait_for_complete();
+    ASSERT_EQ(-ENOENT, c->get_return_value());
+    c->release();
+  }
+}
+
+string set_pool_str(string pool, string var, string val)
+{
+  return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
+    + string("\",\"var\": \"") + var + string("\",\"val\": \"")
+    + val + string("\"}");
+}
+
+string set_pool_str(string pool, string var, int val)
+{
+  return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
+    + string("\",\"var\": \"") + var + string("\",\"val\": \"")
+    + stringify(val) + string("\"}");
+}
+
+TEST_F(LibRadosTwoPoolsPP, HitSetRead) {
+  SKIP_IF_CRIMSON();
+  // make it a tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
+						"explicit_object"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  cache_ioctx.set_namespace("");
+
+  // keep reading until we see our object appear in the HitSet
+  utime_t start = ceph_clock_now();
+  utime_t hard_stop = start + utime_t(600, 0);
+
+  while (true) {
+    utime_t now = ceph_clock_now();
+    ASSERT_TRUE(now < hard_stop);
+
+    string name = "foo";
+    uint32_t hash; 
+    ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
+    hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
+		  cluster.pool_lookup(cache_pool_name.c_str()), "");
+
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
+
+    bufferlist hbl;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
+    c->wait_for_complete();
+    c->release();
+
+    if (hbl.length()) {
+      auto p = hbl.cbegin();
+      HitSet hs;
+      decode(hs, p);
+      if (hs.contains(oid)) {
+	cout << "ok, hit_set contains " << oid << std::endl;
+	break;
+      }
+      cout << "hmm, not in HitSet yet" << std::endl;
+    } else {
+      cout << "hmm, no HitSet yet" << std::endl;
+    }
+
+    sleep(1);
+  }
+}
+
+static int _get_pg_num(Rados& cluster, string pool_name)
+{
+  bufferlist inbl;
+  string cmd = string("{\"prefix\": \"osd pool get\",\"pool\":\"")
+    + pool_name
+    + string("\",\"var\": \"pg_num\",\"format\": \"json\"}");
+  bufferlist outbl;
+  int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
+  ceph_assert(r >= 0);
+  string outstr(outbl.c_str(), outbl.length());
+  json_spirit::Value v;
+  if (!json_spirit::read(outstr, v)) {
+    cerr <<" unable to parse json " << outstr << std::endl;
+    return -1;
+  }
+
+  json_spirit::Object& o = v.get_obj();
+  for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
+    json_spirit::Pair& p = o[i];
+    if (p.name_ == "pg_num") {
+      cout << "pg_num = " << p.value_.get_int() << std::endl;
+      return p.value_.get_int();
+    }
+  }
+  cerr << "didn't find pg_num in " << outstr << std::endl;
+  return -1;
+}
+
+int make_hitset(Rados& cluster, librados::IoCtx& cache_ioctx, int num_pg, 
+    int num, std::map<int, HitSet>& hitsets, std::string& cache_pool_name) 
+{
+  int pg = num_pg;
+  // do a bunch of writes
+  for (int i=0; i<num; ++i) {
+    bufferlist bl;
+    bl.append("a");
+    ceph_assert(0 == cache_ioctx.write(stringify(i), bl, 1, 0));
+  }
+
+  // get HitSets
+  for (int i=0; i<pg; ++i) {
+    list< pair<time_t,time_t> > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ceph_assert(0 == cache_ioctx.hit_set_list(i, c, &ls));
+    c->wait_for_complete();
+    c->release();
+    std::cout << "pg " << i << " ls " << ls << std::endl;
+    ceph_assert(!ls.empty());
+
+    // get the latest
+    c = librados::Rados::aio_create_completion();
+    bufferlist bl;
+    ceph_assert(0 == cache_ioctx.hit_set_get(i, c, ls.back().first, &bl));
+    c->wait_for_complete();
+    c->release();
+
+    try {
+      auto p = bl.cbegin();
+      decode(hitsets[i], p);
+    }
+    catch (buffer::error& e) {
+      std::cout << "failed to decode hit set; bl len is " << bl.length() << "\n";
+      bl.hexdump(std::cout);
+      std::cout << std::endl;
+      throw e;
+    }
+
+    // cope with racing splits by refreshing pg_num
+    if (i == pg - 1)
+      pg = _get_pg_num(cluster, cache_pool_name);
+  }
+  return pg;
+}
+
+TEST_F(LibRadosTwoPoolsPP, HitSetWrite) {
+  SKIP_IF_CRIMSON();
+  int num_pg = _get_pg_num(cluster, pool_name);
+  ceph_assert(num_pg > 0);
+
+  // make it a tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 8),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
+						"explicit_hash"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  cache_ioctx.set_namespace("");
+
+  int num = 200;
+
+  std::map<int,HitSet> hitsets;
+
+  num_pg = make_hitset(cluster, cache_ioctx, num_pg, num, hitsets, cache_pool_name);
+
+  int retry = 0;
+
+  for (int i=0; i<num; ++i) {
+    string n = stringify(i);
+    uint32_t hash;
+    ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(n, &hash));
+    hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
+		  cluster.pool_lookup(cache_pool_name.c_str()), "");
+    std::cout << "checking for " << oid << std::endl;
+    bool found = false;
+    for (int p=0; p<num_pg; ++p) {
+      if (hitsets[p].contains(oid)) {
+	found = true;
+	break;
+      }
+    }
+    if (!found && retry < 5) {
+      num_pg = make_hitset(cluster, cache_ioctx, num_pg, num, hitsets, cache_pool_name);
+      i--;
+      retry++;
+      continue;
+    }
+    ASSERT_TRUE(found);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, HitSetTrim) {
+  SKIP_IF_CRIMSON();
+  unsigned count = 3;
+  unsigned period = 3;
+
+  // make it a tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
+				   inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  cache_ioctx.set_namespace("");
+
+  // do a bunch of writes and make sure the hitsets rotate
+  utime_t start = ceph_clock_now();
+  utime_t hard_stop = start + utime_t(count * period * 50, 0);
+
+  time_t first = 0;
+  while (true) {
+    string name = "foo";
+    uint32_t hash; 
+    ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
+    hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
+
+    bufferlist bl;
+    bl.append("f");
+    ASSERT_EQ(0, cache_ioctx.write("foo", bl, 1, 0));
+
+    list<pair<time_t, time_t> > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
+    c->wait_for_complete();
+    c->release();
+
+    cout << " got ls " << ls << std::endl;
+    if (!ls.empty()) {
+      if (!first) {
+	first = ls.front().first;
+	cout << "first is " << first << std::endl;
+      } else {
+	if (ls.front().first != first) {
+	  cout << "first now " << ls.front().first << ", trimmed" << std::endl;
+	  break;
+	}
+      }
+    }
+
+    utime_t now = ceph_clock_now();
+    ASSERT_TRUE(now < hard_stop);
+
+    sleep(1);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, PromoteOn2ndRead) {
+  SKIP_IF_CRIMSON();
+  // create object
+  for (int i=0; i<20; ++i) {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_count", 2),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_period", 600),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  int fake = 0;  // set this to non-zero to test spurious promotion,
+		 // e.g. from thrashing
+  int attempt = 0;
+  string obj;
+  while (true) {
+    // 1st read, don't trigger a promote
+    obj = "foo" + stringify(attempt);
+    cout << obj << std::endl;
+    {
+      bufferlist bl;
+      ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
+      if (--fake >= 0) {
+	sleep(1);
+	ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
+	sleep(1);
+      }
+    }
+
+    // verify the object is NOT present in the cache tier
+    {
+      bool found = false;
+      NObjectIterator it = cache_ioctx.nobjects_begin();
+      while (it != cache_ioctx.nobjects_end()) {
+	cout << " see " << it->get_oid() << std::endl;
+	if (it->get_oid() == string(obj.c_str())) {
+	  found = true;
+	  break;
+	}
+	++it;
+      }
+      if (!found)
+	break;
+    }
+
+    ++attempt;
+    ASSERT_LE(attempt, 20);
+    cout << "hrm, object is present in cache on attempt " << attempt
+	 << ", retrying" << std::endl;
+  }
+
+  // Read until the object is present in the cache tier
+  cout << "verifying " << obj << " is eventually promoted" << std::endl;
+  while (true) {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
+
+    bool there = false;
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    while (it != cache_ioctx.nobjects_end()) {
+      if (it->get_oid() == string(obj.c_str())) {
+	there = true;
+	break;
+      }
+      ++it;
+    }
+    if (there)
+      break;
+
+    sleep(1);
+  }
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsPP, ProxyRead) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"readproxy\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read and verify the object
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // Verify 10 times the object is NOT present in the cache tier
+  uint32_t i = 0;
+  while (i++ < 10) {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+    sleep(1);
+  }
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsPP, CachePin) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+  }
+
+  // verify the objects are present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    for (uint32_t i = 0; i < 4; i++) {
+      ASSERT_TRUE(it->get_oid() == string("foo") ||
+                  it->get_oid() == string("bar") ||
+                  it->get_oid() == string("baz") ||
+                  it->get_oid() == string("bam"));
+      ++it;
+    }
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // pin objects
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // enable agent
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_count", 2),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_period", 600),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "target_max_objects", 1),
+    inbl, NULL, NULL));
+
+  sleep(10);
+
+  // Verify the pinned object 'foo' is not flushed/evicted
+  uint32_t count = 0;
+  while (true) {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
+
+    count = 0;
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    while (it != cache_ioctx.nobjects_end()) {
+      ASSERT_TRUE(it->get_oid() == string("foo") ||
+                  it->get_oid() == string("bar") ||
+                  it->get_oid() == string("baz") ||
+                  it->get_oid() == string("bam"));
+      ++count;
+      ++it;
+    }
+    if (count == 2) {
+      ASSERT_TRUE(it->get_oid() == string("foo") ||
+                  it->get_oid() == string("baz"));
+      break;
+    }
+
+    sleep(1);
+  }
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsPP, SetRedirectRead) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  {
+    ObjectWriteOperation op;
+    op.set_redirect("bar", cache_ioctx, 0);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // read and verify the object
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('t', bl[0]);
+  }
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestPromoteRead) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet mimic
+  if (_get_required_osd_release(cluster) < "mimic") {
+    GTEST_SKIP() << "cluster is not yet mimic, skipping test";
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("base chunk");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("CHUNK");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // set-redirect
+  {
+    ObjectWriteOperation op;
+    op.set_redirect("bar", cache_ioctx, 0);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // set-chunk
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, "bar-chunk", "foo-chunk");
+
+  // promote
+  {
+    ObjectWriteOperation op;
+    op.tier_promote();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // read and verify the object (redirect)
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('t', bl[0]);
+  }
+  // promote
+  {
+    ObjectWriteOperation op;
+    op.tier_promote();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // read and verify the object
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo-chunk", bl, 1, 0));
+    ASSERT_EQ('C', bl[0]);
+  }
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestRefRead) {
+  SKIP_IF_CRIMSON();
+  // note: require >= mimic
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("base chunk");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("CHUNK");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // set-redirect
+  {
+    ObjectWriteOperation op;
+    op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // set-chunk
+  {
+    ObjectReadOperation op;
+    op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op,
+	      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // redirect's refcount 
+  {
+    bufferlist t;
+    cache_ioctx.getxattr("bar", CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_EQ(1U, refs.count());
+  }
+  // chunk's refcount 
+  {
+    bufferlist t;
+    cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_EQ(1u, refs.count());
+  }
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet nautilus
+  if (_get_required_osd_release(cluster) < "nautilus") {
+    GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("base chunk");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("CHUNK");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // set-redirect
+  {
+    ObjectWriteOperation op;
+    op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // set-chunk
+  {
+    ObjectReadOperation op;
+    op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op,
+	      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // redirect's refcount 
+  {
+    bufferlist t;
+    cache_ioctx.getxattr("bar", CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_EQ(1u, refs.count());
+  }
+  // chunk's refcount 
+  {
+    bufferlist t;
+    cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_EQ(1u, refs.count());
+  }
+
+  // unset-manifest for set-redirect
+  {
+    ObjectWriteOperation op;
+    op.unset_manifest();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // unset-manifest for set-chunk
+  {
+    ObjectWriteOperation op;
+    op.unset_manifest();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // redirect's refcount 
+  {
+    bufferlist t;
+    cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
+    if (t.length() != 0U) {
+      ObjectWriteOperation op;
+      op.unset_manifest();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+      completion->wait_for_complete();
+      ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
+      completion->release();
+    }
+  }
+  // chunk's refcount 
+  {
+    bufferlist t;
+    cache_ioctx.getxattr("bar-chunk", CHUNK_REFCOUNT_ATTR, t);
+    if (t.length() != 0U) {
+      ObjectWriteOperation op;
+      op.unset_manifest();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
+      completion->wait_for_complete();
+      ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
+      completion->release();
+    }
+  }
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestDedupRefRead) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet nautilus
+  if (_get_required_osd_release(cluster) < "nautilus") {
+    GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
+  }
+
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+	    set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
+	    inbl, NULL, NULL));
+  cluster.wait_for_latest_osdmap();
+  string tgt_oid;
+
+  // get fp_oid
+  tgt_oid = get_fp_oid("There hi", "sha1");
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("There hi");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("There hi");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo-dedup", &op));
+  }
+
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("There hi");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(tgt_oid, &op));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 8, tgt_oid, "foo-dedup");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 8, tgt_oid, "foo");
+  // chunk's refcount 
+  {
+    bufferlist t;
+    cache_ioctx.getxattr(tgt_oid, CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_LE(2u, refs.count());
+  }
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestSnapRefcount) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
+	inbl, NULL, NULL));
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("there hi");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hi");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  string er_fp_oid, hi_fp_oid, bb_fp_oid;
+
+  // get fp_oid
+  er_fp_oid = get_fp_oid("er", "sha1");
+  hi_fp_oid = get_fp_oid("hi", "sha1");
+  bb_fp_oid = get_fp_oid("bb", "sha1");
+
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("er");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(er_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("hi");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(hi_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("bb");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(bb_fp_oid, &op));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, hi_fp_oid, "foo");
+
+  // make all chunks dirty --> flush
+  // foo: [er] [hi]
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("er");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"er", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_LE(1u, refs.count());
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // foo: [bb] [hi]
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("Thbbe");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // make clean
+  {
+    bufferlist bl;
+    bl.append("Thbbe");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, bb_fp_oid, "foo");
+
+  // and another
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // foo: [er] [hi]
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("There");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // make clean
+  {
+    bufferlist bl;
+    bl.append("There");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("er");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"er", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_LE(2u, refs.count());
+  }
+  
+  // and another
+  my_snaps.resize(3);
+  my_snaps[2] = my_snaps[1];
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // foo: [bb] [hi]
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("Thbbe");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // make clean
+  {
+    bufferlist bl;
+    bl.append("Thbbe");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, bb_fp_oid, "foo");
+
+  /*
+   *  snap[2]: [er] [hi]
+   *  snap[1]: [bb] [hi]
+   *  snap[0]: [er] [hi]
+   *  head:    [bb] [hi]
+   */
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("hi");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"hi", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
+  }
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("er");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"er", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_LE(2u, refs.count());
+  }
+
+  // remove snap
+  ioctx.selfmanaged_snap_remove(my_snaps[2]);
+
+  /*
+   *  snap[1]: [bb] [hi]
+   *  snap[0]: [er] [hi]
+   *  head:    [bb] [hi]
+   */
+
+  sleep(10);
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("hi");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"hi", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
+  }
+
+  // remove snap
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+
+  /*
+   *  snap[1]: [bb] [hi]
+   *  head:    [bb] [hi]
+   */
+
+  sleep(10);
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("bb");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"bb", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
+  }
+
+  // remove snap
+  ioctx.selfmanaged_snap_remove(my_snaps[1]);
+
+  /*
+   *  snap[1]: [bb] [hi]
+   */
+
+  sleep(10);
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("bb");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"bb", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
+  }
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("hi");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"hi", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 1);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestSnapRefcount2) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
+	inbl, NULL, NULL));
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("Thabe cdHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
+  }
+
+  string ab_fp_oid, cd_fp_oid, ef_fp_oid, BB_fp_oid;
+
+  // get fp_oid
+  ab_fp_oid = get_fp_oid("ab", "sha1");
+  cd_fp_oid = get_fp_oid("cd", "sha1");
+  ef_fp_oid = get_fp_oid("ef", "sha1");
+  BB_fp_oid = get_fp_oid("BB", "sha1");
+
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("ab");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(ab_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("cd");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(cd_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("ef");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(ef_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("BB");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(BB_fp_oid, &op));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, ab_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, cd_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, ef_fp_oid, "foo");
+
+
+  // make all chunks dirty --> flush
+  // foo: [ab] [cd] [ef]
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // foo: [BB] [BB] [ef]
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("ThBBe BB");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // make clean
+  {
+    bufferlist bl;
+    bl.append("ThBBe BB");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, BB_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, BB_fp_oid, "foo");
+
+  // and another
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // foo: [ab] [cd] [ef]
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("Thabe cd");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // make clean
+  {
+    bufferlist bl;
+    bl.append("Thabe cd");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, ab_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, cd_fp_oid, "foo");
+
+  /*
+   *  snap[1]: [ab] [cd] [ef]
+   *  snap[0]: [BB] [BB] [ef]
+   *  head:    [ab] [cd] [ef]
+   */
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("ab");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"ab", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_LE(2u, refs.count());
+  }
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("cd");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"cd", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_LE(2u, refs.count());
+  }
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("BB");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"BB", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    cache_ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_LE(2u, refs.count());
+  }
+
+  // remove snap
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+
+  /*
+   *  snap[1]: [ab] [cd] [ef]
+   *  head:    [ab] [cd] [ef]
+   */
+
+  sleep(10);
+
+  // check chunk's refcount
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("BB");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"BB", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 0);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestTestSnapCreate) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    GTEST_SKIP() << "cluster is not yet octopus, skipping test";
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("base chunk");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("CHUNKS CHUNKS");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
+  }
+
+  string ba_fp_oid, se_fp_oid, ch_fp_oid;
+
+  // get fp_oid
+  ba_fp_oid = get_fp_oid("ba", "sha1");
+  se_fp_oid = get_fp_oid("se", "sha1");
+  ch_fp_oid = get_fp_oid("ch", "sha1");
+
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("ba");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(ba_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("se");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(se_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("ch");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(ch_fp_oid, &op));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, ba_fp_oid, "foo");
+
+  // try to create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, se_fp_oid, "foo");
+
+  // check whether clone is created
+  ioctx.snap_set_read(librados::SNAP_DIR);
+  {
+    snap_set_t snap_set;
+    int snap_ret;
+    ObjectReadOperation op;
+    op.list_snaps(&snap_set, &snap_ret);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      0, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, snap_ret);
+    ASSERT_LT(0u, snap_set.clones.size());
+    ASSERT_EQ(1, snap_set.clones.size());
+  }
+
+  // create a clone
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    bl.append("B");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 0));
+  }
+
+  ioctx.snap_set_read(my_snaps[0]);
+  // set-chunk to clone
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, ch_fp_oid, "foo");
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestRedirectAfterPromote) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus 
+  if (_get_required_osd_release(cluster) < "octopus") {
+    GTEST_SKIP() << "cluster is not yet octopus, skipping test";
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("base chunk");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("BASE CHUNK");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
+  }
+
+  // set-redirect
+  {
+    ObjectWriteOperation op;
+    op.set_redirect("bar", cache_ioctx, 0);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // promote
+  {
+    ObjectWriteOperation op;
+    op.tier_promote();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // write
+  {
+    bufferlist bl;
+    bl.append("a");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 0));
+  }
+
+  // read and verify the object (redirect)
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('a', bl[0]);
+  }
+
+  // read and verify the object (redirect)
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('B', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestCheckRefcountWhenModification) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus 
+  if (_get_required_osd_release(cluster) < "octopus") {
+    GTEST_SKIP() << "cluster is not yet octopus, skipping test";
+  }
+
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
+	inbl, NULL, NULL));
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  string er_fp_oid, hi_fp_oid, HI_fp_oid, ai_fp_oid, bi_fp_oid,
+	  Er_fp_oid, Hi_fp_oid, Si_fp_oid;
+
+  // get fp_oid
+  er_fp_oid = get_fp_oid("er", "sha1");
+  hi_fp_oid = get_fp_oid("hi", "sha1");
+  HI_fp_oid = get_fp_oid("HI", "sha1");
+  ai_fp_oid = get_fp_oid("ai", "sha1");
+  bi_fp_oid = get_fp_oid("bi", "sha1");
+  Er_fp_oid = get_fp_oid("Er", "sha1");
+  Hi_fp_oid = get_fp_oid("Hi", "sha1");
+  Si_fp_oid = get_fp_oid("Si", "sha1");
+
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("er");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(er_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("hi");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(hi_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("HI");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(HI_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("ai");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(ai_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("bi");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(bi_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("Er");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(Er_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("Hi");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(Hi_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("Si");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(Si_fp_oid, &op));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, hi_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, HI_fp_oid, "foo");
+
+  // foo head: [er] [hi] [HI]
+  
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+
+  // foo snap[0]: [er] [hi] [HI]
+  // foo head   : [er] [ai] [HI]
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("a");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
+  }
+  // write
+  {
+    bufferlist bl;
+    bl.append("a");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, ai_fp_oid, "foo");
+
+  // foo snap[0]: [er] [hi] [HI]
+  // foo head   : [er] [bi] [HI]
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("b");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
+  }
+  // write
+  {
+    bufferlist bl;
+    bl.append("b");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, bi_fp_oid, "foo");
+  
+  sleep(10);
+
+  // check chunk's refcount
+  // [ai]'s refcount should be 0
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("ai");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"ai", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 0);
+  }
+
+  // foo snap[0]: [er] [hi] [HI]
+  // foo head   : [Er] [Hi] [Si]
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("thEre HiSi");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  // write
+  {
+    bufferlist bl;
+    bl.append("thEre HiSi");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, Er_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, Hi_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, Si_fp_oid, "foo");
+
+  // foo snap[0]: [er] [hi] [HI]
+  // foo head   : [ER] [HI] [SI]
+  // write
+  {
+    bufferlist bl;
+    bl.append("thERe HISI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  sleep(10);
+
+  // check chunk's refcount
+  // [Er]'s refcount should be 0
+  {
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = strlen("Er");
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    sha1_gen.Update((const unsigned char *)"Er", size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    is_intended_refcount_state(ioctx, "foo", cache_ioctx, p_str, 0);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestSnapIncCount) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk4", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
+  // foo snap[1]: 
+  // foo snap[0]:  
+  // foo head   : [chunk1]          [chunk4]
+
+  ioctx.snap_set_read(my_snaps[1]);
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk2", "foo");
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
+  // foo snap[1]:          [chunk2] [chunk4]
+  // foo snap[0]:      
+  // foo head   : [chunk1]          [chunk4]
+ 
+  ioctx.snap_set_read(my_snaps[0]);
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk2", "foo");
+  // foo snap[1]:          [chunk2] [chunk4]
+  // foo snap[0]:          [chunk2]
+  // foo head   : [chunk1]          [chunk4]
+  
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk3", "foo");
+  // foo snap[1]:          [chunk2] [chunk4]
+  // foo snap[0]: [chunk3] [chunk2]
+  // foo head   : [chunk1]          [chunk4]
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
+  // foo snap[1]:          [chunk2] [chunk4]
+  // foo snap[0]: [chunk3] [chunk2] [chunk4]
+  // foo head   : [chunk1]          [chunk4]
+
+  // check chunk's refcount
+  check_fp_oid_refcount(cache_ioctx, "chunk1", 1u, "");
+
+  // check chunk's refcount
+  check_fp_oid_refcount(cache_ioctx, "chunk2", 1u, "");
+
+  // check chunk's refcount
+  check_fp_oid_refcount(cache_ioctx, "chunk3", 1u, "");
+  sleep(10);
+
+  // check chunk's refcount
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk4", 1);
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestEvict) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk4", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
+  // foo snap[1]: 
+  // foo snap[0]:  
+  // foo head   :          [chunk1]                    [chunk4]
+
+  ioctx.snap_set_read(my_snaps[1]);
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk2", "foo");
+  // foo snap[1]: [                  chunk2                   ] 
+  // foo snap[0]:      
+  // foo head   :          [chunk1]                    [chunk4]
+ 
+  ioctx.snap_set_read(my_snaps[0]);
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk2", "foo");
+  // foo snap[1]: [                  chunk2                   ] 
+  // foo snap[0]:                             [chunk2]
+  // foo head   :           [chunk1]                   [chunk4]
+  
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk3", "foo");
+  // foo snap[1]: [                  chunk2                   ] 
+  // foo snap[0]:	    [chunk3]          [chunk2]
+  // foo head   :	    [chunk1]                   [chunk4]
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk4", "foo");
+  // foo snap[1]: [                  chunk2                   ] 
+  // foo snap[0]:	    [chunk3]          [chunk2] [chunk4]
+  // foo head   :	    [chunk1]                   [chunk4]
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, "chunk4", "foo");
+  // foo snap[1]: [                  chunk2                   ] 
+  // foo snap[0]: [chunk4] [chunk3]           [chunk2] [chunk4]
+  // foo head   :          [chunk1]                    [chunk4]
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 4, 2, "chunk1", "foo");
+  // foo snap[1]: [                  chunk2                   ] 
+  // foo snap[0]: [chunk4] [chunk3] [chunk1]  [chunk2] [chunk4]
+  // foo head   :          [chunk1]                    [chunk4]
+
+  {
+    ObjectReadOperation op, stat_op;
+    uint64_t size;
+    op.tier_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+	"foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+
+    stat_op.stat(&size, NULL, NULL);
+    ASSERT_EQ(0, ioctx.operate("foo", &stat_op, NULL));
+    ASSERT_EQ(10, size);
+  }
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op, stat_op;
+    uint64_t size;
+    op.tier_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+	"foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+
+    stat_op.stat(&size, NULL, NULL);
+    ASSERT_EQ(0, ioctx.operate("foo", &stat_op, NULL));
+    ASSERT_EQ(strlen("there hiHI"), size);
+  }
+
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestEvictPromote) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("EREHT hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("THERE HIHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("there");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 2, "chunk1", "foo");
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk2", "foo");
+  // foo snap[0]:  
+  // foo head   :  [chunk1]                           [chunk2]
+
+  ioctx.snap_set_read(my_snaps[0]);
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk3", "foo");
+  // foo snap[0]: [                  chunk3                   ] 
+  // foo head   : [chunk1]                             [chunk2]
+ 
+
+  {
+    ObjectReadOperation op, stat_op;
+    uint64_t size;
+    op.tier_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+	"foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+
+    stat_op.stat(&size, NULL, NULL);
+    ASSERT_EQ(0, ioctx.operate("foo", &stat_op, NULL));
+    ASSERT_EQ(10, size);
+    
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('T', bl[0]);
+  }
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(10, ioctx.read("foo", bl, 10, 0));
+    ASSERT_EQ('H', bl[8]);
+  }
+}
+
+
+TEST_F(LibRadosTwoPoolsPP, ManifestSnapSizeMismatch) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("chunk1", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there HIHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("chunk2", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+       my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("There hiHI");
+    ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+       my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("tHere hiHI");
+    ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // set-chunk 
+  manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 10, "chunk1", "foo");
+
+  cache_ioctx.snap_set_read(my_snaps[1]);
+
+  // set-chunk 
+  manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 10, "chunk2", "foo");
+
+  // evict
+  {
+    ObjectReadOperation op, stat_op;
+    op.tier_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+       "foo", completion, &op,
+       librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+  }
+
+  uint32_t hash;
+  ASSERT_EQ(0, cache_ioctx.get_object_pg_hash_position2("foo", &hash));
+
+  // scrub
+  {
+    for (int tries = 0; tries < 5; ++tries) {
+      bufferlist inbl;
+      ostringstream ss;
+      ss << "{\"prefix\": \"pg deep-scrub\", \"pgid\": \""
+        << cache_ioctx.get_id() << "."
+        << std::hex << hash
+        << "\"}";
+      int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
+      if (r == -ENOENT ||  
+         r == -EAGAIN) {
+       sleep(5);
+       continue;
+      }
+      ASSERT_EQ(0, r);
+      break;
+    }
+    cout << "waiting for scrubs..." << std::endl;
+    sleep(20);
+    cout << "done waiting" << std::endl;
+  }
+
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('t', bl[0]);
+  }
+}
+
+#include <common/CDC.h>
+TEST_F(LibRadosTwoPoolsPP, DedupFlushRead) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    GTEST_SKIP() << "cluster is not yet octopus, skipping test";
+  }
+
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_tier", pool_name),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
+	inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  bufferlist gbl;
+  {
+    generate_buffer(1024*8, &gbl);
+    ObjectWriteOperation op;
+    op.write_full(gbl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo-chunk", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("DDse chunk");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar-chunk", &op));
+  }
+
+  // set-chunk to set manifest object
+  {
+    ObjectReadOperation op;
+    op.set_chunk(0, 2, ioctx, "bar-chunk", 0,
+		CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo-chunk", completion, &op,
+	      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo-chunk", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  std::unique_ptr<CDC> cdc = CDC::create("fastcdc", cbits(1024)-1);
+  vector<pair<uint64_t, uint64_t>> chunks;
+  bufferlist chunk;
+  cdc->calc_chunks(gbl, &chunks);
+  chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
+  string tgt_oid;
+  {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    SHA1 sha1_gen;
+    int size = chunk.length();
+    sha1_gen.Update((const unsigned char *)chunk.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    tgt_oid = string(p_str);
+  }
+
+  // read and verify the chunked object
+  {
+    bufferlist test_bl;
+    ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
+    ASSERT_EQ(test_bl[1], chunk[1]);
+  }
+
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 512),
+	inbl, NULL, NULL));
+  cluster.wait_for_latest_osdmap();
+
+  // make a dirty chunks
+  {
+    bufferlist bl;
+    bl.append("hi");
+    ASSERT_EQ(0, cache_ioctx.write("foo-chunk", bl, bl.length(), 0));
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo-chunk", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  cdc = CDC::create("fastcdc", cbits(512)-1);
+  chunks.clear();
+  cdc->calc_chunks(gbl, &chunks);
+  bufferlist chunk_512;
+  chunk_512.substr_of(gbl, chunks[3].first, chunks[3].second);
+  {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    SHA1 sha1_gen;
+    int size = chunk_512.length();
+    sha1_gen.Update((const unsigned char *)chunk_512.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    tgt_oid = string(p_str);
+  }
+
+  // read and verify the chunked object
+  {
+    bufferlist test_bl;
+    ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
+    ASSERT_EQ(test_bl[1], chunk_512[1]);
+  }
+
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 16384),
+	inbl, NULL, NULL));
+  cluster.wait_for_latest_osdmap();
+
+  // make a dirty chunks
+  {
+    bufferlist bl;
+    bl.append("hi");
+    ASSERT_EQ(0, cache_ioctx.write("foo-chunk", bl, bl.length(), 0));
+    gbl.begin(0).copy_in(bl.length(), bl);
+  }
+  // flush
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo-chunk", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  cdc = CDC::create("fastcdc", cbits(16384)-1);
+  chunks.clear();
+  cdc->calc_chunks(gbl, &chunks);
+  bufferlist chunk_16384;
+  chunk_16384.substr_of(gbl, chunks[0].first, chunks[0].second);
+  {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    SHA1 sha1_gen;
+    int size = chunk_16384.length();
+    sha1_gen.Update((const unsigned char *)chunk_16384.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    tgt_oid = string(p_str);
+  }
+  // read and verify the chunked object
+  {
+    bufferlist test_bl;
+    ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
+    ASSERT_EQ(test_bl[0], chunk_16384[0]);
+  }
+
+  // less than object size
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
+	inbl, NULL, NULL));
+  cluster.wait_for_latest_osdmap();
+
+  // make a dirty chunks
+  // a chunk_info is deleted by write, which converts the manifest object to non-manifest object
+  {
+    bufferlist bl;
+    bl.append("hi");
+    ASSERT_EQ(0, cache_ioctx.write("foo-chunk", bl, bl.length(), 0));
+  }
+
+  // reset set-chunk
+  {
+    bufferlist bl;
+    bl.append("DDse chunk");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar-chunk", &op));
+  }
+  // set-chunk to set manifest object
+  {
+    ObjectReadOperation op;
+    op.set_chunk(0, 2, ioctx, "bar-chunk", 0,
+		CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo-chunk", completion, &op,
+	      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo-chunk", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  cdc = CDC::create("fastcdc", cbits(1024)-1);
+  chunks.clear();
+  cdc->calc_chunks(gbl, &chunks);
+  bufferlist small_chunk;
+  small_chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
+  {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    SHA1 sha1_gen;
+    int size = small_chunk.length();
+    sha1_gen.Update((const unsigned char *)small_chunk.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    tgt_oid = string(p_str);
+  }
+  // read and verify the chunked object
+  {
+    bufferlist test_bl;
+    ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
+    ASSERT_EQ(test_bl[0], small_chunk[0]);
+  }
+
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestFlushSnap) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_tier", pool_name),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
+	inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  bufferlist gbl;
+  {
+    //bufferlist bl;
+    //bl.append("there hi");
+    generate_buffer(1024*8, &gbl);
+    ObjectWriteOperation op;
+    op.write_full(gbl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hi");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, ioctx, cache_ioctx, 2, 2, "bar", "foo");
+  manifest_set_chunk(cluster, ioctx, cache_ioctx, 6, 2, "bar", "foo");
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // make a dirty chunks
+  {
+    bufferlist bl;
+    bl.append("Thbbe");
+    ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // and another
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // make a dirty chunks
+  {
+    bufferlist bl;
+    bl.append("Thcce");
+    ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // flush on head (should fail)
+  cache_ioctx.snap_set_read(librados::SNAP_HEAD);
+  // flush
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush on recent snap (should fail)
+  cache_ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush on oldest snap
+  cache_ioctx.snap_set_read(my_snaps[1]);
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush on oldest snap
+  cache_ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush on oldest snap
+  cache_ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // check chunk's refcount
+  std::unique_ptr<CDC> cdc = CDC::create("fastcdc", cbits(1024)-1);
+  vector<pair<uint64_t, uint64_t>> chunks;
+  bufferlist chunk;
+  cdc->calc_chunks(gbl, &chunks);
+  chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
+  string tgt_oid;
+  {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    SHA1 sha1_gen;
+    int size = chunk.length();
+    sha1_gen.Update((const unsigned char *)chunk.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    tgt_oid = string(p_str);
+  }
+  // read and verify the chunked object
+  {
+    bufferlist test_bl;
+    ASSERT_EQ(2, ioctx.read(tgt_oid, test_bl, 2, 0));
+    ASSERT_EQ(test_bl[1], chunk[1]);
+  }
+
+  cache_ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(4, cache_ioctx.read("foo", bl, 4, 0));
+    ASSERT_EQ('c', bl[2]);
+  }
+
+  cache_ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(4, cache_ioctx.read("foo", bl, 4, 0));
+    ASSERT_EQ('b', bl[2]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestFlushDupCount) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_tier", pool_name),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
+	inbl, NULL, NULL));
+
+  // create object
+  bufferlist gbl;
+  {
+    //bufferlist bl;
+    generate_buffer(1024*8, &gbl);
+    ObjectWriteOperation op;
+    op.write_full(gbl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // set-chunk to set manifest object
+  {
+    ObjectReadOperation op;
+    op.set_chunk(0, 2, ioctx, "bar", 0,
+		CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+	      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // make a dirty chunks
+  {
+    bufferlist bl;
+    bl.append("Thbbe hi");
+    ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // and another
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, cache_ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // make a dirty chunks
+  {
+    bufferlist bl;
+    bl.append("Thcce hi");
+    ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  //flush on oldest snap
+  cache_ioctx.snap_set_read(my_snaps[1]);
+  // flush
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush on oldest snap
+  cache_ioctx.snap_set_read(my_snaps[0]);
+  // flush
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  cache_ioctx.snap_set_read(librados::SNAP_HEAD);
+  // flush
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  std::unique_ptr<CDC> cdc = CDC::create("fastcdc", cbits(1024)-1);
+  vector<pair<uint64_t, uint64_t>> chunks;
+  bufferlist chunk;
+  cdc->calc_chunks(gbl, &chunks);
+  chunk.substr_of(gbl, chunks[1].first, chunks[1].second);
+  string tgt_oid;
+  // check chunk's refcount
+  {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = chunk.length();
+    sha1_gen.Update((const unsigned char *)chunk.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    tgt_oid = string(p_str);
+    ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_LE(1u, refs.count());
+  }
+
+  bufferlist chunk2;
+  chunk2.substr_of(gbl, chunks[0].first, chunks[0].second);
+  // check chunk's refcount
+  {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = chunk2.length();
+    sha1_gen.Update((const unsigned char *)chunk2.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    tgt_oid = string(p_str);
+    ioctx.getxattr(p_str, CHUNK_REFCOUNT_ATTR, t);
+    chunk_refs_t refs;
+    try {
+      auto iter = t.cbegin();
+      decode(refs, iter);
+    } catch (buffer::error& err) {
+      ASSERT_TRUE(0);
+    }
+    ASSERT_LE(1u, refs.count());
+  }
+
+  // make a dirty chunks
+  {
+    bufferlist bl;
+    bl.append("ThDDe hi");
+    ASSERT_EQ(0, cache_ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  bufferlist tmp;
+  tmp.append("Thcce hi");
+  gbl.begin(0).copy_in(tmp.length(), tmp);
+  bufferlist chunk3;
+  cdc->calc_chunks(gbl, &chunks);
+  chunk3.substr_of(gbl, chunks[0].first, chunks[0].second);
+  // check chunk's refcount
+  {
+    unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1] = {0};
+    char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
+    bufferlist t;
+    SHA1 sha1_gen;
+    int size = chunk2.length();
+    sha1_gen.Update((const unsigned char *)chunk2.c_str(), size);
+    sha1_gen.Final(fingerprint);
+    buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
+    is_intended_refcount_state(cache_ioctx, "foo", ioctx, p_str, 0);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, TierFlushDuringFlush) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  bufferlist inbl;
+
+  // create a new pool 
+  std::string temp_pool_name = get_temp_pool_name() + "-test-flush";
+  ASSERT_EQ(0, cluster.pool_create(temp_pool_name.c_str()));
+
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_tier", temp_pool_name),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
+	inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
+	inbl, NULL, NULL));
+
+  // create object
+  bufferlist gbl;
+  {
+    //bufferlist bl;
+    generate_buffer(1024*8, &gbl);
+    ObjectWriteOperation op;
+    op.write_full(gbl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // set-chunk to set manifest object
+  {
+    ObjectReadOperation op;
+    op.set_chunk(0, 2, ioctx, "bar", 0,
+		CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+	      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // delete temp pool, so flushing chunk will fail
+  ASSERT_EQ(0, s_cluster.pool_delete(temp_pool_name.c_str()));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // flush to check if proper error is returned
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestSnapHasChunk) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+	set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
+	inbl, NULL, NULL));
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("there HIHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  string er_fp_oid, hi_fp_oid, HI_fp_oid, ai_fp_oid, bi_fp_oid,
+	  Er_fp_oid, Hi_fp_oid, SI_fp_oid;
+
+  // get fp_oid
+  er_fp_oid = get_fp_oid("er", "sha1");
+  hi_fp_oid = get_fp_oid("hi", "sha1");
+  HI_fp_oid = get_fp_oid("HI", "sha1");
+  ai_fp_oid = get_fp_oid("ai", "sha1");
+  bi_fp_oid = get_fp_oid("bi", "sha1");
+  Er_fp_oid = get_fp_oid("Er", "sha1");
+  Hi_fp_oid = get_fp_oid("Hi", "sha1");
+  SI_fp_oid = get_fp_oid("SI", "sha1");
+
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("er");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(er_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("hi");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(hi_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("HI");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(HI_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("ai");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(ai_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("bi");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(bi_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("Er");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(Er_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("Hi");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(Hi_fp_oid, &op));
+  }
+  // write
+  {
+    ObjectWriteOperation op;
+    bufferlist bl;
+    bl.append("SI");
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate(SI_fp_oid, &op));
+  }
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, HI_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, HI_fp_oid, "foo");
+
+  // foo head:     [hi] [HI]
+  
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("a");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
+  }
+  // write
+  {
+    bufferlist bl;
+    bl.append("a");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
+  }
+  // write
+  {
+    bufferlist bl;
+    bl.append("S");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 8));
+  }
+
+  // foo snap[0]:      [hi] [HI]
+  // foo head   : [er] [ai] [SI]
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, er_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, ai_fp_oid, "foo");
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, SI_fp_oid, "foo");
+
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  // create a clone
+  {
+    bufferlist bl;
+    bl.append("b");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
+  }
+  // write
+  {
+    bufferlist bl;
+    bl.append("b");
+    ASSERT_EQ(0, ioctx.write("foo", bl, 1, 6));
+  }
+
+  // foo snap[1]:      [HI] [HI]
+  // foo snap[0]: [er] [ai] [SI]
+  // foo head   : [er] [bi] [SI]
+
+  // set-chunk (dedup)
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, bi_fp_oid, "foo");
+
+  {
+    ASSERT_EQ(1, cls_cas_references_chunk(ioctx, "foo", SI_fp_oid));
+    ASSERT_EQ(1, cls_cas_references_chunk(ioctx, "foo", er_fp_oid));
+    ASSERT_EQ(1, cls_cas_references_chunk(ioctx, "foo", ai_fp_oid));
+    ASSERT_EQ(2, cls_cas_references_chunk(ioctx, "foo", HI_fp_oid));
+    ASSERT_EQ(-ENOLINK, cls_cas_references_chunk(ioctx, "foo", Hi_fp_oid));
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestRollback) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet pacific
+  if (_get_required_osd_release(cluster) < "pacific") {
+    cout << "cluster is not yet pacific, skipping test" << std::endl;
+    return;
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("CDere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ABere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("CDere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("EFere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("thABe hiEF");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk3", "foo");
+  // foo snap[1]: 
+  // foo snap[0]:  
+  // foo head   :          [chunk1]                    [chunk3]
+
+  ioctx.snap_set_read(my_snaps[1]);
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk2", "foo");
+  // foo snap[1]: [                  chunk2                   ] 
+  // foo snap[0]:      
+  // foo head   :          [chunk1]                    [chunk3]
+ 
+  // foo snap[1]: [                  chunk2                   ] 
+  // foo snap[0]:                             
+  // foo head   :           [chunk1]                   [chunk3]
+  
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_rollback("foo", my_snaps[0]));
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('t', bl[0]);
+  }
+
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_rollback("foo", my_snaps[1]));
+
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('C', bl[0]);
+  }
+
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestRollbackRefcount) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet pacific
+  if (_get_required_osd_release(cluster) < "pacific") {
+    cout << "cluster is not yet pacific, skipping test" << std::endl;
+    return;
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("CDere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ABere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("CDere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("EFere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("DDDDD hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk4", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("EEEEE hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk5", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("thABe hiEF");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk3", "foo");
+  // foo snap[1]: 
+  // foo snap[0]:  
+  // foo head   :          [chunk1]                    [chunk3]
+
+  ioctx.snap_set_read(my_snaps[1]);
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk4", "foo");
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 6, 2, "chunk5", "foo");
+  // foo snap[1]:           [chunk4]          [chunk5]       
+  // foo snap[0]:  
+  // foo head   :           [chunk1]                   [chunk3]
+  
+  ioctx.snap_set_read(my_snaps[0]);
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk2", "foo");
+  // foo snap[1]:           [chunk4]          [chunk5]       
+  // foo snap[0]: [                  chunk2                   ] 
+  // foo head   :          [chunk1]                    [chunk3]
+  
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_rollback("foo", my_snaps[1]));
+  // foo snap[1]:          [chunk4]          [chunk5]       
+  // foo snap[0]: [                  chunk2                   ] 
+  // foo head   :          [chunk4]          [chunk5] <-- will contain these contents
+
+  sleep(10);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk1", 0);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk3", 0);
+
+  ioctx.selfmanaged_snap_remove(my_snaps[1]);
+  sleep(10);
+  // foo snap[1]:          
+  // foo snap[0]: [                  chunk2                   ] 
+  // foo head   :          [chunk4]          [chunk5] 
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk4", 1);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk5", 1);
+
+  {
+    bufferlist bl;
+    bl.append("thABe hiEF");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+  // foo snap[1]:          
+  // foo snap[0]: [                  chunk2                   ] 
+  // foo head   :          
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk1", 0);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk3", 0);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk4", 0);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk5", 0);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk2", 1);
+}
+
+TEST_F(LibRadosTwoPoolsPP, ManifestEvictRollback) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet pacific
+  if (_get_required_osd_release(cluster) < "pacific") {
+    cout << "cluster is not yet pacific, skipping test" << std::endl;
+    return;
+  }
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("CDere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ABere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk1", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("CDere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk2", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("EFere hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("chunk3", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+	my_snaps));
+
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ASSERT_EQ(0, ioctx.write("foo", bl, bl.length(), 0));
+  }
+
+
+  // set-chunk 
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 2, 2, "chunk1", "foo");
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 8, 2, "chunk3", "foo");
+  // foo snap[0]:  
+  // foo head   :          [chunk1]                    [chunk3]
+
+  ioctx.snap_set_read(my_snaps[0]);
+  manifest_set_chunk(cluster, cache_ioctx, ioctx, 0, 10, "chunk2", "foo");
+  // foo snap[0]: [                  chunk2                   ] 
+  // foo head   :          [chunk1]                    [chunk3]
+  
+  sleep(10);
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk1", 1);
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk3", 1);
+
+
+  ioctx.snap_set_read(my_snaps[0]);
+  // evict--this makes the chunk missing state
+  {
+    ObjectReadOperation op, stat_op;
+    op.tier_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+	"foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+  }
+
+  // rollback to my_snaps[0]
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_rollback("foo", my_snaps[0]));
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('C', bl[0]);
+  }
+
+  is_intended_refcount_state(ioctx, "foo", cache_ioctx, "chunk2", 1);
+}
+
+class LibRadosTwoPoolsECPP : public RadosTestECPP
+{
+public:
+  LibRadosTwoPoolsECPP() {};
+  ~LibRadosTwoPoolsECPP() override {};
+protected:
+  static void SetUpTestCase() {
+    SKIP_IF_CRIMSON();
+    pool_name = get_temp_pool_name();
+    ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
+  }
+  static void TearDownTestCase() {
+    SKIP_IF_CRIMSON();
+    ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
+  }
+  static std::string cache_pool_name;
+
+  void SetUp() override {
+    SKIP_IF_CRIMSON();
+    cache_pool_name = get_temp_pool_name();
+    ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
+    RadosTestECPP::SetUp();
+
+    ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+    cache_ioctx.application_enable("rados", true);
+    cache_ioctx.set_namespace(nspace);
+  }
+  void TearDown() override {
+    SKIP_IF_CRIMSON();
+    // flush + evict cache
+    flush_evict_all(cluster, cache_ioctx);
+
+    bufferlist inbl;
+    // tear down tiers
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+      "\"}",
+      inbl, NULL, NULL));
+    ASSERT_EQ(0, cluster.mon_command(
+      "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+      "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+    // wait for maps to settle before next test
+    cluster.wait_for_latest_osdmap();
+
+    RadosTestECPP::TearDown();
+
+    cleanup_default_namespace(cache_ioctx);
+    cleanup_namespace(cache_ioctx, nspace);
+
+    cache_ioctx.close();
+    ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
+  }
+
+  librados::IoCtx cache_ioctx;
+};
+
+std::string LibRadosTwoPoolsECPP::cache_pool_name;
+
+TEST_F(LibRadosTierECPP, Dirty) {
+  SKIP_IF_CRIMSON();
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
+  }
+  {
+    ObjectWriteOperation op;
+    op.create(true);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.undirty();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));  // still 0 if already clean
+  }
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+  //{
+  //  ObjectWriteOperation op;
+  //  op.truncate(0);  // still a write even tho it is a no-op
+  //  ASSERT_EQ(0, ioctx.operate("foo", &op));
+  //}
+  //{
+  //  bool dirty = false;
+  //  int r = -1;
+  //  ObjectReadOperation op;
+  //  op.is_dirty(&dirty, &r);
+  //  ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
+  //  ASSERT_TRUE(dirty);
+  //  ASSERT_EQ(0, r);
+  //}
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Overlay) {
+  SKIP_IF_CRIMSON();
+  // create objects
+  {
+    bufferlist bl;
+    bl.append("base");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("cache");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // by default, the overlay sends us to cache pool
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // unless we say otherwise
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(0, 1, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+	"foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+    ASSERT_EQ('b', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Promote) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+  }
+
+  // read, trigger a whiteout
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, PromoteSnap) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  ioctx.snap_set_read(my_snaps[0]);
+
+  // stop and scrub this pg (to make sure scrub can handle missing
+  // clones in the cache tier)
+  // This test requires cache tier and base tier to have the same pg_num/pgp_num
+  {
+    for (int tries = 0; tries < 5; ++tries) {
+      IoCtx cache_ioctx;
+      ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+      uint32_t hash;
+      ASSERT_EQ(0, ioctx.get_object_pg_hash_position2("foo", &hash));
+      ostringstream ss;
+      ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
+	 << cache_ioctx.get_id() << "."
+	 << hash
+	 << "\"}";
+      int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
+      if (r == -EAGAIN ||
+	  r == -ENOENT) {  // in case mgr osdmap is a bit stale
+	sleep(5);
+	continue;
+      }
+      ASSERT_EQ(0, r);
+      break;
+    }
+    // give it a few seconds to go.  this is sloppy but is usually enough time
+    cout << "waiting for scrub..." << std::endl;
+    sleep(15);
+    cout << "done waiting" << std::endl;
+  }
+
+  // read foo snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // read bar snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // read baz snap
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+
+  // read foo
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // read bar
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // read baz
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
+  }
+
+  // cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+}
+
+TEST_F(LibRadosTwoPoolsECPP, PromoteSnapTrimRace) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // delete the snap
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
+
+  ioctx.snap_set_read(my_snaps[0]);
+
+  // read foo snap.  the OSD may or may not realize that this snap has
+  // been logically deleted; either response is valid.
+  {
+    bufferlist bl;
+    int r = ioctx.read("foo", bl, 1, 0);
+    ASSERT_TRUE(r == 1 || r == -ENOENT);
+  }
+
+  // cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Whiteout) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create some whiteouts, verify they behave
+  {
+    ObjectWriteOperation op;
+    op.assert_exists();
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  {
+    ObjectWriteOperation op;
+    op.assert_exists();
+    op.remove();
+    ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.assert_exists();
+    op.remove();
+    ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
+  }
+
+  // verify the whiteouts are there in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // delete a whiteout and verify it goes away
+  ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
+				   librados::OPERATION_IGNORE_CACHE));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // recreate an object and verify we can read it
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Evict) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+  }
+
+  // read, trigger a whiteout, and a dirty object
+  {
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // pin
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict the pinned object with -EPERM
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+					 librados::OPERATION_IGNORE_CACHE,
+					 NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EPERM, completion->get_return_value());
+    completion->release();
+  }
+
+  // unpin
+  {
+    ObjectWriteOperation op;
+    op.cache_unpin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+					 librados::OPERATION_IGNORE_CACHE,
+					 NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("ciao!");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger a promote on the head
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+
+  // evict bam
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bam", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "bam", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+
+  // read foo snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // evict foo snap
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // snap is gone...
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-ENOENT, completion->get_return_value());
+    completion->release();
+  }
+  // head is still there...
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // promote head + snap of bar
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // evict bar head (fail)
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict bar snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // ...and then head
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ObjectReadOperation op;
+    op.read(1, 0, &bl, NULL);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "bar", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+}
+
+TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // verify dirty
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // pin
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush the pinned object with -EPERM
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EPERM, completion->get_return_value());
+    completion->release();
+  }
+
+  // unpin
+  {
+    ObjectWriteOperation op;
+    op.cache_unpin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // verify in base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it != ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // evict it
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, FailedFlush) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // set omap
+  {
+    ObjectWriteOperation op;
+    std::map<std::string, bufferlist> omap;
+    omap["somekey"] = bufferlist();
+    op.omap_set(omap);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_NE(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // get omap
+  {
+    ObjectReadOperation op;
+    bufferlist bl;
+    int prval = 0;
+    std::set<std::string> keys;
+    keys.insert("somekey");
+    std::map<std::string, bufferlist> map;
+
+    op.omap_get_vals_by_keys(keys, &map, &prval);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op, &bl));
+    sleep(5);
+    bool completed = completion->is_complete();
+    if( !completed ) {
+      cache_ioctx.aio_cancel(completion); 
+      std::cerr << "Most probably test case will hang here, please reset manually" << std::endl;
+      ASSERT_TRUE(completed); //in fact we are locked forever at test case shutdown unless fix for http://tracker.ceph.com/issues/14511 is applied. Seems there is no workaround for that
+    }
+    completion->release();
+  }
+  // verify still not in base tier
+  {
+    ASSERT_TRUE(ioctx.nobjects_begin() == ioctx.nobjects_end());
+  }
+  // erase it
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  // flush whiteout
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+  // or base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, Flush) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  uint64_t user_version = 0;
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // verify dirty
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_TRUE(dirty);
+    ASSERT_EQ(0, r);
+    user_version = cache_ioctx.get_last_version();
+  }
+
+  // pin
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush the pinned object with -EPERM
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EPERM, completion->get_return_value());
+    completion->release();
+  }
+
+  // unpin
+  {
+    ObjectWriteOperation op;
+    op.cache_unpin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify clean
+  {
+    bool dirty = false;
+    int r = -1;
+    ObjectReadOperation op;
+    op.is_dirty(&dirty, &r);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
+    ASSERT_FALSE(dirty);
+    ASSERT_EQ(0, r);
+  }
+
+  // verify in base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it != ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // evict it
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // read it again and verify the version is consistent
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ(user_version, cache_ioctx.get_last_version());
+  }
+
+  // erase it
+  {
+    ObjectWriteOperation op;
+    op.remove();
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush whiteout
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+	 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify no longer in cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+  // or base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("a");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // create a snapshot, clone
+  vector<uint64_t> my_snaps(1);
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("b");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // and another
+  my_snaps.resize(2);
+  my_snaps[1] = my_snaps[0];
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
+  ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
+							 my_snaps));
+  {
+    bufferlist bl;
+    bl.append("c");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // verify the object is present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // verify the object is NOT present in the base tier
+  {
+    NObjectIterator it = ioctx.nobjects_begin();
+    ASSERT_TRUE(it == ioctx.nobjects_end());
+  }
+
+  // flush on head (should fail)
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+  // flush on recent snap (should fail)
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EBUSY, completion->get_return_value());
+    completion->release();
+  }
+  // flush on oldest snap
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush on next oldest snap
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // flush on head
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // verify i can read the snaps from the cache pool
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('b', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('a', bl[0]);
+  }
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // verify i can read the snaps from the base pool
+  ioctx.snap_set_read(librados::SNAP_HEAD);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('c', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[0]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('b', bl[0]);
+  }
+  ioctx.snap_set_read(my_snaps[1]);
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('a', bl[0]);
+  }
+
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  cluster.wait_for_latest_osdmap();
+
+  // cleanup
+  ioctx.selfmanaged_snap_remove(my_snaps[0]);
+}
+
+TEST_F(LibRadosTierECPP, FlushWriteRaces) {
+  SKIP_IF_CRIMSON();
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  std::string cache_pool_name = pool_name + "-cache";
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+  IoCtx cache_ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+  cache_ioctx.application_enable("rados", true);
+  IoCtx ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  bufferlist bl;
+  bl.append("hi there");
+  {
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + write
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectWriteOperation op2;
+    op2.write_full(bl);
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate(
+      "foo", completion2, &op2, 0));
+
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  int tries = 1000;
+  do {
+    // create/dirty object
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, ioctx.operate("foo", &op));
+    }
+
+    // try-flush + write
+    {
+      ObjectReadOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY |
+	librados::OPERATION_SKIPRWLOCKS, NULL));
+
+      ObjectWriteOperation op2;
+      op2.write_full(bl);
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
+
+      completion->wait_for_complete();
+      completion2->wait_for_complete();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+	break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+
+  ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush + try-flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_try_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+
+  // create/dirty object
+  int tries = 1000;
+  do {
+    {
+      bufferlist bl;
+      bl.append("hi there");
+      ObjectWriteOperation op;
+      op.write_full(bl);
+      ASSERT_EQ(0, ioctx.operate("foo", &op));
+    }
+
+    // try-flush + flush
+    //  (flush will not piggyback on try-flush)
+    {
+      ObjectReadOperation op;
+      op.cache_try_flush();
+      librados::AioCompletion *completion = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion, &op,
+	librados::OPERATION_IGNORE_OVERLAY |
+	librados::OPERATION_SKIPRWLOCKS, NULL));
+
+      ObjectReadOperation op2;
+      op2.cache_flush();
+      librados::AioCompletion *completion2 = cluster.aio_create_completion();
+      ASSERT_EQ(0, cache_ioctx.aio_operate(
+        "foo", completion2, &op2,
+	librados::OPERATION_IGNORE_OVERLAY, NULL));
+
+      completion->wait_for_complete();
+      completion2->wait_for_complete();
+      int r = completion->get_return_value();
+      ASSERT_TRUE(r == -EBUSY || r == 0);
+      ASSERT_EQ(0, completion2->get_return_value());
+      completion->release();
+      completion2->release();
+      if (r == -EBUSY)
+	break;
+      cout << "didn't get EBUSY, trying again" << std::endl;
+    }
+    ASSERT_TRUE(--tries);
+  } while (true);
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // try-flush + try-flush
+  {
+    ObjectReadOperation op;
+    op.cache_try_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    ObjectReadOperation op2;
+    op2.cache_try_flush();
+    librados::AioCompletion *completion2 = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion2, &op2,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+    completion->wait_for_complete();
+    completion2->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    ASSERT_EQ(0, completion2->get_return_value());
+    completion->release();
+    completion2->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, TryFlushReadRace) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    bufferptr bp(4000000);  // make it big!
+    bp.zero();
+    bl.append(bp);
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // start a continuous stream of reads
+  read_ioctx = &ioctx;
+  test_lock.lock();
+  for (int i = 0; i < max_reads; ++i) {
+    start_flush_read();
+    num_reads++;
+  }
+  test_lock.unlock();
+
+  // try-flush
+  ObjectReadOperation op;
+  op.cache_try_flush();
+  librados::AioCompletion *completion = cluster.aio_create_completion();
+  ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY |
+      librados::OPERATION_SKIPRWLOCKS, NULL));
+
+  completion->wait_for_complete();
+  ASSERT_EQ(0, completion->get_return_value());
+  completion->release();
+
+  // stop reads
+  std::unique_lock locker{test_lock};
+  max_reads = 0;  
+  cond.wait(locker, [] { return num_reads == 0;});
+}
+
+TEST_F(LibRadosTierECPP, CallForcesPromote) {
+  SKIP_IF_CRIMSON();
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  std::string cache_pool_name = pool_name + "-cache";
+  ASSERT_EQ("", create_one_ec_pool_pp(pool_name, cluster));
+  ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
+  IoCtx cache_ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
+  cache_ioctx.application_enable("rados", true);
+  IoCtx ioctx;
+  ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // set things up such that the op would normally be proxied
+  ASSERT_EQ(0, cluster.mon_command(
+	      set_pool_str(cache_pool_name, "hit_set_count", 2),
+	      inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	      set_pool_str(cache_pool_name, "hit_set_period", 600),
+	      inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	      set_pool_str(cache_pool_name, "hit_set_type",
+			   "explicit_object"),
+	      inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	      set_pool_str(cache_pool_name, "min_read_recency_for_promote",
+			   "4"),
+	      inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // create/dirty object
+  bufferlist bl;
+  bl.append("hi there");
+  {
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // flush
+  {
+    ObjectReadOperation op;
+    op.cache_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op,
+      librados::OPERATION_IGNORE_OVERLAY, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // evict
+  {
+    ObjectReadOperation op;
+    op.cache_evict();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+					 librados::OPERATION_IGNORE_CACHE,
+					 NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // call
+  {
+    ObjectReadOperation op;
+    bufferlist bl;
+    op.exec("rbd", "get_id", bl);
+    bufferlist out;
+    // should get EIO (not an rbd object), not -EOPNOTSUPP (we didn't promote)
+    ASSERT_EQ(-5, ioctx.operate("foo", &op, &out));
+  }
+
+  // make sure foo is back in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    ASSERT_TRUE(it->get_oid() == string("foo"));
+    ++it;
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+
+  ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
+  ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, cluster));
+}
+
+TEST_F(LibRadosTierECPP, HitSetNone) {
+  SKIP_IF_CRIMSON();
+  {
+    list< pair<time_t,time_t> > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
+    c->wait_for_complete();
+    ASSERT_EQ(0, c->get_return_value());
+    ASSERT_TRUE(ls.empty());
+    c->release();
+  }
+  {
+    bufferlist bl;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
+    c->wait_for_complete();
+    ASSERT_EQ(-ENOENT, c->get_return_value());
+    c->release();
+  }
+}
+
+TEST_F(LibRadosTwoPoolsECPP, HitSetRead) {
+  SKIP_IF_CRIMSON();
+  // make it a tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
+						"explicit_object"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  cache_ioctx.set_namespace("");
+
+  // keep reading until we see our object appear in the HitSet
+  utime_t start = ceph_clock_now();
+  utime_t hard_stop = start + utime_t(600, 0);
+
+  while (true) {
+    utime_t now = ceph_clock_now();
+    ASSERT_TRUE(now < hard_stop);
+
+    string name = "foo";
+    uint32_t hash;
+    ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
+    hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
+		  cluster.pool_lookup(cache_pool_name.c_str()), "");
+
+    bufferlist bl;
+    ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
+
+    bufferlist hbl;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
+    c->wait_for_complete();
+    c->release();
+
+    if (hbl.length()) {
+      auto p = hbl.cbegin();
+      HitSet hs;
+      decode(hs, p);
+      if (hs.contains(oid)) {
+	cout << "ok, hit_set contains " << oid << std::endl;
+	break;
+      }
+      cout << "hmm, not in HitSet yet" << std::endl;
+    } else {
+      cout << "hmm, no HitSet yet" << std::endl;
+    }
+
+    sleep(1);
+  }
+}
+
+// disable this test until hitset-get reliably works on EC pools
+#if 0
+TEST_F(LibRadosTierECPP, HitSetWrite) {
+  int num_pg = _get_pg_num(cluster, pool_name);
+  ceph_assert(num_pg > 0);
+
+  // enable hitset tracking for this pool
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", 8),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_period", 600),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_type",
+						"explicit_hash"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  ioctx.set_namespace("");
+
+  // do a bunch of writes
+  for (int i=0; i<1000; ++i) {
+    bufferlist bl;
+    bl.append("a");
+    ASSERT_EQ(0, ioctx.write(stringify(i), bl, 1, 0));
+  }
+
+  // get HitSets
+  std::map<int,HitSet> hitsets;
+  for (int i=0; i<num_pg; ++i) {
+    list< pair<time_t,time_t> > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, ioctx.hit_set_list(i, c, &ls));
+    c->wait_for_complete();
+    c->release();
+    std::cout << "pg " << i << " ls " << ls << std::endl;
+    ASSERT_FALSE(ls.empty());
+
+    // get the latest
+    c = librados::Rados::aio_create_completion();
+    bufferlist bl;
+    ASSERT_EQ(0, ioctx.hit_set_get(i, c, ls.back().first, &bl));
+    c->wait_for_complete();
+    c->release();
+
+    //std::cout << "bl len is " << bl.length() << "\n";
+    //bl.hexdump(std::cout);
+    //std::cout << std::endl;
+
+    auto p = bl.cbegin();
+    decode(hitsets[i], p);
+
+    // cope with racing splits by refreshing pg_num
+    if (i == num_pg - 1)
+      num_pg = _get_pg_num(cluster, pool_name);
+  }
+
+  for (int i=0; i<1000; ++i) {
+    string n = stringify(i);
+    uint32_t hash = ioctx.get_object_hash_position(n);
+    hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
+		  cluster.pool_lookup(pool_name.c_str()), "");
+    std::cout << "checking for " << oid << std::endl;
+    bool found = false;
+    for (int p=0; p<num_pg; ++p) {
+      if (hitsets[p].contains(oid)) {
+	found = true;
+	break;
+      }
+    }
+    ASSERT_TRUE(found);
+  }
+}
+#endif
+
+TEST_F(LibRadosTwoPoolsECPP, HitSetTrim) {
+  SKIP_IF_CRIMSON();
+  unsigned count = 3;
+  unsigned period = 3;
+
+  // make it a tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
+						inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
+				   inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
+				   inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  cache_ioctx.set_namespace("");
+
+  // do a bunch of writes and make sure the hitsets rotate
+  utime_t start = ceph_clock_now();
+  utime_t hard_stop = start + utime_t(count * period * 50, 0);
+
+  time_t first = 0;
+  int bsize = alignment;
+  char *buf = (char *)new char[bsize];
+  memset(buf, 'f', bsize);
+
+  while (true) {
+    string name = "foo";
+    uint32_t hash;
+    ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
+    hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
+
+    bufferlist bl;
+    bl.append(buf, bsize);
+    ASSERT_EQ(0, cache_ioctx.append("foo", bl, bsize));
+
+    list<pair<time_t, time_t> > ls;
+    AioCompletion *c = librados::Rados::aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
+    c->wait_for_complete();
+    c->release();
+
+    cout << " got ls " << ls << std::endl;
+    if (!ls.empty()) {
+      if (!first) {
+	first = ls.front().first;
+	cout << "first is " << first << std::endl;
+      } else {
+	if (ls.front().first != first) {
+	  cout << "first now " << ls.front().first << ", trimmed" << std::endl;
+	  break;
+	}
+      }
+    }
+
+    utime_t now = ceph_clock_now();
+    ASSERT_TRUE(now < hard_stop);
+
+    sleep(1);
+  }
+  delete[] buf;
+}
+
+TEST_F(LibRadosTwoPoolsECPP, PromoteOn2ndRead) {
+  SKIP_IF_CRIMSON();
+  // create object
+  for (int i=0; i<20; ++i) {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // enable hitset tracking for this pool
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_count", 2),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_period", 600),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  int fake = 0;  // set this to non-zero to test spurious promotion,
+		 // e.g. from thrashing
+  int attempt = 0;
+  string obj;
+  while (true) {
+    // 1st read, don't trigger a promote
+    obj = "foo" + stringify(attempt);
+    cout << obj << std::endl;
+    {
+      bufferlist bl;
+      ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
+      if (--fake >= 0) {
+	sleep(1);
+	ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
+	sleep(1);
+      }
+    }
+
+    // verify the object is NOT present in the cache tier
+    {
+      bool found = false;
+      NObjectIterator it = cache_ioctx.nobjects_begin();
+      while (it != cache_ioctx.nobjects_end()) {
+	cout << " see " << it->get_oid() << std::endl;
+	if (it->get_oid() == string(obj.c_str())) {
+	  found = true;
+	  break;
+	}
+	++it;
+      }
+      if (!found)
+	break;
+    }
+
+    ++attempt;
+    ASSERT_LE(attempt, 20);
+    cout << "hrm, object is present in cache on attempt " << attempt
+	 << ", retrying" << std::endl;
+  }
+
+  // Read until the object is present in the cache tier
+  cout << "verifying " << obj << " is eventually promoted" << std::endl;
+  while (true) {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
+
+    bool there = false;
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    while (it != cache_ioctx.nobjects_end()) {
+      if (it->get_oid() == string(obj.c_str())) {
+	there = true;
+	break;
+      }
+      ++it;
+    }
+    if (there)
+      break;
+
+    sleep(1);
+  }
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsECPP, ProxyRead) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"readproxy\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read and verify the object
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('h', bl[0]);
+  }
+
+  // Verify 10 times the object is NOT present in the cache tier
+  uint32_t i = 0;
+  while (i++ < 10) {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+    sleep(1);
+  }
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsECPP, CachePin) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("baz", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bam", &op));
+  }
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // read, trigger promote
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
+    ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
+    ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
+  }
+
+  // verify the objects are present in the cache tier
+  {
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    ASSERT_TRUE(it != cache_ioctx.nobjects_end());
+    for (uint32_t i = 0; i < 4; i++) {
+      ASSERT_TRUE(it->get_oid() == string("foo") ||
+                  it->get_oid() == string("bar") ||
+                  it->get_oid() == string("baz") ||
+                  it->get_oid() == string("bam"));
+      ++it;
+    }
+    ASSERT_TRUE(it == cache_ioctx.nobjects_end());
+  }
+
+  // pin objects
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  {
+    ObjectWriteOperation op;
+    op.cache_pin();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // enable agent
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_count", 2),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_period", 600),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "target_max_objects", 1),
+    inbl, NULL, NULL));
+
+  sleep(10);
+
+  // Verify the pinned object 'foo' is not flushed/evicted
+  uint32_t count = 0;
+  while (true) {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
+
+    count = 0;
+    NObjectIterator it = cache_ioctx.nobjects_begin();
+    while (it != cache_ioctx.nobjects_end()) {
+      ASSERT_TRUE(it->get_oid() == string("foo") ||
+                  it->get_oid() == string("bar") ||
+                  it->get_oid() == string("baz") ||
+                  it->get_oid() == string("bam"));
+      ++count;
+      ++it;
+    }
+    if (count == 2) {
+      ASSERT_TRUE(it->get_oid() == string("foo") ||
+                  it->get_oid() == string("baz"));
+      break;
+    }
+
+    sleep(1);
+  }
+
+  // tear down tiers
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
+    "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+TEST_F(LibRadosTwoPoolsECPP, SetRedirectRead) {
+  SKIP_IF_CRIMSON();
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hi there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
+  }
+
+  // configure tier
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  {
+    ObjectWriteOperation op;
+    op.set_redirect("bar", cache_ioctx, 0);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // read and verify the object
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('t', bl[0]);
+  }
+
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsECPP, SetChunkRead) {
+  SKIP_IF_CRIMSON();
+  // note: require >= mimic
+
+  {
+    bufferlist bl;
+    bl.append("there hi");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+
+  {
+    bufferlist bl;
+    bl.append("There hi");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // set_chunk
+  manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 4, "bar", "foo");
+
+  // promote
+  {
+    ObjectWriteOperation op;
+    op.tier_promote();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // read and verify the object
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('T', bl[0]);
+  }
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsECPP, ManifestPromoteRead) {
+  SKIP_IF_CRIMSON();
+  // note: require >= mimic
+
+  // create object
+  {
+    bufferlist bl;
+    bl.append("hiaa there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("base chunk");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo-chunk", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("HIaa there");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("BASE CHUNK");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar-chunk", &op));
+  }
+
+  // set-redirect
+  {
+    ObjectWriteOperation op;
+    op.set_redirect("bar", ioctx, 0);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // set-chunk
+  manifest_set_chunk(cluster, ioctx, cache_ioctx, 0, 10, "bar-chunk", "foo-chunk");
+  // promote
+  {
+    ObjectWriteOperation op;
+    op.tier_promote();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // read and verify the object (redirect)
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
+    ASSERT_EQ('H', bl[0]);
+  }
+  // promote
+  {
+    ObjectWriteOperation op;
+    op.tier_promote();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo-chunk", completion, &op));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+  // read and verify the object
+  {
+    bufferlist bl;
+    ASSERT_EQ(1, cache_ioctx.read("foo-chunk", bl, 1, 0));
+    ASSERT_EQ('B', bl[0]);
+  }
+
+  // wait for maps to settle before next test
+  cluster.wait_for_latest_osdmap();
+}
+
+TEST_F(LibRadosTwoPoolsECPP, TrySetDedupTier) {
+  SKIP_IF_CRIMSON();
+  // note: require >= mimic
+  
+  bufferlist inbl;
+  ASSERT_EQ(-EOPNOTSUPP, cluster.mon_command(
+	set_pool_str(pool_name, "dedup_tier", cache_pool_name),
+	inbl, NULL, NULL));
+}
+
+TEST_F(LibRadosTwoPoolsPP, PropagateBaseTierError) {
+  SKIP_IF_CRIMSON();
+  // write object  to base tier
+  bufferlist omap_bl;
+  encode(static_cast<uint32_t>(0U), omap_bl);
+
+  ObjectWriteOperation op1;
+  op1.omap_set({{"somekey", omap_bl}});
+  ASSERT_EQ(0, ioctx.operate("propagate-base-tier-error", &op1));
+
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_count", 1),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "hit_set_period", 600),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "target_max_objects", 250),
+    inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // guarded op should fail so expect error to propagate to cache tier
+  bufferlist test_omap_bl;
+  encode(static_cast<uint32_t>(1U), test_omap_bl);
+
+  ObjectWriteOperation op2;
+  op2.omap_cmp({{"somekey", {test_omap_bl, CEPH_OSD_CMPXATTR_OP_EQ}}}, nullptr);
+  op2.omap_set({{"somekey", test_omap_bl}});
+
+  ASSERT_EQ(-ECANCELED, ioctx.operate("propagate-base-tier-error", &op2));
+}
+
+TEST_F(LibRadosTwoPoolsPP, HelloWriteReturn) {
+  SKIP_IF_CRIMSON();
+  // configure cache
+  bufferlist inbl;
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
+    "\", \"tierpool\": \"" + cache_pool_name +
+    "\", \"force_nonempty\": \"--force-nonempty\" }",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
+    "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
+    "\", \"mode\": \"writeback\"}",
+    inbl, NULL, NULL));
+
+  // set things up such that the op would normally be proxied
+  ASSERT_EQ(0, cluster.mon_command(
+	      set_pool_str(cache_pool_name, "hit_set_count", 2),
+	      inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	      set_pool_str(cache_pool_name, "hit_set_period", 600),
+	      inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	      set_pool_str(cache_pool_name, "hit_set_type",
+			   "explicit_object"),
+	      inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+	      set_pool_str(cache_pool_name, "min_read_recency_for_promote",
+			   "10000"),
+	      inbl, NULL, NULL));
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // this *will* return data due to the RETURNVEC flag
+  {
+    bufferlist in, out;
+    int rval;
+    ObjectWriteOperation o;
+    o.exec("hello", "write_return_data", in, &out, &rval);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
+				   librados::OPERATION_RETURNVEC));
+    completion->wait_for_complete();
+    ASSERT_EQ(42, completion->get_return_value());
+    ASSERT_EQ(42, rval);
+    out.hexdump(std::cout);
+    ASSERT_EQ("you might see this", std::string(out.c_str(), out.length()));
+  }
+
+  // this will overflow because the return data is too big
+  {
+    bufferlist in, out;
+    int rval;
+    ObjectWriteOperation o;
+    o.exec("hello", "write_too_much_return_data", in, &out, &rval);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
+				   librados::OPERATION_RETURNVEC));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EOVERFLOW, completion->get_return_value());
+    ASSERT_EQ(-EOVERFLOW, rval);
+    ASSERT_EQ("", std::string(out.c_str(), out.length()));
+  }
+}
+
+TEST_F(LibRadosTwoPoolsPP, TierFlushDuringUnsetDedupTier) {
+  SKIP_IF_CRIMSON();
+  // skip test if not yet octopus
+  if (_get_required_osd_release(cluster) < "octopus") {
+    cout << "cluster is not yet octopus, skipping test" << std::endl;
+    return;
+  }
+
+  bufferlist inbl;
+
+  // set dedup parameters without dedup_tier
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "fingerprint_algorithm", "sha1"),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "dedup_chunk_algorithm", "fastcdc"),
+    inbl, NULL, NULL));
+  ASSERT_EQ(0, cluster.mon_command(
+    set_pool_str(cache_pool_name, "dedup_cdc_chunk_size", 1024),
+    inbl, NULL, NULL));
+
+  // create object
+  bufferlist gbl;
+  {
+    generate_buffer(1024*8, &gbl);
+    ObjectWriteOperation op;
+    op.write_full(gbl);
+    ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
+  }
+  {
+    bufferlist bl;
+    bl.append("there hiHI");
+    ObjectWriteOperation op;
+    op.write_full(bl);
+    ASSERT_EQ(0, ioctx.operate("bar", &op));
+  }
+
+  // wait for maps to settle
+  cluster.wait_for_latest_osdmap();
+
+  // set-chunk to set manifest object
+  {
+    ObjectReadOperation op;
+    op.set_chunk(0, 2, ioctx, "bar", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
+      librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(0, completion->get_return_value());
+    completion->release();
+  }
+
+  // flush to check if proper error is returned
+  {
+    ObjectReadOperation op;
+    op.tier_flush();
+    librados::AioCompletion *completion = cluster.aio_create_completion();
+    ASSERT_EQ(0, cache_ioctx.aio_operate(
+      "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
+    completion->wait_for_complete();
+    ASSERT_EQ(-EINVAL, completion->get_return_value());
+    completion->release();
+  }
+}
+
diff --git a/src/test/librados/watch_notify.cc b/src/test/librados/watch_notify.cc
new file mode 100644
index 000000000..4d880f2c2
--- /dev/null
+++ b/src/test/librados/watch_notify.cc
@@ -0,0 +1,657 @@
+#include "include/rados/librados.h"
+#include "include/rados/rados_types.h"
+#include "test/librados/test.h"
+#include "test/librados/TestCase.h"
+#include "crimson_utils.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <semaphore.h>
+#include "gtest/gtest.h"
+#include "include/encoding.h"
+#include <set>
+#include <map>
+
+typedef RadosTestEC LibRadosWatchNotifyEC;
+
+int notify_sleep = 0;
+
+// notify
+static sem_t sem;
+
+static void watch_notify_test_cb(uint8_t opcode, uint64_t ver, void *arg)
+{
+  std::cout << __func__ << std::endl;
+  sem_post(&sem);
+}
+
+class LibRadosWatchNotify : public RadosTest
+{
+protected:
+  // notify 2
+  bufferlist notify_bl;
+  std::set<uint64_t> notify_cookies;
+  rados_ioctx_t notify_io;
+  const char *notify_oid = nullptr;
+  int notify_err = 0;
+  rados_completion_t notify_comp;
+
+  static void watch_notify2_test_cb(void *arg,
+                                    uint64_t notify_id,
+                                    uint64_t cookie,
+                                    uint64_t notifier_gid,
+                                    void *data,
+                                    size_t data_len);
+  static void watch_notify2_test_errcb(void *arg, uint64_t cookie, int err);
+  static void watch_notify2_test_errcb_reconnect(void *arg, uint64_t cookie, int err);
+  static void watch_notify2_test_errcb_aio_reconnect(void *arg, uint64_t cookie, int err);
+};
+
+
+void LibRadosWatchNotify::watch_notify2_test_cb(void *arg,
+				  uint64_t notify_id,
+				  uint64_t cookie,
+				  uint64_t notifier_gid,
+				  void *data,
+				  size_t data_len)
+{
+  std::cout << __func__ << " from " << notifier_gid << " notify_id " << notify_id
+	    << " cookie " << cookie << std::endl;
+  ceph_assert(notifier_gid > 0);
+  auto thiz = reinterpret_cast<LibRadosWatchNotify*>(arg);
+  ceph_assert(thiz);
+  thiz->notify_cookies.insert(cookie);
+  thiz->notify_bl.clear();
+  thiz->notify_bl.append((char*)data, data_len);
+  if (notify_sleep)
+    sleep(notify_sleep);
+  thiz->notify_err = 0;
+  rados_notify_ack(thiz->notify_io, thiz->notify_oid, notify_id, cookie,
+                   "reply", 5);
+}
+
+void LibRadosWatchNotify::watch_notify2_test_errcb(void *arg,
+                                                   uint64_t cookie,
+                                                   int err)
+{
+  std::cout << __func__ << " cookie " << cookie << " err " << err << std::endl;
+  ceph_assert(cookie > 1000);
+  auto thiz = reinterpret_cast<LibRadosWatchNotify*>(arg);
+  ceph_assert(thiz);
+  thiz->notify_err = err;
+}
+
+void LibRadosWatchNotify::watch_notify2_test_errcb_reconnect(void *arg,
+                                                   uint64_t cookie,
+                                                   int err)
+{
+  std::cout << __func__ << " cookie " << cookie << " err " << err << std::endl;
+  ceph_assert(cookie > 1000);
+  auto thiz = reinterpret_cast<LibRadosWatchNotify*>(arg);
+  ceph_assert(thiz);
+  thiz->notify_err = rados_unwatch2(thiz->ioctx, cookie);
+  thiz->notify_cookies.erase(cookie); //delete old cookie
+  thiz->notify_err = rados_watch2(thiz->ioctx, thiz->notify_oid, &cookie,
+                  watch_notify2_test_cb, watch_notify2_test_errcb_reconnect, thiz);
+  if (thiz->notify_err < 0) {
+    std::cout << __func__ << " reconnect watch failed with error " << thiz->notify_err << std::endl;
+    return;
+  }
+  return;
+}
+
+
+void LibRadosWatchNotify::watch_notify2_test_errcb_aio_reconnect(void *arg,
+                                                   uint64_t cookie,
+                                                   int err)
+{
+  std::cout << __func__ << " cookie " << cookie << " err " << err << std::endl;
+  ceph_assert(cookie > 1000);
+  auto thiz = reinterpret_cast<LibRadosWatchNotify*>(arg);
+  ceph_assert(thiz);
+  thiz->notify_err = rados_aio_unwatch(thiz->ioctx, cookie, thiz->notify_comp);
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &thiz->notify_comp));
+  thiz->notify_cookies.erase(cookie); //delete old cookie
+  thiz->notify_err = rados_aio_watch(thiz->ioctx, thiz->notify_oid, thiz->notify_comp, &cookie,
+                  watch_notify2_test_cb, watch_notify2_test_errcb_aio_reconnect, thiz);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(thiz->notify_comp));
+  ASSERT_EQ(0, rados_aio_get_return_value(thiz->notify_comp));
+  rados_aio_release(thiz->notify_comp);
+  if (thiz->notify_err < 0) {
+    std::cout << __func__ << " reconnect watch failed with error " << thiz->notify_err << std::endl;
+    return;
+  }
+  return;
+}
+
+class WatchNotifyTestCtx2;
+
+// --
+
+#pragma GCC diagnostic ignored "-Wpragmas"
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+
+TEST_F(LibRadosWatchNotify, WatchNotify) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+      rados_watch(ioctx, "foo", 0, &handle, watch_notify_test_cb, NULL));
+  for (unsigned i=0; i<10; ++i) {
+    int r = rados_notify(ioctx, "foo", 0, NULL, 0);
+    if (r == 0) {
+      break;
+    }
+    if (!getenv("ALLOW_TIMEOUTS")) {
+      ASSERT_EQ(0, r);
+    }
+  }
+  TestAlarm alarm;
+  sem_wait(&sem);
+  rados_unwatch(ioctx, "foo", handle);
+
+  // when dne ...
+  ASSERT_EQ(-ENOENT,
+      rados_watch(ioctx, "dne", 0, &handle, watch_notify_test_cb, NULL));
+
+  sem_destroy(&sem);
+}
+
+TEST_F(LibRadosWatchNotifyEC, WatchNotify) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, "foo", buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+      rados_watch(ioctx, "foo", 0, &handle, watch_notify_test_cb, NULL));
+  for (unsigned i=0; i<10; ++i) {
+    int r = rados_notify(ioctx, "foo", 0, NULL, 0);
+    if (r == 0) {
+      break;
+    }
+    if (!getenv("ALLOW_TIMEOUTS")) {
+      ASSERT_EQ(0, r);
+    }
+  }
+  TestAlarm alarm;
+  sem_wait(&sem);
+  rados_unwatch(ioctx, "foo", handle);
+  sem_destroy(&sem);
+}
+
+#pragma GCC diagnostic pop
+#pragma GCC diagnostic warning "-Wpragmas"
+
+
+// --
+
+TEST_F(LibRadosWatchNotify, Watch2Delete) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_err = 0;
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+	    rados_watch2(ioctx, notify_oid, &handle,
+			 watch_notify2_test_cb,
+			 watch_notify2_test_errcb, this));
+  ASSERT_EQ(0, rados_remove(ioctx, notify_oid));
+  int left = 300;
+  std::cout << "waiting up to " << left << " for disconnect notification ..."
+	    << std::endl;
+  while (notify_err == 0 && --left) {
+    sleep(1);
+  }
+  ASSERT_TRUE(left > 0);
+  ASSERT_EQ(-ENOTCONN, notify_err);
+  int rados_watch_check_err = rados_watch_check(ioctx, handle);
+  // We may hit ENOENT due to socket failure and a forced reconnect
+  EXPECT_TRUE(rados_watch_check_err == -ENOTCONN || rados_watch_check_err == -ENOENT)
+    << "Where rados_watch_check_err = " << rados_watch_check_err;
+  rados_unwatch2(ioctx, handle);
+  rados_watch_flush(cluster);
+}
+
+TEST_F(LibRadosWatchNotify, AioWatchDelete) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_err = 0;
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+
+
+  rados_completion_t comp;
+  uint64_t handle;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &comp));
+  rados_aio_watch(ioctx, notify_oid, comp, &handle,
+                  watch_notify2_test_cb, watch_notify2_test_errcb, this);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(comp));
+  ASSERT_EQ(0, rados_aio_get_return_value(comp));
+  rados_aio_release(comp);
+  ASSERT_EQ(0, rados_remove(ioctx, notify_oid));
+  int left = 300;
+  std::cout << "waiting up to " << left << " for disconnect notification ..."
+	    << std::endl;
+  while (notify_err == 0 && --left) {
+    sleep(1);
+  }
+  ASSERT_TRUE(left > 0);
+  ASSERT_EQ(-ENOTCONN, notify_err);
+  int rados_watch_check_err = rados_watch_check(ioctx, handle);
+  // We may hit ENOENT due to socket failure injection and a forced reconnect
+  EXPECT_TRUE(rados_watch_check_err == -ENOTCONN || rados_watch_check_err == -ENOENT)
+    << "Where rados_watch_check_err = " << rados_watch_check_err;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &comp));
+  rados_aio_unwatch(ioctx, handle, comp);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(comp));
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(comp));
+  rados_aio_release(comp);
+}
+
+// --
+
+TEST_F(LibRadosWatchNotify, WatchNotify2) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+      rados_watch2(ioctx, notify_oid, &handle,
+		   watch_notify2_test_cb,
+		   watch_notify2_test_errcb_reconnect, this));
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+  char *reply_buf = 0;
+  size_t reply_buf_len;
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+			     "notify", 6, 300000,
+			     &reply_buf, &reply_buf_len));
+  bufferlist reply;
+  reply.append(reply_buf, reply_buf_len);
+  std::map<std::pair<uint64_t,uint64_t>, bufferlist> reply_map;
+  std::set<std::pair<uint64_t,uint64_t> > missed_map;
+  auto reply_p = reply.cbegin();
+  decode(reply_map, reply_p);
+  decode(missed_map, reply_p);
+  ASSERT_EQ(1u, reply_map.size());
+  ASSERT_EQ(0u, missed_map.size());
+  ASSERT_EQ(1u, notify_cookies.size());
+  handle = *notify_cookies.begin();
+  ASSERT_EQ(1u, notify_cookies.count(handle));
+  ASSERT_EQ(5u, reply_map.begin()->second.length());
+  ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+  rados_buffer_free(reply_buf);
+
+  // try it on a non-existent object ... our buffer pointers
+  // should get zeroed.
+  ASSERT_EQ(-ENOENT, rados_notify2(ioctx, "doesnotexist",
+				   "notify", 6, 300000,
+				   &reply_buf, &reply_buf_len));
+  ASSERT_EQ((char*)0, reply_buf);
+  ASSERT_EQ(0u, reply_buf_len);
+
+  rados_unwatch2(ioctx, handle);
+  rados_watch_flush(cluster);
+}
+
+TEST_F(LibRadosWatchNotify, AioWatchNotify2) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &notify_comp));
+  rados_aio_watch(ioctx, notify_oid, notify_comp, &handle,
+                  watch_notify2_test_cb, watch_notify2_test_errcb_aio_reconnect, this);
+  
+  ASSERT_EQ(0, rados_aio_wait_for_complete(notify_comp));
+  ASSERT_EQ(0, rados_aio_get_return_value(notify_comp));
+  rados_aio_release(notify_comp);
+
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+  char *reply_buf = 0;
+  size_t reply_buf_len;
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+			     "notify", 6, 300000,
+			     &reply_buf, &reply_buf_len));
+  bufferlist reply;
+  reply.append(reply_buf, reply_buf_len);
+  std::map<std::pair<uint64_t,uint64_t>, bufferlist> reply_map;
+  std::set<std::pair<uint64_t,uint64_t> > missed_map;
+  auto reply_p = reply.cbegin();
+  decode(reply_map, reply_p);
+  decode(missed_map, reply_p);
+  ASSERT_EQ(1u, reply_map.size());
+  ASSERT_EQ(0u, missed_map.size());
+  ASSERT_EQ(1u, notify_cookies.size());
+  handle = *notify_cookies.begin();
+  ASSERT_EQ(1u, notify_cookies.count(handle));
+  ASSERT_EQ(5u, reply_map.begin()->second.length());
+  ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+  rados_buffer_free(reply_buf);
+
+  // try it on a non-existent object ... our buffer pointers
+  // should get zeroed.
+  ASSERT_EQ(-ENOENT, rados_notify2(ioctx, "doesnotexist",
+				   "notify", 6, 300000,
+				   &reply_buf, &reply_buf_len));
+  ASSERT_EQ((char*)0, reply_buf);
+  ASSERT_EQ(0u, reply_buf_len);
+
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &notify_comp));
+  rados_aio_unwatch(ioctx, handle, notify_comp);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(notify_comp));
+  ASSERT_EQ(0, rados_aio_get_return_value(notify_comp));
+  rados_aio_release(notify_comp);
+}
+
+TEST_F(LibRadosWatchNotify, AioNotify) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+      rados_watch2(ioctx, notify_oid, &handle,
+		   watch_notify2_test_cb,
+		   watch_notify2_test_errcb, this));
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+  char *reply_buf = 0;
+  size_t reply_buf_len;
+  rados_completion_t comp;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &comp));
+  ASSERT_EQ(0, rados_aio_notify(ioctx, "foo", comp, "notify", 6, 300000,
+                                &reply_buf, &reply_buf_len));
+  ASSERT_EQ(0, rados_aio_wait_for_complete(comp));
+  ASSERT_EQ(0, rados_aio_get_return_value(comp));
+  rados_aio_release(comp);
+
+  size_t nr_acks, nr_timeouts;
+  notify_ack_t *acks = nullptr;
+  notify_timeout_t *timeouts = nullptr;
+  ASSERT_EQ(0, rados_decode_notify_response(reply_buf, reply_buf_len,
+                                            &acks, &nr_acks, &timeouts, &nr_timeouts));
+  ASSERT_EQ(1u, nr_acks);
+  ASSERT_EQ(0u, nr_timeouts);
+  ASSERT_EQ(1u, notify_cookies.size());
+  ASSERT_EQ(1u, notify_cookies.count(handle));
+  ASSERT_EQ(5u, acks[0].payload_len);
+  ASSERT_EQ(0, strncmp("reply", acks[0].payload, acks[0].payload_len));
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+  rados_free_notify_response(acks, nr_acks, timeouts);
+  rados_buffer_free(reply_buf);
+
+  // try it on a non-existent object ... our buffer pointers
+  // should get zeroed.
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &comp));
+  ASSERT_EQ(0, rados_aio_notify(ioctx, "doesnotexist", comp, "notify", 6,
+                                300000, &reply_buf, &reply_buf_len));
+  ASSERT_EQ(0, rados_aio_wait_for_complete(comp));
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(comp));
+  rados_aio_release(comp);
+  ASSERT_EQ((char*)0, reply_buf);
+  ASSERT_EQ(0u, reply_buf_len);
+
+  rados_unwatch2(ioctx, handle);
+  rados_watch_flush(cluster);
+}
+
+// --
+
+TEST_F(LibRadosWatchNotify, WatchNotify2Multi) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+  uint64_t handle1, handle2;
+  ASSERT_EQ(0,
+      rados_watch2(ioctx, notify_oid, &handle1,
+		   watch_notify2_test_cb,
+		   watch_notify2_test_errcb, this));
+  ASSERT_EQ(0,
+      rados_watch2(ioctx, notify_oid, &handle2,
+		   watch_notify2_test_cb,
+		   watch_notify2_test_errcb, this));
+  ASSERT_GT(rados_watch_check(ioctx, handle1), 0);
+  ASSERT_GT(rados_watch_check(ioctx, handle2), 0);
+  ASSERT_NE(handle1, handle2);
+  char *reply_buf = 0;
+  size_t reply_buf_len;
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+			     "notify", 6, 300000,
+			     &reply_buf, &reply_buf_len));
+  bufferlist reply;
+  reply.append(reply_buf, reply_buf_len);
+  std::map<std::pair<uint64_t,uint64_t>, bufferlist> reply_map;
+  std::set<std::pair<uint64_t,uint64_t> > missed_map;
+  auto reply_p = reply.cbegin();
+  decode(reply_map, reply_p);
+  decode(missed_map, reply_p);
+  ASSERT_EQ(2u, reply_map.size());
+  ASSERT_EQ(5u, reply_map.begin()->second.length());
+  ASSERT_EQ(0u, missed_map.size());
+  ASSERT_EQ(2u, notify_cookies.size());
+  ASSERT_EQ(1u, notify_cookies.count(handle1));
+  ASSERT_EQ(1u, notify_cookies.count(handle2));
+  ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  ASSERT_GT(rados_watch_check(ioctx, handle1), 0);
+  ASSERT_GT(rados_watch_check(ioctx, handle2), 0);
+  rados_buffer_free(reply_buf);
+  rados_unwatch2(ioctx, handle1);
+  rados_unwatch2(ioctx, handle2);
+  rados_watch_flush(cluster);
+}
+
+// --
+
+TEST_F(LibRadosWatchNotify, WatchNotify2Timeout) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_sleep = 3; // 3s
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+  uint64_t handle;
+  ASSERT_EQ(0,
+      rados_watch2(ioctx, notify_oid, &handle,
+		   watch_notify2_test_cb,
+		   watch_notify2_test_errcb, this));
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+  char *reply_buf = 0;
+  size_t reply_buf_len;
+  ASSERT_EQ(-ETIMEDOUT, rados_notify2(ioctx, notify_oid,
+				      "notify", 6, 1000, // 1s
+				      &reply_buf, &reply_buf_len));
+  ASSERT_EQ(1u, notify_cookies.size());
+  {
+    bufferlist reply;
+    reply.append(reply_buf, reply_buf_len);
+    std::map<std::pair<uint64_t,uint64_t>, bufferlist> reply_map;
+    std::set<std::pair<uint64_t,uint64_t> > missed_map;
+    auto reply_p = reply.cbegin();
+    decode(reply_map, reply_p);
+    decode(missed_map, reply_p);
+    ASSERT_EQ(0u, reply_map.size());
+    ASSERT_EQ(1u, missed_map.size());
+  }
+  rados_buffer_free(reply_buf);
+
+  // we should get the next notify, though!
+  notify_sleep = 0;
+  notify_cookies.clear();
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+			     "notify", 6, 300000, // 300s
+			     &reply_buf, &reply_buf_len));
+  ASSERT_EQ(1u, notify_cookies.size());
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+
+  rados_unwatch2(ioctx, handle);
+
+  rados_completion_t comp;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &comp));
+  rados_aio_watch_flush(cluster, comp);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(comp));
+  ASSERT_EQ(0, rados_aio_get_return_value(comp));
+  rados_aio_release(comp);
+  rados_buffer_free(reply_buf);
+
+}
+
+TEST_F(LibRadosWatchNotify, Watch3Timeout) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_cookies.clear();
+  notify_err = 0;
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+  uint64_t handle;
+  time_t start = time(0);
+  const uint32_t timeout = 4;
+  {
+    // make sure i timeout before the messenger reconnects to the OSD,
+    // it will resend a watch request on behalf of the client, and the
+    // timer of timeout on OSD side will be reset by the new request.
+    char conf[128];
+    ASSERT_EQ(0, rados_conf_get(cluster,
+                                "ms_connection_idle_timeout",
+                                conf, sizeof(conf)));
+    auto connection_idle_timeout = std::stoll(conf);
+    ASSERT_LT(timeout, connection_idle_timeout);
+  }
+  ASSERT_EQ(0,
+	    rados_watch3(ioctx, notify_oid, &handle,
+                     watch_notify2_test_cb, watch_notify2_test_errcb,
+                     timeout, this));
+  int age = rados_watch_check(ioctx, handle);
+  time_t age_bound = time(0) + 1 - start;
+  ASSERT_LT(age, age_bound * 1000);
+  ASSERT_GT(age, 0);
+  rados_conf_set(cluster, "objecter_inject_no_watch_ping", "true");
+  // allow a long time here since an osd peering event will renew our
+  // watch.
+  int left = 256 * timeout;
+  std::cout << "waiting up to " << left << " for osd to time us out ..."
+	    << std::endl;
+  while (notify_err == 0 && --left) {
+    sleep(1);
+  }
+  ASSERT_GT(left, 0);
+  rados_conf_set(cluster, "objecter_inject_no_watch_ping", "false");
+  ASSERT_EQ(-ENOTCONN, notify_err);
+  ASSERT_EQ(-ENOTCONN, rados_watch_check(ioctx, handle));
+
+  // a subsequent notify should not reach us
+  char *reply_buf = nullptr;
+  size_t reply_buf_len;
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+			     "notify", 6, 300000,
+			     &reply_buf, &reply_buf_len));
+  {
+    bufferlist reply;
+    reply.append(reply_buf, reply_buf_len);
+    std::map<std::pair<uint64_t,uint64_t>, bufferlist> reply_map;
+    std::set<std::pair<uint64_t,uint64_t> > missed_map;
+    auto reply_p = reply.cbegin();
+    decode(reply_map, reply_p);
+    decode(missed_map, reply_p);
+    ASSERT_EQ(0u, reply_map.size());
+    ASSERT_EQ(0u, missed_map.size());
+  }
+  ASSERT_EQ(0u, notify_cookies.size());
+  ASSERT_EQ(-ENOTCONN, rados_watch_check(ioctx, handle));
+  rados_buffer_free(reply_buf);
+
+  // re-watch
+  rados_unwatch2(ioctx, handle);
+  rados_watch_flush(cluster);
+
+  handle = 0;
+  ASSERT_EQ(0,
+	    rados_watch2(ioctx, notify_oid, &handle,
+			 watch_notify2_test_cb,
+			 watch_notify2_test_errcb, this));
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+
+  // and now a notify will work.
+  ASSERT_EQ(0, rados_notify2(ioctx, notify_oid,
+			     "notify", 6, 300000,
+			     &reply_buf, &reply_buf_len));
+  {
+    bufferlist reply;
+    reply.append(reply_buf, reply_buf_len);
+    std::map<std::pair<uint64_t,uint64_t>, bufferlist> reply_map;
+    std::set<std::pair<uint64_t,uint64_t> > missed_map;
+    auto reply_p = reply.cbegin();
+    decode(reply_map, reply_p);
+    decode(missed_map, reply_p);
+    ASSERT_EQ(1u, reply_map.size());
+    ASSERT_EQ(0u, missed_map.size());
+    ASSERT_EQ(1u, notify_cookies.count(handle));
+    ASSERT_EQ(5u, reply_map.begin()->second.length());
+    ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  }
+  ASSERT_EQ(1u, notify_cookies.size());
+  ASSERT_GT(rados_watch_check(ioctx, handle), 0);
+
+  rados_buffer_free(reply_buf);
+  rados_unwatch2(ioctx, handle);
+  rados_watch_flush(cluster);
+}
+
+TEST_F(LibRadosWatchNotify, AioWatchDelete2) {
+  notify_io = ioctx;
+  notify_oid = "foo";
+  notify_err = 0;
+  char buf[128];
+  uint32_t timeout = 3;
+  memset(buf, 0xcc, sizeof(buf));
+  ASSERT_EQ(0, rados_write(ioctx, notify_oid, buf, sizeof(buf), 0));
+
+
+  rados_completion_t comp;
+  uint64_t handle;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &comp));
+  rados_aio_watch2(ioctx, notify_oid, comp, &handle,
+                  watch_notify2_test_cb, watch_notify2_test_errcb, timeout, this);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(comp));
+  ASSERT_EQ(0, rados_aio_get_return_value(comp));
+  rados_aio_release(comp);
+  ASSERT_EQ(0, rados_remove(ioctx, notify_oid));
+  int left = 30;
+  std::cout << "waiting up to " << left << " for disconnect notification ..."
+      << std::endl;
+  while (notify_err == 0 && --left) {
+    sleep(1);
+  }
+  ASSERT_TRUE(left > 0);
+  ASSERT_EQ(-ENOTCONN, notify_err);
+  int rados_watch_check_err = rados_watch_check(ioctx, handle);
+  // We may hit ENOENT due to socket failure injection and a forced reconnect
+  EXPECT_TRUE(rados_watch_check_err == -ENOTCONN || rados_watch_check_err == -ENOENT)
+    << "Where rados_watch_check_err = " << rados_watch_check_err;
+  ASSERT_EQ(0, rados_aio_create_completion2(nullptr, nullptr, &comp));
+  rados_aio_unwatch(ioctx, handle, comp);
+  ASSERT_EQ(0, rados_aio_wait_for_complete(comp));
+  ASSERT_EQ(-ENOENT, rados_aio_get_return_value(comp));
+  rados_aio_release(comp);
+}
diff --git a/src/test/librados/watch_notify_cxx.cc b/src/test/librados/watch_notify_cxx.cc
new file mode 100644
index 000000000..808384bcc
--- /dev/null
+++ b/src/test/librados/watch_notify_cxx.cc
@@ -0,0 +1,416 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <semaphore.h>
+#include <set>
+#include <map>
+
+#include "gtest/gtest.h"
+
+#include "include/encoding.h"
+#include "include/rados/librados.hpp"
+#include "include/rados/rados_types.h"
+#include "test/librados/test_cxx.h"
+#include "test/librados/testcase_cxx.h"
+#include "crimson_utils.h"
+
+using namespace librados;
+
+typedef RadosTestECPP LibRadosWatchNotifyECPP;
+
+int notify_sleep = 0;
+
+#pragma GCC diagnostic ignored "-Wpragmas"
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+
+class LibRadosWatchNotifyPP : public RadosTestParamPP
+{
+protected:
+  bufferlist notify_bl;
+  std::set<uint64_t> notify_cookies;
+  rados_ioctx_t notify_io;
+  const char *notify_oid = nullptr;
+  int notify_err = 0;
+
+  friend class WatchNotifyTestCtx2;
+  friend class WatchNotifyTestCtx2TimeOut;
+};
+
+IoCtx *notify_ioctx;
+
+class WatchNotifyTestCtx2 : public WatchCtx2
+{
+  LibRadosWatchNotifyPP *notify;
+
+public:
+  WatchNotifyTestCtx2(LibRadosWatchNotifyPP *notify)
+    : notify(notify)
+  {}
+
+  void handle_notify(uint64_t notify_id, uint64_t cookie, uint64_t notifier_gid,
+		     bufferlist& bl) override {
+    std::cout << __func__ << " cookie " << cookie << " notify_id " << notify_id
+	      << " notifier_gid " << notifier_gid << std::endl;
+    notify->notify_bl = bl;
+    notify->notify_cookies.insert(cookie);
+    bufferlist reply;
+    reply.append("reply", 5);
+    if (notify_sleep)
+      sleep(notify_sleep);
+    notify_ioctx->notify_ack(notify->notify_oid, notify_id, cookie, reply);
+  }
+
+  void handle_error(uint64_t cookie, int err) override {
+    std::cout << __func__ << " cookie " << cookie
+	      << " err " << err << std::endl;
+    ceph_assert(cookie > 1000);
+    notify_ioctx->unwatch2(cookie);
+    notify->notify_cookies.erase(cookie);
+    notify->notify_err = notify_ioctx->watch2(notify->notify_oid, &cookie, this);
+    if (notify->notify_err < err ) {
+      std::cout << "reconnect notify_err " << notify->notify_err << " err " << err << std::endl;
+    }
+  }
+};
+
+class WatchNotifyTestCtx2TimeOut : public WatchCtx2
+{
+  LibRadosWatchNotifyPP *notify;
+
+public:
+  WatchNotifyTestCtx2TimeOut(LibRadosWatchNotifyPP *notify)
+    : notify(notify)
+  {}
+
+  void handle_notify(uint64_t notify_id, uint64_t cookie, uint64_t notifier_gid,
+		     bufferlist& bl) override {
+    std::cout << __func__ << " cookie " << cookie << " notify_id " << notify_id
+	      << " notifier_gid " << notifier_gid << std::endl;
+    notify->notify_bl = bl;
+    notify->notify_cookies.insert(cookie);
+    bufferlist reply;
+    reply.append("reply", 5);
+    if (notify_sleep)
+      sleep(notify_sleep);
+    notify_ioctx->notify_ack(notify->notify_oid, notify_id, cookie, reply);
+  }
+
+  void handle_error(uint64_t cookie, int err) override {
+    std::cout << __func__ << " cookie " << cookie
+	      << " err " << err << std::endl;
+    ceph_assert(cookie > 1000);
+    notify->notify_err = err;
+  }
+};
+
+// notify
+static sem_t sem;
+
+class WatchNotifyTestCtx : public WatchCtx
+{
+public:
+  void notify(uint8_t opcode, uint64_t ver, bufferlist& bl) override
+  {
+    std::cout << __func__ << std::endl;
+    sem_post(&sem);
+  }
+};
+
+TEST_P(LibRadosWatchNotifyPP, WatchNotify) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  uint64_t handle;
+  WatchNotifyTestCtx ctx;
+  ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
+  std::list<obj_watch_t> watches;
+  ASSERT_EQ(0, ioctx.list_watchers("foo", &watches));
+  ASSERT_EQ(1u, watches.size());
+  bufferlist bl2;
+  for (unsigned i=0; i<10; ++i) {
+    int r = ioctx.notify("foo", 0, bl2);
+    if (r == 0) {
+      break;
+    }
+    if (!getenv("ALLOW_TIMEOUTS")) {
+      ASSERT_EQ(0, r);
+    }
+  }
+  TestAlarm alarm;
+  sem_wait(&sem);
+  ioctx.unwatch("foo", handle);
+  sem_destroy(&sem);
+}
+
+TEST_F(LibRadosWatchNotifyECPP, WatchNotify) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+  uint64_t handle;
+  WatchNotifyTestCtx ctx;
+  ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
+  std::list<obj_watch_t> watches;
+  ASSERT_EQ(0, ioctx.list_watchers("foo", &watches));
+  ASSERT_EQ(1u, watches.size());
+  bufferlist bl2;
+  for (unsigned i=0; i<10; ++i) {
+    int r = ioctx.notify("foo", 0, bl2);
+    if (r == 0) {
+      break;
+    }
+    if (!getenv("ALLOW_TIMEOUTS")) {
+      ASSERT_EQ(0, r);
+    }
+  }
+  TestAlarm alarm;
+  sem_wait(&sem);
+  ioctx.unwatch("foo", handle);
+  sem_destroy(&sem);
+}
+
+// --
+
+TEST_P(LibRadosWatchNotifyPP, WatchNotifyTimeout) {
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  ioctx.set_notify_timeout(1);
+  uint64_t handle;
+  WatchNotifyTestCtx ctx;
+
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
+  sem_destroy(&sem);
+  ASSERT_EQ(0, ioctx.unwatch("foo", handle));
+}
+
+TEST_F(LibRadosWatchNotifyECPP, WatchNotifyTimeout) {
+  SKIP_IF_CRIMSON();
+  ASSERT_EQ(0, sem_init(&sem, 0, 0));
+  ioctx.set_notify_timeout(1);
+  uint64_t handle;
+  WatchNotifyTestCtx ctx;
+
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write("foo", bl1, sizeof(buf), 0));
+
+  ASSERT_EQ(0, ioctx.watch("foo", 0, &handle, &ctx));
+  sem_destroy(&sem);
+  ASSERT_EQ(0, ioctx.unwatch("foo", handle));
+}
+
+#pragma GCC diagnostic pop
+#pragma GCC diagnostic warning "-Wpragmas"
+
+TEST_P(LibRadosWatchNotifyPP, WatchNotify2) {
+  notify_oid = "foo";
+  notify_ioctx = &ioctx;
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write(notify_oid, bl1, sizeof(buf), 0));
+  uint64_t handle;
+  WatchNotifyTestCtx2 ctx(this);
+  ASSERT_EQ(0, ioctx.watch2(notify_oid, &handle, &ctx));
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+  std::list<obj_watch_t> watches;
+  ASSERT_EQ(0, ioctx.list_watchers(notify_oid, &watches));
+  ASSERT_EQ(watches.size(), 1u);
+  bufferlist bl2, bl_reply;
+  ASSERT_EQ(0, ioctx.notify2(notify_oid, bl2, 300000, &bl_reply));
+  auto p = bl_reply.cbegin();
+  std::map<std::pair<uint64_t,uint64_t>,bufferlist> reply_map;
+  std::set<std::pair<uint64_t,uint64_t> > missed_map;
+  decode(reply_map, p);
+  decode(missed_map, p);
+  ASSERT_EQ(1u, notify_cookies.size());
+  ASSERT_EQ(1u, notify_cookies.count(handle));
+  ASSERT_EQ(1u, reply_map.size());
+  ASSERT_EQ(5u, reply_map.begin()->second.length());
+  ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  ASSERT_EQ(0u, missed_map.size());
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+  ioctx.unwatch2(handle);
+}
+
+TEST_P(LibRadosWatchNotifyPP, AioWatchNotify2) {
+  notify_oid = "foo";
+  notify_ioctx = &ioctx;
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write(notify_oid, bl1, sizeof(buf), 0));
+
+  uint64_t handle;
+  WatchNotifyTestCtx2 ctx(this);
+  librados::AioCompletion *comp = cluster.aio_create_completion();
+  ASSERT_EQ(0, ioctx.aio_watch(notify_oid, comp, &handle, &ctx));
+  ASSERT_EQ(0, comp->wait_for_complete());
+  ASSERT_EQ(0, comp->get_return_value());
+  comp->release();
+
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+  std::list<obj_watch_t> watches;
+  ASSERT_EQ(0, ioctx.list_watchers(notify_oid, &watches));
+  ASSERT_EQ(watches.size(), 1u);
+  bufferlist bl2, bl_reply;
+  ASSERT_EQ(0, ioctx.notify2(notify_oid, bl2, 300000, &bl_reply));
+  auto p = bl_reply.cbegin();
+  std::map<std::pair<uint64_t,uint64_t>,bufferlist> reply_map;
+  std::set<std::pair<uint64_t,uint64_t> > missed_map;
+  decode(reply_map, p);
+  decode(missed_map, p);
+  ASSERT_EQ(1u, notify_cookies.size());
+  ASSERT_EQ(1u, notify_cookies.count(handle));
+  ASSERT_EQ(1u, reply_map.size());
+  ASSERT_EQ(5u, reply_map.begin()->second.length());
+  ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  ASSERT_EQ(0u, missed_map.size());
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+
+  comp = cluster.aio_create_completion();
+  ioctx.aio_unwatch(handle, comp);
+  ASSERT_EQ(0, comp->wait_for_complete());
+  comp->release();
+}
+
+
+TEST_P(LibRadosWatchNotifyPP, AioNotify) {
+  notify_oid = "foo";
+  notify_ioctx = &ioctx;
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write(notify_oid, bl1, sizeof(buf), 0));
+  uint64_t handle;
+  WatchNotifyTestCtx2 ctx(this);
+  ASSERT_EQ(0, ioctx.watch2(notify_oid, &handle, &ctx));
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+  std::list<obj_watch_t> watches;
+  ASSERT_EQ(0, ioctx.list_watchers(notify_oid, &watches));
+  ASSERT_EQ(watches.size(), 1u);
+  bufferlist bl2, bl_reply;
+  librados::AioCompletion *comp = cluster.aio_create_completion();
+  ASSERT_EQ(0, ioctx.aio_notify(notify_oid, comp, bl2, 300000, &bl_reply));
+  ASSERT_EQ(0, comp->wait_for_complete());
+  ASSERT_EQ(0, comp->get_return_value());
+  comp->release();
+  std::vector<librados::notify_ack_t> acks;
+  std::vector<librados::notify_timeout_t> timeouts;
+  ioctx.decode_notify_response(bl_reply, &acks, &timeouts);
+  ASSERT_EQ(1u, notify_cookies.size());
+  ASSERT_EQ(1u, notify_cookies.count(handle));
+  ASSERT_EQ(1u, acks.size());
+  ASSERT_EQ(5u, acks[0].payload_bl.length());
+  ASSERT_EQ(0, strncmp("reply", acks[0].payload_bl.c_str(), acks[0].payload_bl.length()));
+  ASSERT_EQ(0u, timeouts.size());
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+  ioctx.unwatch2(handle);
+  cluster.watch_flush();
+}
+
+// --
+TEST_P(LibRadosWatchNotifyPP, WatchNotify2Timeout) {
+  notify_oid = "foo";
+  notify_ioctx = &ioctx;
+  notify_sleep = 3;  // 3s
+  notify_cookies.clear();
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write(notify_oid, bl1, sizeof(buf), 0));
+  uint64_t handle;
+  WatchNotifyTestCtx2TimeOut ctx(this);
+  ASSERT_EQ(0, ioctx.watch2(notify_oid, &handle, &ctx));
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+  std::list<obj_watch_t> watches;
+  ASSERT_EQ(0, ioctx.list_watchers(notify_oid, &watches));
+  ASSERT_EQ(watches.size(), 1u);
+  ASSERT_EQ(0u, notify_cookies.size());
+  bufferlist bl2, bl_reply;
+  std::cout << " trying..." << std::endl;
+  ASSERT_EQ(-ETIMEDOUT, ioctx.notify2(notify_oid, bl2, 1000 /* 1s */,
+				      &bl_reply));
+  std::cout << " timed out" << std::endl;
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+  ioctx.unwatch2(handle);
+
+  std::cout << " flushing" << std::endl;
+  librados::AioCompletion *comp = cluster.aio_create_completion();
+  cluster.aio_watch_flush(comp);
+  ASSERT_EQ(0, comp->wait_for_complete());
+  ASSERT_EQ(0, comp->get_return_value());
+  std::cout << " flushed" << std::endl;
+  comp->release();
+}
+
+TEST_P(LibRadosWatchNotifyPP, WatchNotify3) {
+  notify_oid = "foo";
+  notify_ioctx = &ioctx;
+  notify_cookies.clear();
+  uint32_t timeout = 12; // configured timeout
+  char buf[128];
+  memset(buf, 0xcc, sizeof(buf));
+  bufferlist bl1;
+  bl1.append(buf, sizeof(buf));
+  ASSERT_EQ(0, ioctx.write(notify_oid, bl1, sizeof(buf), 0));
+  uint64_t handle;
+  WatchNotifyTestCtx2TimeOut ctx(this);
+  ASSERT_EQ(0, ioctx.watch3(notify_oid, &handle, &ctx, timeout));
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+  std::list<obj_watch_t> watches;
+  ASSERT_EQ(0, ioctx.list_watchers(notify_oid, &watches));
+  ASSERT_EQ(watches.size(), 1u);
+  std::cout << "List watches" << std::endl;
+  for (std::list<obj_watch_t>::iterator it = watches.begin();
+    it != watches.end(); ++it) {
+    ASSERT_EQ(it->timeout_seconds, timeout);
+  }
+  bufferlist bl2, bl_reply;
+  std::cout << "notify2" << std::endl;
+  ASSERT_EQ(0, ioctx.notify2(notify_oid, bl2, 300000, &bl_reply));
+  std::cout << "notify2 done" << std::endl;
+  auto p = bl_reply.cbegin();
+  std::map<std::pair<uint64_t,uint64_t>,bufferlist> reply_map;
+  std::set<std::pair<uint64_t,uint64_t> > missed_map;
+  decode(reply_map, p);
+  decode(missed_map, p);
+  ASSERT_EQ(1u, notify_cookies.size());
+  ASSERT_EQ(1u, notify_cookies.count(handle));
+  ASSERT_EQ(1u, reply_map.size());
+  ASSERT_EQ(5u, reply_map.begin()->second.length());
+  ASSERT_EQ(0, strncmp("reply", reply_map.begin()->second.c_str(), 5));
+  ASSERT_EQ(0u, missed_map.size());
+  std::cout << "watch_check" << std::endl;
+  ASSERT_GT(ioctx.watch_check(handle), 0);
+  std::cout << "unwatch2" << std::endl;
+  ioctx.unwatch2(handle);
+
+  std::cout << " flushing" << std::endl;
+  cluster.watch_flush();
+  std::cout << "done" << std::endl;
+}
+// --
+
+INSTANTIATE_TEST_SUITE_P(LibRadosWatchNotifyPPTests, LibRadosWatchNotifyPP,
+			::testing::Values("", "cache"));