20 files changed, 10492 insertions, 0 deletions
diff --git a/src/test/libcephfs/CMakeLists.txt b/src/test/libcephfs/CMakeLists.txt
new file mode 100644
index 000000000..672e6dd8f
--- /dev/null
+++ b/src/test/libcephfs/CMakeLists.txt
@@ -0,0 +1,96 @@
+if(WITH_LIBCEPHFS)
+  add_executable(ceph_test_libcephfs
+    test.cc
+    readdir_r_cb.cc
+    caps.cc
+    multiclient.cc
+    flock.cc
+    recordlock.cc
+    acl.cc
+    main.cc
+    deleg.cc
+    monconfig.cc
+    vxattr.cc
+    snapdiff.cc
+  )
+  target_link_libraries(ceph_test_libcephfs
+    ceph-common
+    cephfs
+    ${UNITTEST_LIBS}
+    ${EXTRALIBS}
+    ${CMAKE_DL_LIBS}
+    )
+  install(TARGETS ceph_test_libcephfs
+    DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+  add_executable(ceph_test_libcephfs_suidsgid
+    suidsgid.cc
+  )
+  target_link_libraries(ceph_test_libcephfs_suidsgid
+    ceph-common
+    cephfs
+    librados
+    ${UNITTEST_LIBS}
+    ${EXTRALIBS}
+    ${CMAKE_DL_LIBS}
+    )
+  install(TARGETS ceph_test_libcephfs_suidsgid
+    DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+  add_executable(ceph_test_libcephfs_newops
+    main.cc
+    newops.cc
+  )
+  target_link_libraries(ceph_test_libcephfs_newops
+    ceph-common
+    cephfs
+    ${UNITTEST_LIBS}
+    ${EXTRALIBS}
+    ${CMAKE_DL_LIBS}
+    )
+  install(TARGETS ceph_test_libcephfs_newops
+    DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+  # uses fork, not available on Windows
+  if(NOT WIN32)
+    add_executable(ceph_test_libcephfs_reclaim
+      reclaim.cc
+    )
+    target_link_libraries(ceph_test_libcephfs_reclaim
+      cephfs
+      ${UNITTEST_LIBS}
+      ${EXTRALIBS}
+      ${CMAKE_DL_LIBS}
+      )
+    install(TARGETS ceph_test_libcephfs_reclaim
+      DESTINATION ${CMAKE_INSTALL_BINDIR})
+  endif(NOT WIN32)
+
+  add_executable(ceph_test_libcephfs_lazyio
+    lazyio.cc
+  )
+  target_link_libraries(ceph_test_libcephfs_lazyio
+    cephfs
+    librados
+    ${UNITTEST_LIBS}
+    ${EXTRALIBS}
+    ${CMAKE_DL_LIBS}
+    )   
+  install(TARGETS ceph_test_libcephfs_lazyio
+    DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+  add_executable(ceph_test_libcephfs_access
+    test.cc
+    access.cc
+  )
+  target_link_libraries(ceph_test_libcephfs_access
+    ceph-common
+    cephfs
+    librados
+    ${UNITTEST_LIBS}
+    ${EXTRALIBS}
+    ${CMAKE_DL_LIBS}
+    )
+  install(TARGETS ceph_test_libcephfs_access
+    DESTINATION ${CMAKE_INSTALL_BINDIR})
+endif(WITH_LIBCEPHFS)
diff --git a/src/test/libcephfs/access.cc b/src/test/libcephfs/access.cc
new file mode 100644
index 000000000..57b1a89fa
--- /dev/null
+++ b/src/test/libcephfs/access.cc
@@ -0,0 +1,399 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "common/ceph_argparse.h"
+#include "include/buffer.h"
+#include "include/stringify.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include "include/rados/librados.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <sys/uio.h>
+#include <iostream>
+#include <vector>
+#include "json_spirit/json_spirit.h"
+
+#include "include/fs_types.h"
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#endif
+
+using namespace std;
+
+rados_t cluster;
+
+string key;
+
+int do_mon_command(string s, string *key)
+{
+  char *outs, *outbuf;
+  size_t outs_len, outbuf_len;
+  const char *ss = s.c_str();
+  int r = rados_mon_command(cluster, (const char **)&ss, 1,
+			    0, 0,
+			    &outbuf, &outbuf_len,
+			    &outs, &outs_len);
+  if (outbuf_len) {
+    string s(outbuf, outbuf_len);
+    std::cout << "out: " << s << std::endl;
+
+    // parse out the key
+    json_spirit::mValue v, k;
+    json_spirit::read_or_throw(s, v);
+    k = v.get_array()[0].get_obj().find("key")->second;
+    *key = k.get_str();
+    std::cout << "key: " << *key << std::endl;
+    free(outbuf);
+  } else {
+    return -CEPHFS_EINVAL;
+  }
+  if (outs_len) {
+    string s(outs, outs_len);
+    std::cout << "outs: " << s << std::endl;
+    free(outs);
+  }
+  return r;
+}
+
+string get_unique_dir()
+{
+  return string("/ceph_test_libcephfs_access.") + stringify(rand());
+}
+
+TEST(AccessTest, Foo) {
+  string dir = get_unique_dir();
+  string user = "libcephfs_foo_test." + stringify(rand());
+  // admin mount to set up test
+  struct ceph_mount_info *admin;
+  ASSERT_EQ(0, ceph_create(&admin, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(admin, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(admin, NULL));
+  ASSERT_EQ(0, ceph_mount(admin, "/"));
+  ASSERT_EQ(0, ceph_mkdir(admin, dir.c_str(), 0755));
+
+  // create access key
+  string key;
+  ASSERT_EQ(0, do_mon_command(
+      "{\"prefix\": \"auth get-or-create\", \"entity\": \"client." + user + "\", "
+      "\"caps\": [\"mon\", \"allow *\", \"osd\", \"allow rw\", "
+      "\"mds\", \"allow rw\""
+      "], \"format\": \"json\"}", &key));
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, user.c_str()));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "key", key.c_str()));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ceph_shutdown(cmount);
+
+  // clean up
+  ASSERT_EQ(0, ceph_rmdir(admin, dir.c_str()));
+  ceph_shutdown(admin);
+}
+
+TEST(AccessTest, Path) {
+  string good = get_unique_dir();
+  string bad = get_unique_dir();
+  string user = "libcephfs_path_test." + stringify(rand());
+  struct ceph_mount_info *admin;
+  ASSERT_EQ(0, ceph_create(&admin, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(admin, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(admin, NULL));
+  ASSERT_EQ(0, ceph_mount(admin, "/"));
+  ASSERT_EQ(0, ceph_mkdir(admin, good.c_str(), 0755));
+  ASSERT_EQ(0, ceph_mkdir(admin, string(good + "/p").c_str(), 0755));
+  ASSERT_EQ(0, ceph_mkdir(admin, bad.c_str(), 0755));
+  ASSERT_EQ(0, ceph_mkdir(admin, string(bad + "/p").c_str(), 0755));
+  int fd = ceph_open(admin, string(good + "/q").c_str(), O_CREAT|O_WRONLY, 0755);
+  ceph_close(admin, fd);
+  fd = ceph_open(admin, string(bad + "/q").c_str(), O_CREAT|O_WRONLY, 0755);
+  ceph_close(admin, fd);
+  fd = ceph_open(admin, string(bad + "/z").c_str(), O_CREAT|O_WRONLY, 0755);
+  ceph_write(admin, fd, "TEST FAILED", 11, 0);
+  ceph_close(admin, fd);
+
+  string key;
+  ASSERT_EQ(0, do_mon_command(
+      "{\"prefix\": \"auth get-or-create\", \"entity\": \"client." + user + "\", "
+      "\"caps\": [\"mon\", \"allow r\", \"osd\", \"allow rwx\", "
+      "\"mds\", \"allow r, allow rw path=" + good + "\""
+      "], \"format\": \"json\"}", &key));
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, user.c_str()));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "key", key.c_str()));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  // allowed
+  ASSERT_GE(ceph_mkdir(cmount, string(good + "/x").c_str(), 0755), 0);
+  ASSERT_GE(ceph_rmdir(cmount, string(good + "/p").c_str()), 0);
+  ASSERT_GE(ceph_unlink(cmount, string(good + "/q").c_str()), 0);
+  fd = ceph_open(cmount, string(good + "/y").c_str(), O_CREAT|O_WRONLY, 0755);
+  ASSERT_GE(fd, 0);
+  ceph_write(cmount, fd, "bar", 3, 0);
+  ceph_close(cmount, fd);
+  ASSERT_GE(ceph_unlink(cmount, string(good + "/y").c_str()), 0);
+  ASSERT_GE(ceph_rmdir(cmount, string(good + "/x").c_str()), 0);
+
+  fd = ceph_open(cmount, string(bad + "/z").c_str(), O_RDONLY, 0644);
+  ASSERT_GE(fd, 0);
+  ceph_close(cmount, fd);
+
+  // not allowed
+  ASSERT_LT(ceph_mkdir(cmount, string(bad + "/x").c_str(), 0755), 0);
+  ASSERT_LT(ceph_rmdir(cmount, string(bad + "/p").c_str()), 0);
+  ASSERT_LT(ceph_unlink(cmount, string(bad + "/q").c_str()), 0);
+  fd = ceph_open(cmount, string(bad + "/y").c_str(), O_CREAT|O_WRONLY, 0755);
+  ASSERT_LT(fd, 0);
+
+  // unlink open file
+  fd = ceph_open(cmount, string(good + "/unlinkme").c_str(), O_CREAT|O_WRONLY, 0755);
+  ceph_unlink(cmount, string(good + "/unlinkme").c_str());
+  ASSERT_GE(ceph_write(cmount, fd, "foo", 3, 0), 0);
+  ASSERT_GE(ceph_fchmod(cmount, fd, 0777), 0);
+  ASSERT_GE(ceph_ftruncate(cmount, fd, 0), 0);
+  ASSERT_GE(ceph_fsetxattr(cmount, fd, "user.any", "bar", 3, 0), 0);
+  ceph_close(cmount, fd);
+
+  // rename open file
+  fd = ceph_open(cmount, string(good + "/renameme").c_str(), O_CREAT|O_WRONLY, 0755);
+  ASSERT_EQ(ceph_rename(admin, string(good + "/renameme").c_str(),
+			string(bad + "/asdf").c_str()), 0);
+  ASSERT_GE(ceph_write(cmount, fd, "foo", 3, 0), 0);
+  ASSERT_GE(ceph_fchmod(cmount, fd, 0777), -CEPHFS_EACCES);
+  ASSERT_GE(ceph_ftruncate(cmount, fd, 0), -CEPHFS_EACCES);
+  ASSERT_GE(ceph_fsetxattr(cmount, fd, "user.any", "bar", 3, 0), -CEPHFS_EACCES);
+  ceph_close(cmount, fd);
+
+  ceph_shutdown(cmount);
+  ASSERT_EQ(0, ceph_unlink(admin, string(bad + "/q").c_str()));
+  ASSERT_EQ(0, ceph_unlink(admin, string(bad + "/z").c_str()));
+  ASSERT_EQ(0, ceph_rmdir(admin, string(bad + "/p").c_str()));
+  ASSERT_EQ(0, ceph_unlink(admin, string(bad + "/asdf").c_str()));
+  ASSERT_EQ(0, ceph_rmdir(admin, good.c_str()));
+  ASSERT_EQ(0, ceph_rmdir(admin, bad.c_str()));
+  ceph_shutdown(admin);
+}
+
+TEST(AccessTest, ReadOnly) {
+  string dir = get_unique_dir();
+  string dir2 = get_unique_dir();
+  string user = "libcephfs_readonly_test." + stringify(rand());
+  struct ceph_mount_info *admin;
+  ASSERT_EQ(0, ceph_create(&admin, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(admin, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(admin, NULL));
+  ASSERT_EQ(0, ceph_mount(admin, "/"));
+  ASSERT_EQ(0, ceph_mkdir(admin, dir.c_str(), 0755));
+  int fd = ceph_open(admin, string(dir + "/out").c_str(), O_CREAT|O_WRONLY, 0755);
+  ceph_write(admin, fd, "foo", 3, 0);
+  ceph_close(admin,fd);
+
+  string key;
+  ASSERT_EQ(0, do_mon_command(
+      "{\"prefix\": \"auth get-or-create\", \"entity\": \"client." + user + "\", "
+      "\"caps\": [\"mon\", \"allow r\", \"osd\", \"allow rw\", "
+      "\"mds\", \"allow r\""
+      "], \"format\": \"json\"}", &key));
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, user.c_str()));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "key", key.c_str()));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  // allowed
+  fd = ceph_open(cmount, string(dir + "/out").c_str(), O_RDONLY, 0644);
+  ASSERT_GE(fd, 0);
+  ceph_close(cmount,fd);
+
+  // not allowed
+  fd = ceph_open(cmount, string(dir + "/bar").c_str(), O_CREAT|O_WRONLY, 0755);
+  ASSERT_LT(fd, 0);
+  ASSERT_LT(ceph_mkdir(cmount, dir2.c_str(), 0755), 0);
+
+  ceph_shutdown(cmount);
+  ASSERT_EQ(0, ceph_unlink(admin, string(dir + "/out").c_str()));
+  ASSERT_EQ(0, ceph_rmdir(admin, dir.c_str()));
+  ceph_shutdown(admin);
+}
+
+TEST(AccessTest, User) {
+  string dir = get_unique_dir();
+  string user = "libcephfs_user_test." + stringify(rand());
+
+  // admin mount to set up test
+  struct ceph_mount_info *admin;
+  ASSERT_EQ(0, ceph_create(&admin, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(admin, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(admin, NULL));
+  ASSERT_EQ(0, ceph_conf_set(admin, "client_permissions", "0"));
+  ASSERT_EQ(0, ceph_mount(admin, "/"));
+  ASSERT_EQ(0, ceph_mkdir(admin, dir.c_str(), 0755));
+
+  // create access key
+  string key;
+  ASSERT_EQ(0, do_mon_command(
+      "{\"prefix\": \"auth get-or-create\", \"entity\": \"client." + user + "\", "
+      "\"caps\": [\"mon\", \"allow *\", \"osd\", \"allow rw\", "
+      "\"mds\", \"allow rw uid=123 gids=456,789\""
+      "], \"format\": \"json\"}", &key));
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, user.c_str()));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "key", key.c_str()));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_init(cmount));
+
+  UserPerm *perms = ceph_userperm_new(123, 456, 0, NULL);
+  ASSERT_NE(nullptr, perms);
+  ASSERT_EQ(0, ceph_mount_perms_set(cmount, perms));
+  ceph_userperm_destroy(perms);
+
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_permissions", "0"));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  // user bits
+  ASSERT_EQ(0, ceph_chmod(admin, dir.c_str(), 0700));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 123, 456));
+  ASSERT_EQ(0, ceph_mkdir(cmount, string(dir + "/u1").c_str(), 0755));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 1, 456));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_mkdir(cmount, string(dir + "/no").c_str(), 0755));
+
+  // group bits
+  ASSERT_EQ(0, ceph_chmod(admin, dir.c_str(), 0770));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 1, 456));
+  ASSERT_EQ(0, ceph_mkdir(cmount, string(dir + "/u2").c_str(), 0755));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 1, 2));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_mkdir(cmount, string(dir + "/no").c_str(), 0755));
+
+  // user overrides group
+  ASSERT_EQ(0, ceph_chmod(admin, dir.c_str(), 0470));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 123, 456));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_mkdir(cmount, string(dir + "/no").c_str(), 0755));
+
+  // other
+  ASSERT_EQ(0, ceph_chmod(admin, dir.c_str(), 0777));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 1, 1));
+  ASSERT_EQ(0, ceph_mkdir(cmount, string(dir + "/u3").c_str(), 0755));
+  ASSERT_EQ(0, ceph_chmod(admin, dir.c_str(), 0770));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_mkdir(cmount, string(dir + "/no").c_str(), 0755));
+
+  // user and group overrides other
+  ASSERT_EQ(0, ceph_chmod(admin, dir.c_str(), 07));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 1, 456));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_mkdir(cmount, string(dir + "/no").c_str(), 0755));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 123, 1));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_mkdir(cmount, string(dir + "/no").c_str(), 0755));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 123, 456));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_mkdir(cmount, string(dir + "/no").c_str(), 0755));
+
+  // chown and chgrp
+  ASSERT_EQ(0, ceph_chmod(admin, dir.c_str(), 0700));
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 123, 456));
+  // FIXME: Re-enable these 789 tests once we can set multiple GIDs via libcephfs/config
+  // ASSERT_EQ(0, ceph_chown(cmount, dir.c_str(), 123, 789));
+  ASSERT_EQ(0, ceph_chown(cmount, dir.c_str(), 123, 456));
+  // ASSERT_EQ(0, ceph_chown(cmount, dir.c_str(), -1, 789));
+  ASSERT_EQ(0, ceph_chown(cmount, dir.c_str(), -1, 456));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_chown(cmount, dir.c_str(), 123, 1));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_chown(cmount, dir.c_str(), 1, 456));
+
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 1, 1));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_chown(cmount, dir.c_str(), 123, 456));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_chown(cmount, dir.c_str(), 123, -1));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_chown(cmount, dir.c_str(), -1, 456));
+
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 1, 456));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_chown(cmount, dir.c_str(), 123, 456));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_chown(cmount, dir.c_str(), 123, -1));
+  ASSERT_EQ(-CEPHFS_EACCES, ceph_chown(cmount, dir.c_str(), -1, 456));
+
+  ASSERT_EQ(0, ceph_chown(admin, dir.c_str(), 123, 1));
+  ASSERT_EQ(0, ceph_chown(cmount, dir.c_str(), -1, 456));
+  // ASSERT_EQ(0, ceph_chown(cmount, dir.c_str(), 123, 789));
+
+  ceph_shutdown(cmount);
+
+  // clean up
+  ASSERT_EQ(0, ceph_rmdir(admin, string(dir + "/u1").c_str()));
+  ASSERT_EQ(0, ceph_rmdir(admin, string(dir + "/u2").c_str()));
+  ASSERT_EQ(0, ceph_rmdir(admin, string(dir + "/u3").c_str()));
+  ASSERT_EQ(0, ceph_rmdir(admin, dir.c_str()));
+  ceph_shutdown(admin);
+}
+
+static int update_root_mode()
+{
+  struct ceph_mount_info *admin;
+  int r = ceph_create(&admin, NULL);
+  if (r < 0)
+    return r;
+  ceph_conf_read_file(admin, NULL);
+  ceph_conf_parse_env(admin, NULL);
+  ceph_conf_set(admin, "client_permissions", "false");
+  r = ceph_mount(admin, "/");
+  if (r < 0)
+    goto out;
+  r = ceph_chmod(admin, "/", 0777);
+out:
+  ceph_shutdown(admin);
+  return r;
+}
+
+
+int main(int argc, char **argv)
+{
+  int r = update_root_mode();
+  if (r < 0)
+    exit(1);
+
+  ::testing::InitGoogleTest(&argc, argv);
+
+  srand(getpid());
+
+  r = rados_create(&cluster, NULL);
+  if (r < 0)
+    exit(1);
+  
+  r = rados_conf_read_file(cluster, NULL);
+  if (r < 0)
+    exit(1);
+
+  rados_conf_parse_env(cluster, NULL);
+  r = rados_connect(cluster);
+  if (r < 0)
+    exit(1);
+
+  r = RUN_ALL_TESTS();
+
+  rados_shutdown(cluster);
+
+  return r;
+}
diff --git a/src/test/libcephfs/acl.cc b/src/test/libcephfs/acl.cc
new file mode 100644
index 000000000..e263ef2fb
--- /dev/null
+++ b/src/test/libcephfs/acl.cc
@@ -0,0 +1,367 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "include/types.h"
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include "include/ceph_fs.h"
+#include "client/posix_acl.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#ifdef __linux__
+#include <sys/xattr.h>
+#endif
+
+static size_t acl_ea_size(int count)
+{
+  return sizeof(acl_ea_header) + count * sizeof(acl_ea_entry);
+}
+
+static int acl_ea_count(size_t size)
+{
+  if (size < sizeof(acl_ea_header))
+    return -1;
+  size -= sizeof(acl_ea_header);
+  if (size % sizeof(acl_ea_entry))
+    return -1;
+  return size / sizeof(acl_ea_entry);
+}
+
+static int check_acl_and_mode(const void *buf, size_t size, mode_t mode)
+{
+  const acl_ea_entry *group_entry = NULL, *mask_entry = NULL;
+  const acl_ea_header *header = reinterpret_cast<const acl_ea_header*>(buf);
+  const acl_ea_entry *entry = header->a_entries;
+  int count = (size - sizeof(*header)) / sizeof(*entry);
+  for (int i = 0; i < count; ++i) {
+    __u16 tag = entry->e_tag;
+    __u16 perm = entry->e_perm;
+    switch(tag) {
+      case ACL_USER_OBJ:
+	if (perm != ((mode >> 6) & 7))
+	  return -CEPHFS_EINVAL;
+	break;
+      case ACL_USER:
+      case ACL_GROUP:
+	break;
+      case ACL_GROUP_OBJ:
+	group_entry = entry;
+	break;
+      case ACL_OTHER:
+	if (perm != (mode & 7))
+	  return -CEPHFS_EINVAL;
+	break;
+      case ACL_MASK:
+	mask_entry = entry;
+	break;
+      default:
+	return -CEPHFS_EIO;
+    }
+    ++entry;
+  }
+  if (mask_entry) {
+    __u16 perm = mask_entry->e_perm;
+    if (perm != ((mode >> 3) & 7))
+      return -CEPHFS_EINVAL;
+  } else {
+    if (!group_entry)
+      return -CEPHFS_EIO;
+    __u16 perm = group_entry->e_perm;
+    if (perm != ((mode >> 3) & 7))
+      return -CEPHFS_EINVAL;
+  }
+  return 0;
+}
+
+static int generate_test_acl(void *buf, size_t size, mode_t mode)
+{
+  if (acl_ea_count(size) != 5)
+    return -1;
+  acl_ea_header *header = reinterpret_cast<acl_ea_header*>(buf);
+  header->a_version = (__u32)ACL_EA_VERSION;
+  acl_ea_entry *entry = header->a_entries;
+  entry->e_tag = ACL_USER_OBJ;
+  entry->e_perm = (mode >> 6) & 7;
+  ++entry;
+  entry->e_tag = ACL_USER;
+  entry->e_perm = 7;
+  entry->e_id = getuid();
+  ++entry;
+  entry->e_tag = ACL_GROUP_OBJ;
+  entry->e_perm = (mode >> 3) & 7;
+  ++entry;
+  entry->e_tag = ACL_MASK;
+  entry->e_perm = 7;
+  ++entry;
+  entry->e_tag = ACL_OTHER;
+  entry->e_perm = mode & 7;
+  return 0;
+}
+
+static int generate_empty_acl(void *buf, size_t size, mode_t mode)
+{
+
+ if (acl_ea_count(size) != 3)
+    return -1;
+  acl_ea_header *header = reinterpret_cast<acl_ea_header*>(buf);
+  header->a_version = (__u32)ACL_EA_VERSION;
+  acl_ea_entry *entry = header->a_entries;
+  entry->e_tag = ACL_USER_OBJ;
+  entry->e_perm = (mode >> 6) & 7;
+  ++entry;
+  entry->e_tag = ACL_GROUP_OBJ;
+  entry->e_perm = (mode >> 3) & 7;
+  ++entry;
+  entry->e_tag = ACL_OTHER;
+  entry->e_perm = mode & 7;
+  return 0;
+}
+
+TEST(ACL, SetACL) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_acl_type", "posix_acl"));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_permissions", "0"));
+
+  char test_file[256];
+  sprintf(test_file, "file1_setacl_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0600);
+  ASSERT_GT(fd, 0);
+  // change ownership to nobody -- we assume nobody exists and id is always 65534
+  ASSERT_EQ(ceph_fchown(cmount, fd, 65534, 65534), 0);
+
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_permissions", "1"));
+  // "nobody" will be ignored on Windows
+  #ifndef _WIN32
+  ASSERT_EQ(ceph_open(cmount, test_file, O_RDWR, 0), -CEPHFS_EACCES);
+  #endif
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_permissions", "0"));
+
+  size_t acl_buf_size = acl_ea_size(5);
+  void *acl_buf = malloc(acl_buf_size);
+  ASSERT_EQ(generate_test_acl(acl_buf, acl_buf_size, 0750), 0);
+
+  // can't set default acl for non-directory
+  ASSERT_EQ(ceph_fsetxattr(cmount, fd, ACL_EA_DEFAULT, acl_buf, acl_buf_size, 0), -CEPHFS_EACCES);
+  ASSERT_EQ(ceph_fsetxattr(cmount, fd, ACL_EA_ACCESS, acl_buf, acl_buf_size, 0), 0);
+
+  int tmpfd = ceph_open(cmount, test_file, O_RDWR, 0);
+  ASSERT_GT(tmpfd, 0);
+  ceph_close(cmount, tmpfd);
+
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_fstatx(cmount, fd, &stx, CEPH_STATX_MODE, 0), 0);
+  // mode was modified according to ACL
+  ASSERT_EQ(stx.stx_mode & 0777u, 0770u);
+  ASSERT_EQ(check_acl_and_mode(acl_buf, acl_buf_size, stx.stx_mode), 0);
+
+  acl_buf_size = acl_ea_size(3);
+  // setting ACL that is equivalent to file mode
+  ASSERT_EQ(generate_empty_acl(acl_buf, acl_buf_size, 0600), 0);
+  ASSERT_EQ(ceph_fsetxattr(cmount, fd, ACL_EA_ACCESS, acl_buf, acl_buf_size, 0), 0);
+  // ACL was deleted
+  ASSERT_EQ(ceph_fgetxattr(cmount, fd, ACL_EA_ACCESS, NULL, 0), -CEPHFS_ENODATA);
+
+  ASSERT_EQ(ceph_fstatx(cmount, fd, &stx, CEPH_STATX_MODE, 0), 0);
+  // mode was modified according to ACL
+  ASSERT_EQ(stx.stx_mode & 0777u, 0600u);
+
+  free(acl_buf);
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(ACL, Chmod) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_acl_type", "posix_acl"));
+
+  char test_file[256];
+  sprintf(test_file, "file1_acl_chmod_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0600);
+  ASSERT_GT(fd, 0);
+
+  int acl_buf_size = acl_ea_size(5);
+  void *acl_buf = malloc(acl_buf_size);
+  ASSERT_EQ(generate_test_acl(acl_buf, acl_buf_size, 0775), 0);
+  ASSERT_EQ(ceph_fsetxattr(cmount, fd, ACL_EA_ACCESS, acl_buf, acl_buf_size, 0), 0);
+
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_fstatx(cmount, fd, &stx, CEPH_STATX_MODE, 0), 0);
+  // mode was updated according to ACL
+  ASSERT_EQ(stx.stx_mode & 0777u, 0775u);
+
+  // change mode
+  ASSERT_EQ(ceph_fchmod(cmount, fd, 0640), 0);
+
+  ASSERT_EQ(ceph_fstatx(cmount, fd, &stx, CEPH_STATX_MODE, 0), 0);
+  ASSERT_EQ(stx.stx_mode & 0777u, 0640u);
+
+  // ACL was updated according to mode
+  ASSERT_EQ(ceph_fgetxattr(cmount, fd, ACL_EA_ACCESS, acl_buf, acl_buf_size), acl_buf_size);
+  ASSERT_EQ(check_acl_and_mode(acl_buf, acl_buf_size, stx.stx_mode), 0);
+
+  free(acl_buf);
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(ACL, DefaultACL) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_acl_type", "posix_acl"));
+
+  int acl_buf_size = acl_ea_size(5);
+  void *acl1_buf = malloc(acl_buf_size);
+  void *acl2_buf = malloc(acl_buf_size);
+
+  ASSERT_EQ(generate_test_acl(acl1_buf, acl_buf_size, 0750), 0);
+
+  char test_dir1[256];
+  sprintf(test_dir1, "dir1_acl_default_%d", getpid());
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir1, 0750), 0);
+
+  // set default acl
+  ASSERT_EQ(ceph_setxattr(cmount, test_dir1, ACL_EA_DEFAULT, acl1_buf, acl_buf_size, 0), 0);
+
+  char test_dir2[262];
+  sprintf(test_dir2, "%s/dir2", test_dir1);
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir2, 0755), 0);
+
+  // inherit default acl
+  ASSERT_EQ(ceph_getxattr(cmount, test_dir2, ACL_EA_DEFAULT, acl2_buf, acl_buf_size), acl_buf_size);
+  ASSERT_EQ(memcmp(acl1_buf, acl2_buf, acl_buf_size), 0);
+
+  // mode and ACL are updated
+  ASSERT_EQ(ceph_getxattr(cmount, test_dir2, ACL_EA_ACCESS, acl2_buf, acl_buf_size), acl_buf_size);
+  {
+    struct ceph_statx stx;
+    ASSERT_EQ(ceph_statx(cmount, test_dir2, &stx, CEPH_STATX_MODE, 0), 0);
+    // other bits of mode &= acl other perm
+    ASSERT_EQ(stx.stx_mode & 0777u, 0750u);
+    ASSERT_EQ(check_acl_and_mode(acl2_buf, acl_buf_size, stx.stx_mode), 0);
+  }
+
+  char test_file1[262];
+  sprintf(test_file1, "%s/file1", test_dir1);
+  int fd = ceph_open(cmount, test_file1, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  // no default acl
+  ASSERT_EQ(ceph_fgetxattr(cmount, fd, ACL_EA_DEFAULT, NULL, 0), -CEPHFS_ENODATA);
+
+  // mode and ACL are updated
+  ASSERT_EQ(ceph_fgetxattr(cmount, fd, ACL_EA_ACCESS, acl2_buf, acl_buf_size), acl_buf_size);
+  {
+    struct ceph_statx stx;
+    ASSERT_EQ(ceph_statx(cmount, test_file1, &stx, CEPH_STATX_MODE, 0), 0);
+    // other bits of mode &= acl other perm
+    ASSERT_EQ(stx.stx_mode & 0777u, 0660u);
+    ASSERT_EQ(check_acl_and_mode(acl2_buf, acl_buf_size, stx.stx_mode), 0);
+  }
+
+  free(acl1_buf);
+  free(acl2_buf);
+  ASSERT_EQ(ceph_unlink(cmount, test_file1), 0);
+  ASSERT_EQ(ceph_rmdir(cmount, test_dir2), 0);
+  ASSERT_EQ(ceph_rmdir(cmount, test_dir1), 0);
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(ACL, Disabled) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_acl_type", ""));
+
+  size_t acl_buf_size = acl_ea_size(3);
+  void *acl_buf = malloc(acl_buf_size);
+  ASSERT_EQ(generate_empty_acl(acl_buf, acl_buf_size, 0755), 0);
+
+  char test_dir[256];
+  sprintf(test_dir, "dir1_acl_disabled_%d", getpid());
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir, 0750), 0);
+
+  ASSERT_EQ(ceph_setxattr(cmount, test_dir, ACL_EA_DEFAULT, acl_buf, acl_buf_size, 0), -CEPHFS_EOPNOTSUPP);
+  ASSERT_EQ(ceph_setxattr(cmount, test_dir, ACL_EA_ACCESS, acl_buf, acl_buf_size, 0), -CEPHFS_EOPNOTSUPP);
+  ASSERT_EQ(ceph_getxattr(cmount, test_dir, ACL_EA_DEFAULT, acl_buf, acl_buf_size), -CEPHFS_EOPNOTSUPP);
+  ASSERT_EQ(ceph_getxattr(cmount, test_dir, ACL_EA_ACCESS, acl_buf, acl_buf_size), -CEPHFS_EOPNOTSUPP);
+
+  free(acl_buf);
+  ceph_shutdown(cmount);
+}
+
+TEST(ACL, SnapdirACL) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_acl_type", "posix_acl"));
+
+  int acl_buf_size = acl_ea_size(5);
+  void *acl1_buf = malloc(acl_buf_size);
+  void *acl2_buf = malloc(acl_buf_size);
+  void *acl3_buf = malloc(acl_buf_size);
+
+  ASSERT_EQ(generate_test_acl(acl1_buf, acl_buf_size, 0750), 0);
+
+  char test_dir1[256];
+  sprintf(test_dir1, "dir1_acl_default_%d", getpid());
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir1, 0750), 0);
+
+  // set default acl
+  ASSERT_EQ(ceph_setxattr(cmount, test_dir1, ACL_EA_DEFAULT, acl1_buf, acl_buf_size, 0), 0);
+
+  char test_dir2[262];
+  sprintf(test_dir2, "%s/dir2", test_dir1);
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir2, 0755), 0);
+
+  // inherit default acl
+  ASSERT_EQ(ceph_getxattr(cmount, test_dir2, ACL_EA_DEFAULT, acl2_buf, acl_buf_size), acl_buf_size);
+  ASSERT_EQ(memcmp(acl1_buf, acl2_buf, acl_buf_size), 0);
+
+  char test_dir2_snapdir[512];
+  sprintf(test_dir2_snapdir, "%s/dir2/.snap", test_dir1);
+
+  // inherit default acl
+  ASSERT_EQ(ceph_getxattr(cmount, test_dir2_snapdir, ACL_EA_DEFAULT, acl3_buf, acl_buf_size), acl_buf_size);
+  ASSERT_EQ(memcmp(acl2_buf, acl3_buf, acl_buf_size), 0);
+
+  memset(acl2_buf, 0, acl_buf_size);
+  memset(acl3_buf, 0, acl_buf_size);
+
+  ASSERT_EQ(ceph_getxattr(cmount, test_dir2, ACL_EA_ACCESS, acl2_buf, acl_buf_size), acl_buf_size);
+  ASSERT_EQ(ceph_getxattr(cmount, test_dir2_snapdir, ACL_EA_ACCESS, acl3_buf, acl_buf_size), acl_buf_size);
+  ASSERT_EQ(memcmp(acl2_buf, acl3_buf, acl_buf_size), 0);
+
+  free(acl1_buf);
+  free(acl2_buf);
+  free(acl3_buf);
+  ASSERT_EQ(ceph_rmdir(cmount, test_dir2), 0);
+  ASSERT_EQ(ceph_rmdir(cmount, test_dir1), 0);
+  ceph_shutdown(cmount);
+}
diff --git a/src/test/libcephfs/caps.cc b/src/test/libcephfs/caps.cc
new file mode 100644
index 000000000..9141f9a6b
--- /dev/null
+++ b/src/test/libcephfs/caps.cc
@@ -0,0 +1,95 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+#include "include/int_types.h"
+
+#include "gtest/gtest.h"
+#include "include/compat.h"
+#include "include/ceph_fs.h"
+#include "include/cephfs/libcephfs.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#ifdef __linux__
+#include <sys/xattr.h>
+#endif
+#include <signal.h>
+
+TEST(Caps, ReadZero) {
+
+  int mypid = getpid();
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  int i = 0;
+  for(; i < 30; ++i) {
+
+    char c_path[1024];
+    sprintf(c_path, "/caps_rzfile_%d_%d", mypid, i);
+    int fd = ceph_open(cmount, c_path, O_CREAT|O_TRUNC|O_WRONLY, 0644);
+    ASSERT_LT(0, fd);
+
+    int expect = CEPH_CAP_FILE_EXCL | CEPH_CAP_FILE_WR | CEPH_CAP_FILE_BUFFER;
+    int caps = ceph_debug_get_fd_caps(cmount, fd);
+
+    ASSERT_EQ(expect, caps & expect);
+    ASSERT_EQ(0, ceph_close(cmount, fd));
+
+    caps = ceph_debug_get_file_caps(cmount, c_path);
+    ASSERT_EQ(expect, caps & expect);
+
+    char cw_path[1024];
+    sprintf(cw_path, "/caps_wzfile_%d_%d", mypid, i);
+    int wfd = ceph_open(cmount, cw_path, O_CREAT|O_TRUNC|O_WRONLY, 0644);
+    ASSERT_LT(0, wfd);
+
+    char wbuf[4096];
+    ASSERT_EQ(4096, ceph_write(cmount, wfd, wbuf, 4096, 0));
+
+    ASSERT_EQ(0, ceph_close(cmount, wfd));
+
+    struct ceph_statx stx;
+    ASSERT_EQ(0, ceph_statx(cmount, c_path, &stx, CEPH_STATX_MTIME, 0));
+
+    caps = ceph_debug_get_file_caps(cmount, c_path);
+    ASSERT_EQ(expect, caps & expect);
+  }
+
+  ASSERT_EQ(0, ceph_conf_set(cmount, "client_debug_inject_tick_delay", "20"));
+
+  for(i = 0; i < 30; ++i) {
+
+    char c_path[1024];
+    sprintf(c_path, "/caps_rzfile_%d_%d", mypid, i);
+
+    int fd = ceph_open(cmount, c_path, O_RDONLY, 0);
+    ASSERT_LT(0, fd);
+    char buf[256];
+
+    int expect = CEPH_CAP_FILE_RD | CEPH_STAT_CAP_SIZE | CEPH_CAP_FILE_CACHE;
+    int caps = ceph_debug_get_fd_caps(cmount, fd);
+    ASSERT_EQ(expect, caps & expect);
+    ASSERT_EQ(0, ceph_read(cmount, fd, buf, 256, 0));
+
+    caps = ceph_debug_get_fd_caps(cmount, fd);
+    ASSERT_EQ(expect, caps & expect);
+    ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  }
+  ceph_shutdown(cmount);
+}
diff --git a/src/test/libcephfs/ceph_pthread_self.h b/src/test/libcephfs/ceph_pthread_self.h
new file mode 100644
index 000000000..9e3cdfa99
--- /dev/null
+++ b/src/test/libcephfs/ceph_pthread_self.h
@@ -0,0 +1,31 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_TEST_LIBCEPHFS_PTHREAD_SELF
+#define CEPH_TEST_LIBCEPHFS_PTHREAD_SELF
+
+#include <pthread.h>
+
+#include <type_traits>
+
+/*
+ * There is a  difference between libc shipped with FreeBSD and
+ * glibc shipped with GNU/Linux for the return type of pthread_self().
+ *
+ * Introduced a conversion function in include/compat.h
+ *     (uint64_t)ceph_pthread_self()
+ *
+ * libc returns an opague pthread_t that is not default convertable
+ * to a uint64_t, which is what gtest expects.
+ * And tests using gtest will not compile because of this difference.
+ * 
+ */
+static uint64_t ceph_pthread_self() {
+  auto me = pthread_self();
+  static_assert(std::is_convertible_v<decltype(me), uint64_t> ||
+                std::is_pointer_v<decltype(me)>,
+                "we need to use pthread_self() for the owner parameter");
+  return static_cast<uint64_t>(me);
+}
+
+#endif
diff --git a/src/test/libcephfs/deleg.cc b/src/test/libcephfs/deleg.cc
new file mode 100644
index 000000000..061e13763
--- /dev/null
+++ b/src/test/libcephfs/deleg.cc
@@ -0,0 +1,401 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Tests for Ceph delegation handling
+ *
+ * (c) 2017, Jeff Layton <jlayton@redhat.com>
+ */
+
+#include "gtest/gtest.h"
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include "include/stat.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <sys/uio.h>
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#endif
+
+#include <map>
+#include <vector>
+#include <thread>
+#include <atomic>
+
+#include "include/ceph_assert.h"
+
+/* in ms -- 1 minute */
+#define MAX_WAIT	(60 * 1000)
+
+static void wait_for_atomic_bool(std::atomic_bool &recalled)
+{
+  int i = 0;
+
+  while (!recalled.load()) {
+    ASSERT_LT(i++, MAX_WAIT);
+    usleep(1000);
+  }
+}
+
+static int ceph_ll_delegation_wait(struct ceph_mount_info *cmount, Fh *fh,
+				   unsigned cmd, ceph_deleg_cb_t cb, void *priv)
+{
+  int ret, retry = 0;
+
+  /* Wait 10s at most */
+  do {
+    ret = ceph_ll_delegation(cmount, fh, cmd, cb, priv);
+    usleep(10000);
+  } while (ret == -CEPHFS_EAGAIN && retry++ < 1000);
+
+  return ret;
+}
+
+static int set_default_deleg_timeout(struct ceph_mount_info *cmount)
+{
+  uint32_t session_timeout = ceph_get_cap_return_timeout(cmount);
+  return ceph_set_deleg_timeout(cmount, session_timeout - 1);
+}
+
+static void dummy_deleg_cb(Fh *fh, void *priv)
+{
+  std::atomic_bool *recalled = (std::atomic_bool *)priv;
+  recalled->store(true);
+}
+
+static void open_breaker_func(struct ceph_mount_info *cmount, const char *filename, int flags, std::atomic_bool *opened)
+{
+  bool do_shutdown = false;
+
+  if (!cmount) {
+    ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+    ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+    ASSERT_EQ(ceph_conf_parse_env(cmount, NULL), 0);
+    ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+    ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
+    do_shutdown = true;
+  }
+
+  Inode *root, *file;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  Fh *fh;
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  ASSERT_EQ(ceph_ll_lookup(cmount, root, filename, &file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
+  int ret, i = 0;
+  for (;;) {
+    ASSERT_EQ(ceph_ll_getattr(cmount, file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
+    ret = ceph_ll_open(cmount, file, flags, &fh, perms);
+    if (ret != -CEPHFS_EAGAIN)
+      break;
+    ASSERT_LT(i++, MAX_WAIT);
+    usleep(1000);
+  }
+  ASSERT_EQ(ret, 0);
+  opened->store(true);
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+
+  if (do_shutdown)
+    ceph_shutdown(cmount);
+}
+
+enum {
+  DelegTestLink,
+  DelegTestRename,
+  DelegTestUnlink
+};
+
+static void namespace_breaker_func(struct ceph_mount_info *cmount, int cmd, const char *oldname, const char *newname)
+{
+  bool do_shutdown = false;
+
+  if (!cmount) {
+    ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+    ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+    ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+    ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+    ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
+    do_shutdown = true;
+  }
+
+  Inode *root, *file = nullptr;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  int ret, i = 0;
+  for (;;) {
+    switch (cmd) {
+    case DelegTestRename:
+      ret = ceph_ll_rename(cmount, root, oldname, root, newname, perms);
+      break;
+    case DelegTestLink:
+      if (!file) {
+	ASSERT_EQ(ceph_ll_lookup(cmount, root, oldname, &file, &stx, 0, 0, perms), 0);
+      }
+      ret = ceph_ll_link(cmount, file, root, newname, perms);
+      break;
+    case DelegTestUnlink:
+      ret = ceph_ll_unlink(cmount, root, oldname, perms);
+      break;
+    default:
+      // Bad command
+      ceph_abort();
+    }
+    if (ret != -CEPHFS_EAGAIN)
+      break;
+    ASSERT_LT(i++, MAX_WAIT);
+    usleep(1000);
+  }
+  ASSERT_EQ(ret, 0);
+
+  if (do_shutdown)
+    ceph_shutdown(cmount);
+}
+
+static void simple_deleg_test(struct ceph_mount_info *cmount, struct ceph_mount_info *tcmount)
+{
+  Inode *root, *file;
+
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  char filename[32];
+
+  Fh *fh;
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  std::atomic_bool recalled(false);
+  std::atomic_bool opened(false);
+
+  // ensure r/w open breaks a r/w delegation
+  sprintf(filename, "deleg.rwrw.%x", getpid());
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+		    O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker1(open_breaker_func, tcmount, filename, O_RDWR, &opened);
+
+  wait_for_atomic_bool(recalled);
+  ASSERT_EQ(opened.load(), false);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker1.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
+
+  // ensure r/o open breaks a r/w delegation
+  recalled.store(false);
+  opened.store(false);
+  sprintf(filename, "deleg.rorw.%x", getpid());
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+		    O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker2(open_breaker_func, tcmount, filename, O_RDONLY, &opened);
+  wait_for_atomic_bool(recalled);
+  ASSERT_EQ(opened.load(), false);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker2.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
+
+  // ensure r/o open does not break a r/o delegation
+  sprintf(filename, "deleg.rwro.%x", getpid());
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+		    O_RDONLY|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  recalled.store(false);
+  ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_RD, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker3(open_breaker_func, tcmount, filename, O_RDONLY, &opened);
+  breaker3.join();
+  ASSERT_EQ(recalled.load(), false);
+
+  // ensure that r/w open breaks r/o delegation
+  opened.store(false);
+  std::thread breaker4(open_breaker_func, tcmount, filename, O_WRONLY, &opened);
+  wait_for_atomic_bool(recalled);
+  usleep(1000);
+  ASSERT_EQ(opened.load(), false);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker4.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
+
+  // ensure hardlinking breaks a r/w delegation
+  recalled.store(false);
+  char newname[32];
+  sprintf(filename, "deleg.old.%x", getpid());
+  sprintf(newname, "deleg.new.%x", getpid());
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+		    O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker5(namespace_breaker_func, tcmount, DelegTestLink, filename, newname);
+  wait_for_atomic_bool(recalled);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker5.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, newname, perms), 0);
+
+  // ensure renaming breaks a r/w delegation
+  recalled.store(false);
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+		    O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker6(namespace_breaker_func, tcmount, DelegTestRename, filename, newname);
+  wait_for_atomic_bool(recalled);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker6.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_unlink(cmount, root, newname, perms), 0);
+
+  // ensure unlinking breaks a r/w delegation
+  recalled.store(false);
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+		    O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
+  std::thread breaker7(namespace_breaker_func, tcmount, DelegTestUnlink, filename, nullptr);
+  wait_for_atomic_bool(recalled);
+  ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
+  breaker7.join();
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+}
+
+TEST(LibCephFS, DelegMultiClient) {
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+  ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
+
+  simple_deleg_test(cmount, nullptr);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DelegSingleClient) {
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+  ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
+
+  simple_deleg_test(cmount, cmount);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DelegTimeout) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+  // tweak timeout to run quickly, since we don't plan to return it anyway
+  ASSERT_EQ(ceph_set_deleg_timeout(cmount, 2), 0);
+
+  Inode *root, *file;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  char filename[32];
+  sprintf(filename, "delegtimeo%x", getpid());
+
+  Fh *fh;
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+		    O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+
+  /* Reopen read-only */
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(ceph_ll_open(cmount, file, O_RDONLY, &fh, perms), 0);
+
+  std::atomic_bool recalled(false);
+  ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_RD, dummy_deleg_cb, &recalled), 0);
+  std::atomic_bool opened(false);
+  std::thread breaker1(open_breaker_func, nullptr, filename, O_RDWR, &opened);
+  breaker1.join();
+  ASSERT_EQ(recalled.load(), true);
+  ASSERT_EQ(ceph_ll_getattr(cmount, root, &stx, 0, 0, perms), -CEPHFS_ENOTCONN);
+  ceph_release(cmount);
+}
+
+TEST(LibCephFS, RecalledGetattr) {
+  struct ceph_mount_info *cmount1;
+  ASSERT_EQ(ceph_create(&cmount1, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount1, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount1, NULL));
+  ASSERT_EQ(ceph_mount(cmount1, "/"), 0);
+  ASSERT_EQ(set_default_deleg_timeout(cmount1), 0);
+
+  Inode *root, *file;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount1, &root), 0);
+
+  char filename[32];
+  sprintf(filename, "recalledgetattr%x", getpid());
+
+  Fh *fh;
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount1);
+
+  ASSERT_EQ(ceph_ll_create(cmount1, root, filename, 0666,
+		    O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_write(cmount1, fh, 0, sizeof(filename), filename),
+	    static_cast<int>(sizeof(filename)));
+  ASSERT_EQ(ceph_ll_close(cmount1, fh), 0);
+
+  /* New mount for read delegation */
+  struct ceph_mount_info *cmount2;
+  ASSERT_EQ(ceph_create(&cmount2, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount2, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount2, NULL));
+  ASSERT_EQ(ceph_mount(cmount2, "/"), 0);
+  ASSERT_EQ(set_default_deleg_timeout(cmount2), 0);
+
+  ASSERT_EQ(ceph_ll_lookup_root(cmount2, &root), 0);
+  perms = ceph_mount_perms(cmount2);
+  ASSERT_EQ(ceph_ll_lookup(cmount2, root, filename, &file, &stx, 0, 0, perms), 0);
+
+  ASSERT_EQ(ceph_ll_open(cmount2, file, O_WRONLY, &fh, perms), 0);
+  ASSERT_EQ(ceph_ll_write(cmount2, fh, 0, sizeof(filename), filename),
+	    static_cast<int>(sizeof(filename)));
+  ASSERT_EQ(ceph_ll_close(cmount2, fh), 0);
+
+  ASSERT_EQ(ceph_ll_open(cmount2, file, O_RDONLY, &fh, perms), 0);
+
+  /* Break delegation */
+  std::atomic_bool recalled(false);
+  ASSERT_EQ(ceph_ll_delegation_wait(cmount2, fh, CEPH_DELEGATION_RD, dummy_deleg_cb, &recalled), 0);
+  ASSERT_EQ(ceph_ll_read(cmount2, fh, 0, sizeof(filename), filename),
+	    static_cast<int>(sizeof(filename)));
+  ASSERT_EQ(ceph_ll_getattr(cmount2, file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
+  std::atomic_bool opened(false);
+  std::thread breaker1(open_breaker_func, cmount1, filename, O_WRONLY, &opened);
+  int i = 0;
+  do {
+    ASSERT_EQ(ceph_ll_getattr(cmount2, file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
+    ASSERT_LT(i++, MAX_WAIT);
+    usleep(1000);
+  } while (!recalled.load());
+  ASSERT_EQ(opened.load(), false);
+  ASSERT_EQ(ceph_ll_getattr(cmount2, file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_delegation(cmount2, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, nullptr), 0);
+  breaker1.join();
+  ASSERT_EQ(ceph_ll_close(cmount2, fh), 0);
+  ceph_unmount(cmount2);
+  ceph_release(cmount2);
+  ceph_unmount(cmount1);
+  ceph_release(cmount1);
+}
diff --git a/src/test/libcephfs/flock.cc b/src/test/libcephfs/flock.cc
new file mode 100644
index 000000000..367483d07
--- /dev/null
+++ b/src/test/libcephfs/flock.cc
@@ -0,0 +1,654 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <pthread.h>
+#include "gtest/gtest.h"
+#ifndef GTEST_IS_THREADSAFE
+#error "!GTEST_IS_THREADSAFE"
+#endif
+
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/file.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <stdlib.h>
+#include <semaphore.h>
+#include <time.h>
+
+#ifndef _WIN32
+#include <sys/mman.h>
+#endif
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#elif __FreeBSD__
+#include <sys/types.h>
+#include <sys/wait.h>
+#endif
+
+#include "include/ceph_assert.h"
+#include "ceph_pthread_self.h"
+
+// Startup common: create and mount ceph fs
+#define STARTUP_CEPH() do {				\
+    ASSERT_EQ(0, ceph_create(&cmount, NULL));		\
+    ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));	\
+    ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));	\
+    ASSERT_EQ(0, ceph_mount(cmount, NULL));		\
+  } while(0)
+
+// Cleanup common: unmount and release ceph fs
+#define CLEANUP_CEPH() do {			\
+    ASSERT_EQ(0, ceph_unmount(cmount));		\
+    ASSERT_EQ(0, ceph_release(cmount));		\
+  } while(0)
+
+static const mode_t fileMode = S_IRWXU | S_IRWXG | S_IRWXO;
+
+// Default wait time for normal and "slow" operations
+// (5" should be enough in case of network congestion)
+static const long waitMs = 10;
+static const long waitSlowMs = 5000;
+
+// Get the absolute struct timespec reference from now + 'ms' milliseconds
+static const struct timespec* abstime(struct timespec &ts, long ms) {
+  if (clock_gettime(CLOCK_REALTIME, &ts) == -1) {
+    ceph_abort();
+  }
+  ts.tv_nsec += ms * 1000000;
+  ts.tv_sec += ts.tv_nsec / 1000000000;
+  ts.tv_nsec %= 1000000000;
+  return &ts;
+}
+
+/* Basic locking */
+TEST(LibCephFS, BasicLocking) {
+  struct ceph_mount_info *cmount = NULL;
+  STARTUP_CEPH();
+
+  char c_file[1024];
+  sprintf(c_file, "/flock_test_%d", getpid());
+  const int fd = ceph_open(cmount, c_file, O_RDWR | O_CREAT, fileMode);
+  ASSERT_GE(fd, 0); 
+
+  // Lock exclusively twice
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, 42));
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, 43));
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, 44));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, 42));
+
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, 43));
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, 44));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, 43));
+
+  // Lock shared three times
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH, 42));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH, 43));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH, 44));
+  // And then attempt to lock exclusively
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, 45));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, 42));
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, 45));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, 44));
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, 45));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, 43));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, 45));
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_SH | LOCK_NB, 42));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, 45));
+
+  // Lock shared with upgrade to exclusive (POSIX) 
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH, 42));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, 42));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, 42));
+
+  // Lock exclusive with downgrade to shared (POSIX) 
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, 42));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH, 42));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, 42));
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_unlink(cmount, c_file));
+  CLEANUP_CEPH();
+}
+
+/* Locking in different threads */
+
+// Used by ConcurrentLocking test
+struct str_ConcurrentLocking {
+  const char *file;
+  struct ceph_mount_info *cmount;  // !NULL if shared
+  sem_t sem[2];
+  sem_t semReply[2];
+  void sem_init(int pshared) {
+    ASSERT_EQ(0, ::sem_init(&sem[0], pshared, 0));
+    ASSERT_EQ(0, ::sem_init(&sem[1], pshared, 0));
+    ASSERT_EQ(0, ::sem_init(&semReply[0], pshared, 0));
+    ASSERT_EQ(0, ::sem_init(&semReply[1], pshared, 0));
+  }
+  void sem_destroy() {
+    ASSERT_EQ(0, ::sem_destroy(&sem[0]));
+    ASSERT_EQ(0, ::sem_destroy(&sem[1]));
+    ASSERT_EQ(0, ::sem_destroy(&semReply[0]));
+    ASSERT_EQ(0, ::sem_destroy(&semReply[1]));
+  }
+};
+
+// Wakeup main (for (N) steps)
+#define PING_MAIN(n) ASSERT_EQ(0, sem_post(&s.sem[n%2]))
+// Wait for main to wake us up (for (RN) steps)
+#define WAIT_MAIN(n) \
+  ASSERT_EQ(0, sem_timedwait(&s.semReply[n%2], abstime(ts, waitSlowMs)))
+
+// Wakeup worker (for (RN) steps)
+#define PING_WORKER(n) ASSERT_EQ(0, sem_post(&s.semReply[n%2]))
+// Wait for worker to wake us up (for (N) steps)
+#define WAIT_WORKER(n) \
+  ASSERT_EQ(0, sem_timedwait(&s.sem[n%2], abstime(ts, waitSlowMs)))
+// Worker shall not wake us up (for (N) steps)
+#define NOT_WAIT_WORKER(n) \
+  ASSERT_EQ(-1, sem_timedwait(&s.sem[n%2], abstime(ts, waitMs)))
+
+// Do twice an operation
+#define TWICE(EXPR) do {			\
+    EXPR;					\
+    EXPR;					\
+  } while(0)
+
+/* Locking in different threads */
+
+// Used by ConcurrentLocking test
+static void thread_ConcurrentLocking(str_ConcurrentLocking& s) {
+  struct ceph_mount_info *const cmount = s.cmount;
+  struct timespec ts;
+
+  const int fd = ceph_open(cmount, s.file, O_RDWR | O_CREAT, fileMode);
+  ASSERT_GE(fd, 0); 
+
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+	    ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, ceph_pthread_self()));
+  PING_MAIN(1); // (1)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, ceph_pthread_self()));
+  PING_MAIN(2); // (2)
+
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, ceph_pthread_self()));
+  PING_MAIN(3); // (3)
+
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH, ceph_pthread_self()));
+  PING_MAIN(4); // (4)
+
+  WAIT_MAIN(1); // (R1)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, ceph_pthread_self()));
+  PING_MAIN(5); // (5)
+
+  WAIT_MAIN(2); // (R2)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, ceph_pthread_self()));
+  PING_MAIN(6); // (6)
+
+  WAIT_MAIN(3); // (R3)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, ceph_pthread_self()));
+  PING_MAIN(7); // (7)
+}
+
+// Used by ConcurrentLocking test
+static void* thread_ConcurrentLocking_(void *arg) {
+  str_ConcurrentLocking *const s =
+    reinterpret_cast<str_ConcurrentLocking*>(arg);
+  thread_ConcurrentLocking(*s);
+  return NULL;
+}
+
+TEST(LibCephFS, ConcurrentLocking) {
+  const pid_t mypid = getpid();
+  struct ceph_mount_info *cmount;
+  STARTUP_CEPH();
+
+  char c_file[1024];
+  sprintf(c_file, "/flock_test_%d", mypid);
+  const int fd = ceph_open(cmount, c_file, O_RDWR | O_CREAT, fileMode);
+  ASSERT_GE(fd, 0); 
+
+  // Lock
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, ceph_pthread_self()));
+
+  // Start locker thread
+  pthread_t thread;
+  struct timespec ts;
+  str_ConcurrentLocking s = { c_file, cmount };
+  s.sem_init(0);
+  ASSERT_EQ(0, pthread_create(&thread, NULL, thread_ConcurrentLocking_, &s));
+  // Synchronization point with thread (failure: thread is dead)
+  WAIT_WORKER(1); // (1)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(2); // (2)
+
+  // Unlock
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, ceph_pthread_self()));
+
+  // Shall have lock
+  // Synchronization point with thread (failure: thread is dead)
+  WAIT_WORKER(2); // (2)
+
+  // Synchronization point with thread (failure: thread is dead)
+  WAIT_WORKER(3); // (3)
+
+  // Wait for thread to share lock
+  WAIT_WORKER(4); // (4)
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+	    ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, ceph_pthread_self()));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH | LOCK_NB, ceph_pthread_self()));
+
+  // Wake up thread to unlock shared lock
+  PING_WORKER(1); // (R1)
+  WAIT_WORKER(5); // (5)
+
+  // Now we can lock exclusively
+  // Upgrade to exclusive lock (as per POSIX)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, ceph_pthread_self()));
+
+  // Wake up thread to lock shared lock
+  PING_WORKER(2); // (R2)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(6); // (6)
+
+  // Release lock ; thread will get it
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, ceph_pthread_self()));
+  WAIT_WORKER(6); // (6)
+
+  // We no longer have the lock
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+	    ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, ceph_pthread_self()));
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+	    ceph_flock(cmount, fd, LOCK_SH | LOCK_NB, ceph_pthread_self()));
+
+  // Wake up thread to unlock exclusive lock
+  PING_WORKER(3); // (R3)
+  WAIT_WORKER(7); // (7)
+
+  // We can lock it again
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, ceph_pthread_self()));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, ceph_pthread_self()));
+
+  // Cleanup
+  void *retval = (void*) (uintptr_t) -1;
+  ASSERT_EQ(0, pthread_join(thread, &retval));
+  ASSERT_EQ(NULL, retval);
+  s.sem_destroy();
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_unlink(cmount, c_file));
+  CLEANUP_CEPH();
+}
+
+TEST(LibCephFS, ThreesomeLocking) {
+  const pid_t mypid = getpid();
+  struct ceph_mount_info *cmount;
+  STARTUP_CEPH();
+
+  char c_file[1024];
+  sprintf(c_file, "/flock_test_%d", mypid);
+  const int fd = ceph_open(cmount, c_file, O_RDWR | O_CREAT, fileMode);
+  ASSERT_GE(fd, 0); 
+
+  // Lock
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, ceph_pthread_self()));
+
+  // Start locker thread
+  pthread_t thread[2];
+  struct timespec ts;
+  str_ConcurrentLocking s = { c_file, cmount };
+  s.sem_init(0);
+  ASSERT_EQ(0, pthread_create(&thread[0], NULL, thread_ConcurrentLocking_, &s));
+  ASSERT_EQ(0, pthread_create(&thread[1], NULL, thread_ConcurrentLocking_, &s));
+  // Synchronization point with thread (failure: thread is dead)
+  TWICE(WAIT_WORKER(1)); // (1)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(2); // (2)
+
+  // Unlock
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, ceph_pthread_self()));
+
+  // Shall have lock
+  TWICE(// Synchronization point with thread (failure: thread is dead)
+	WAIT_WORKER(2); // (2)
+	
+	// Synchronization point with thread (failure: thread is dead)
+	WAIT_WORKER(3)); // (3)
+  
+  // Wait for thread to share lock
+  TWICE(WAIT_WORKER(4)); // (4)
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+	    ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, ceph_pthread_self()));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH | LOCK_NB, ceph_pthread_self()));
+
+  // Wake up thread to unlock shared lock
+  TWICE(PING_WORKER(1); // (R1)
+	WAIT_WORKER(5)); // (5)
+
+  // Now we can lock exclusively
+  // Upgrade to exclusive lock (as per POSIX)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, ceph_pthread_self()));
+
+  TWICE(  // Wake up thread to lock shared lock
+	PING_WORKER(2); // (R2)
+	
+	// Shall not have lock immediately
+	NOT_WAIT_WORKER(6)); // (6)
+  
+  // Release lock ; thread will get it
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, ceph_pthread_self()));
+  TWICE(WAIT_WORKER(6); // (6)
+	
+	// We no longer have the lock
+	ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+		  ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, ceph_pthread_self()));
+	ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+		  ceph_flock(cmount, fd, LOCK_SH | LOCK_NB, ceph_pthread_self()));
+	
+	// Wake up thread to unlock exclusive lock
+	PING_WORKER(3); // (R3)
+	WAIT_WORKER(7); // (7)
+	);
+  
+  // We can lock it again
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, ceph_pthread_self()));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, ceph_pthread_self()));
+
+  // Cleanup
+  void *retval = (void*) (uintptr_t) -1;
+  ASSERT_EQ(0, pthread_join(thread[0], &retval));
+  ASSERT_EQ(NULL, retval);
+  ASSERT_EQ(0, pthread_join(thread[1], &retval));
+  ASSERT_EQ(NULL, retval);
+  s.sem_destroy();
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_unlink(cmount, c_file));
+  CLEANUP_CEPH();
+}
+
+/* Locking in different processes */
+
+#define PROCESS_SLOW_MS() \
+  static const long waitMs = 100; \
+  (void) waitMs
+
+// Used by ConcurrentLocking test
+static void process_ConcurrentLocking(str_ConcurrentLocking& s) {
+  const pid_t mypid = getpid();
+  PROCESS_SLOW_MS();
+
+  struct ceph_mount_info *cmount = NULL;
+  struct timespec ts;
+
+  STARTUP_CEPH();
+  s.cmount = cmount;
+
+  const int fd = ceph_open(cmount, s.file, O_RDWR | O_CREAT, fileMode);
+  ASSERT_GE(fd, 0); 
+  WAIT_MAIN(1); // (R1)
+
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+	    ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, mypid));
+  PING_MAIN(1); // (1)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, mypid));
+  PING_MAIN(2); // (2)
+
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, mypid));
+  PING_MAIN(3); // (3)
+
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH, mypid));
+  PING_MAIN(4); // (4)
+
+  WAIT_MAIN(2); // (R2)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, mypid));
+  PING_MAIN(5); // (5)
+
+  WAIT_MAIN(3); // (R3)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, mypid));
+  PING_MAIN(6); // (6)
+
+  WAIT_MAIN(4); // (R4)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, mypid));
+  PING_MAIN(7); // (7)
+
+  CLEANUP_CEPH();
+
+  s.sem_destroy();
+  exit(EXIT_SUCCESS);
+}
+
+#ifndef _WIN32
+// Disabled because of fork() issues (http://tracker.ceph.com/issues/16556)
+TEST(LibCephFS, DISABLED_InterProcessLocking) {
+  PROCESS_SLOW_MS();
+  // Process synchronization
+  char c_file[1024];
+  const pid_t mypid = getpid();
+  sprintf(c_file, "/flock_test_%d", mypid);
+
+  // Note: the semaphores MUST be on a shared memory segment
+  str_ConcurrentLocking *const shs =
+    reinterpret_cast<str_ConcurrentLocking*>
+    (mmap(0, sizeof(*shs), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
+	  -1, 0));
+  str_ConcurrentLocking &s = *shs;
+  s.file = c_file;
+  s.sem_init(1);
+
+  // Start locker process
+  const pid_t pid = fork();
+  ASSERT_GE(pid, 0);
+  if (pid == 0) {
+    process_ConcurrentLocking(s);
+    exit(EXIT_FAILURE);
+  }
+
+  struct timespec ts;
+  struct ceph_mount_info *cmount;
+  STARTUP_CEPH();
+
+  const int fd = ceph_open(cmount, c_file, O_RDWR | O_CREAT, fileMode);
+  ASSERT_GE(fd, 0); 
+
+  // Lock
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, mypid));
+
+  // Synchronization point with process (failure: process is dead)
+  PING_WORKER(1); // (R1)
+  WAIT_WORKER(1); // (1)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(2); // (2)
+
+  // Unlock
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, mypid));
+
+  // Shall have lock
+  // Synchronization point with process (failure: process is dead)
+  WAIT_WORKER(2); // (2)
+
+  // Synchronization point with process (failure: process is dead)
+  WAIT_WORKER(3); // (3)
+
+  // Wait for process to share lock
+  WAIT_WORKER(4); // (4)
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, mypid));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH | LOCK_NB, mypid));
+
+  // Wake up process to unlock shared lock
+  PING_WORKER(2); // (R2)
+  WAIT_WORKER(5); // (5)
+
+  // Now we can lock exclusively
+  // Upgrade to exclusive lock (as per POSIX)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, mypid));
+
+  // Wake up process to lock shared lock
+  PING_WORKER(3); // (R3)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(6); // (6)
+
+  // Release lock ; process will get it
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, mypid));
+  WAIT_WORKER(6); // (6)
+
+  // We no longer have the lock
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, mypid));
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK, ceph_flock(cmount, fd, LOCK_SH | LOCK_NB, mypid));
+
+  // Wake up process to unlock exclusive lock
+  PING_WORKER(4); // (R4)
+  WAIT_WORKER(7); // (7)
+
+  // We can lock it again
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, mypid));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, mypid));
+
+  // Wait pid
+  int status;
+  ASSERT_EQ(pid, waitpid(pid, &status, 0));
+  ASSERT_EQ(EXIT_SUCCESS, status);
+
+  // Cleanup
+  s.sem_destroy();
+  ASSERT_EQ(0, munmap(shs, sizeof(*shs)));
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_unlink(cmount, c_file));
+  CLEANUP_CEPH();
+}
+#endif
+
+#ifndef _WIN32
+// Disabled because of fork() issues (http://tracker.ceph.com/issues/16556)
+TEST(LibCephFS, DISABLED_ThreesomeInterProcessLocking) {
+  PROCESS_SLOW_MS();
+  // Process synchronization
+  char c_file[1024];
+  const pid_t mypid = getpid();
+  sprintf(c_file, "/flock_test_%d", mypid);
+
+  // Note: the semaphores MUST be on a shared memory segment
+  str_ConcurrentLocking *const shs =
+    reinterpret_cast<str_ConcurrentLocking*>
+    (mmap(0, sizeof(*shs), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
+	  -1, 0));
+  str_ConcurrentLocking &s = *shs;
+  s.file = c_file;
+  s.sem_init(1);
+
+  // Start locker processes
+  pid_t pid[2];
+  pid[0] = fork();
+  ASSERT_GE(pid[0], 0);
+  if (pid[0] == 0) {
+    process_ConcurrentLocking(s);
+    exit(EXIT_FAILURE);
+  }
+  pid[1] = fork();
+  ASSERT_GE(pid[1], 0);
+  if (pid[1] == 0) {
+    process_ConcurrentLocking(s);
+    exit(EXIT_FAILURE);
+  }
+
+  struct timespec ts;
+  struct ceph_mount_info *cmount;
+  STARTUP_CEPH();
+
+  const int fd = ceph_open(cmount, c_file, O_RDWR | O_CREAT, fileMode);
+  ASSERT_GE(fd, 0); 
+
+  // Lock
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, mypid));
+
+  // Synchronization point with process (failure: process is dead)
+  TWICE(PING_WORKER(1)); // (R1)
+  TWICE(WAIT_WORKER(1)); // (1)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(2); // (2)
+
+  // Unlock
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, mypid));
+
+  // Shall have lock
+  TWICE(// Synchronization point with process (failure: process is dead)
+	WAIT_WORKER(2); // (2)
+	
+	// Synchronization point with process (failure: process is dead)
+	WAIT_WORKER(3)); // (3)
+  
+  // Wait for process to share lock
+  TWICE(WAIT_WORKER(4)); // (4)
+  ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+	    ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, mypid));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_SH | LOCK_NB, mypid));
+
+  // Wake up process to unlock shared lock
+  TWICE(PING_WORKER(2); // (R2)
+	WAIT_WORKER(5)); // (5)
+
+  // Now we can lock exclusively
+  // Upgrade to exclusive lock (as per POSIX)
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX, mypid));
+
+  TWICE(  // Wake up process to lock shared lock
+	PING_WORKER(3); // (R3)
+	
+	// Shall not have lock immediately
+	NOT_WAIT_WORKER(6)); // (6)
+  
+  // Release lock ; process will get it
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, mypid));
+  TWICE(WAIT_WORKER(6); // (6)
+	
+	// We no longer have the lock
+	ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+		  ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, mypid));
+	ASSERT_EQ(-CEPHFS_EWOULDBLOCK,
+		  ceph_flock(cmount, fd, LOCK_SH | LOCK_NB, mypid));
+	
+	// Wake up process to unlock exclusive lock
+	PING_WORKER(4); // (R4)
+	WAIT_WORKER(7); // (7)
+	);
+  
+  // We can lock it again
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_EX | LOCK_NB, mypid));
+  ASSERT_EQ(0, ceph_flock(cmount, fd, LOCK_UN, mypid));
+
+  // Wait pids
+  int status;
+  ASSERT_EQ(pid[0], waitpid(pid[0], &status, 0));
+  ASSERT_EQ(EXIT_SUCCESS, status);
+  ASSERT_EQ(pid[1], waitpid(pid[1], &status, 0));
+  ASSERT_EQ(EXIT_SUCCESS, status);
+
+  // Cleanup
+  s.sem_destroy();
+  ASSERT_EQ(0, munmap(shs, sizeof(*shs)));
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_unlink(cmount, c_file));
+  CLEANUP_CEPH();
+}
+#endif
diff --git a/src/test/libcephfs/lazyio.cc b/src/test/libcephfs/lazyio.cc
new file mode 100644
index 000000000..0c17f1f2b
--- /dev/null
+++ b/src/test/libcephfs/lazyio.cc
@@ -0,0 +1,356 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2019 Red Hat Ltd
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include "gtest/gtest.h"
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/rados/librados.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#if defined(__linux__)
+#include <sys/xattr.h>
+#endif
+
+rados_t cluster;
+
+TEST(LibCephFS, LazyIOOneWriterMulipleReaders) {
+  struct ceph_mount_info *ca, *cb;
+  ASSERT_EQ(ceph_create(&ca, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(ca, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+  ASSERT_EQ(ceph_mount(ca, NULL), 0);
+
+  ASSERT_EQ(ceph_create(&cb, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cb, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cb, NULL));
+  ASSERT_EQ(ceph_mount(cb, NULL), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo.%d", getpid());
+
+  int fda = ceph_open(ca, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fda);
+
+  int fdb = ceph_open(cb, name, O_RDONLY, 0644);
+  ASSERT_LE(0, fdb);
+
+  ASSERT_EQ(0, ceph_lazyio(ca, fda, 1));
+  ASSERT_EQ(0, ceph_lazyio(cb, fdb, 1));
+ 
+  char out_buf[] = "fooooooooo";
+
+  /* Client a issues a write and propagates/flushes the buffer */
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(ca, fda, out_buf, sizeof(out_buf), 0));
+  ASSERT_EQ(0, ceph_lazyio_propagate(ca, fda, 0, 0));
+
+  /* Client a issues a write and propagates/flushes the buffer */
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(ca, fda, out_buf, sizeof(out_buf), 10));
+  ASSERT_EQ(0, ceph_lazyio_propagate(ca, fda, 0, 0));
+
+  char in_buf[40];
+  /* Calling ceph_lazyio_synchronize here will invalidate client b's cache and hence enable client a to fetch the propagated write of client a in the subsequent read */
+  ASSERT_EQ(0, ceph_lazyio_synchronize(cb, fdb, 0, 0));
+  ASSERT_EQ(ceph_read(cb, fdb, in_buf, sizeof(in_buf), 0), 2*strlen(out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooofooooooooo");
+
+  /* Client a does not need to call ceph_lazyio_synchronize here because it is the latest writer and fda holds the updated inode*/
+  ASSERT_EQ(ceph_read(ca, fda, in_buf, sizeof(in_buf), 0), 2*strlen(out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooofooooooooo");
+
+  ceph_close(ca, fda);
+  ceph_close(cb, fdb);
+
+  ceph_shutdown(ca);
+  ceph_shutdown(cb);
+}
+
+TEST(LibCephFS, LazyIOMultipleWritersMulipleReaders) {
+  struct ceph_mount_info *ca, *cb;
+  ASSERT_EQ(ceph_create(&ca, NULL), 0); 
+  ASSERT_EQ(ceph_conf_read_file(ca, NULL), 0); 
+  ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+  ASSERT_EQ(ceph_mount(ca, NULL), 0); 
+
+  ASSERT_EQ(ceph_create(&cb, NULL), 0); 
+  ASSERT_EQ(ceph_conf_read_file(cb, NULL), 0); 
+  ASSERT_EQ(0, ceph_conf_parse_env(cb, NULL));
+  ASSERT_EQ(ceph_mount(cb, NULL), 0); 
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo2.%d", getpid());
+
+  int fda = ceph_open(ca, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fda);
+
+  int fdb = ceph_open(cb, name, O_RDWR, 0644);
+  ASSERT_LE(0, fdb);
+
+  ASSERT_EQ(0, ceph_lazyio(ca, fda, 1));
+  ASSERT_EQ(0, ceph_lazyio(cb, fdb, 1));
+
+  char out_buf[] = "fooooooooo";
+  /* Client a issues a write and propagates/flushes the buffer */
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(ca, fda, out_buf, sizeof(out_buf), 0));
+  ASSERT_EQ(0, ceph_lazyio_propagate(ca, fda, 0, 0));
+  
+  /* Client b issues a write and propagates/flushes the buffer*/
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(cb, fdb, out_buf, sizeof(out_buf), 10));
+  ASSERT_EQ(0, ceph_lazyio_propagate(cb, fdb, 0, 0));
+
+  char in_buf[40];
+  /* Calling ceph_lazyio_synchronize here will invalidate client a's cache and hence enable client a to fetch the propagated writes of client b in the subsequent read */
+  ASSERT_EQ(0, ceph_lazyio_synchronize(ca, fda, 0, 0));
+  ASSERT_EQ(ceph_read(ca, fda, in_buf, sizeof(in_buf), 0), 2*strlen(out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooofooooooooo");
+  
+  /* Client b does not need to call ceph_lazyio_synchronize here because it is the latest writer and the writes before it have already been propagated*/
+  ASSERT_EQ(ceph_read(cb, fdb, in_buf, sizeof(in_buf), 0), 2*strlen(out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooofooooooooo");
+
+  /* Client a issues a write */
+  char wait_out_buf[] = "foobarbars";
+  ASSERT_EQ((int)sizeof(wait_out_buf), ceph_write(ca, fda, wait_out_buf, sizeof(wait_out_buf), 20));
+  ASSERT_EQ(0, ceph_lazyio_propagate(ca, fda, 0, 0));
+
+  /* Client a does not need to call ceph_lazyio_synchronize here because it is the latest writer and the writes before it have already been propagated*/
+  ASSERT_EQ(ceph_read(ca, fda, in_buf, sizeof(in_buf), 0), (2*(strlen(out_buf)))+strlen(wait_out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooofooooooooofoobarbars");
+
+  /* Calling ceph_lazyio_synchronize here will invalidate client b's cache and hence enable client a to fetch the propagated write of client a in the subsequent read */
+  ASSERT_EQ(0, ceph_lazyio_synchronize(cb, fdb, 0, 0));
+  ASSERT_EQ(ceph_read(cb, fdb, in_buf, sizeof(in_buf), 0), (2*(strlen(out_buf)))+strlen(wait_out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooofooooooooofoobarbars");
+
+  ceph_close(ca, fda);
+  ceph_close(cb, fdb);
+
+  ceph_shutdown(ca);
+  ceph_shutdown(cb);
+}
+
+TEST(LibCephFS, LazyIOMultipleWritersOneReader) {
+  struct ceph_mount_info *ca, *cb;
+  ASSERT_EQ(ceph_create(&ca, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(ca, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+  ASSERT_EQ(ceph_mount(ca, NULL), 0);
+
+  ASSERT_EQ(ceph_create(&cb, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cb, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cb, NULL));
+  ASSERT_EQ(ceph_mount(cb, NULL), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo3.%d", getpid());
+
+  int fda = ceph_open(ca, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fda);
+
+  int fdb = ceph_open(cb, name, O_RDWR, 0644);
+  ASSERT_LE(0, fdb);
+ 
+  ASSERT_EQ(0, ceph_lazyio(ca, fda, 1));
+  ASSERT_EQ(0, ceph_lazyio(cb, fdb, 1));
+
+  char out_buf[] = "fooooooooo";
+  /* Client a issues a write and propagates/flushes the buffer */
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(ca, fda, out_buf, sizeof(out_buf), 0));
+  ASSERT_EQ(0, ceph_lazyio_propagate(ca, fda, 0, 0));
+  
+  /* Client b issues a write and propagates/flushes the buffer*/
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(cb, fdb, out_buf, sizeof(out_buf), 10));
+  ASSERT_EQ(0, ceph_lazyio_propagate(cb, fdb, 0, 0));
+
+  char in_buf[40];
+  /* Client a reads the file and verifies that it only reads it's propagated writes and not Client b's*/
+  ASSERT_EQ(ceph_read(ca, fda, in_buf, sizeof(in_buf), 0), strlen(out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooo");
+  
+  /* Client a reads the file again, this time with a lazyio_synchronize to check if the cache gets invalidated and data is refetched i.e all the propagated writes are being read*/
+  ASSERT_EQ(0, ceph_lazyio_synchronize(ca, fda, 0, 0));
+  ASSERT_EQ(ceph_read(ca, fda, in_buf, sizeof(in_buf), 0), 2*strlen(out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooofooooooooo");
+
+  ceph_close(ca, fda);
+  ceph_close(cb, fdb);
+
+  ceph_shutdown(ca);
+  ceph_shutdown(cb);
+}
+
+TEST(LibCephFS, LazyIOSynchronizeFlush) {
+  /* Test to make sure lazyio_synchronize flushes dirty buffers */
+  struct ceph_mount_info *ca, *cb;
+  ASSERT_EQ(ceph_create(&ca, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(ca, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+  ASSERT_EQ(ceph_mount(ca, NULL), 0);
+
+  ASSERT_EQ(ceph_create(&cb, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cb, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cb, NULL));
+  ASSERT_EQ(ceph_mount(cb, NULL), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo4.%d", getpid());
+
+  int fda = ceph_open(ca, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fda);
+
+  int fdb = ceph_open(cb, name, O_RDWR, 0644);
+  ASSERT_LE(0, fdb);
+
+  ASSERT_EQ(0, ceph_lazyio(ca, fda, 1));
+  ASSERT_EQ(0, ceph_lazyio(cb, fdb, 1));
+
+  char out_buf[] = "fooooooooo";
+
+  /* Client a issues a write and propagates it*/
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(ca, fda, out_buf, sizeof(out_buf), 0));
+  ASSERT_EQ(0, ceph_lazyio_propagate(ca, fda, 0, 0));
+
+  /* Client b issues writes and without lazyio_propagate*/
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(cb, fdb, out_buf, sizeof(out_buf), 10));
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(cb, fdb, out_buf, sizeof(out_buf), 20));
+  
+  char in_buf[40];
+  /* Calling ceph_lazyio_synchronize here will first flush the possibly pending buffered write of client b and invalidate client b's cache and hence enable client b to fetch all the propagated writes */
+  ASSERT_EQ(0, ceph_lazyio_synchronize(cb, fdb, 0, 0));
+  ASSERT_EQ(ceph_read(cb, fdb, in_buf, sizeof(in_buf), 0), 3*strlen(out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooofooooooooofooooooooo");
+
+  /* Required to call ceph_lazyio_synchronize here since client b is the latest writer and client a is out of sync with updated file*/ 
+  ASSERT_EQ(0, ceph_lazyio_synchronize(ca, fda, 0, 0));
+  ASSERT_EQ(ceph_read(ca, fda, in_buf, sizeof(in_buf), 0), 3*strlen(out_buf)+1);
+  ASSERT_STREQ(in_buf, "fooooooooofooooooooofooooooooo");
+
+  ceph_close(ca, fda);
+  ceph_close(cb, fdb);
+
+  ceph_shutdown(ca);
+  ceph_shutdown(cb);
+}
+
+TEST(LibCephFS, WithoutandWithLazyIO) {
+  struct ceph_mount_info *ca, *cb;
+  ASSERT_EQ(ceph_create(&ca, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(ca, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+  ASSERT_EQ(ceph_mount(ca, NULL), 0);
+
+  ASSERT_EQ(ceph_create(&cb, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cb, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cb, NULL));
+  ASSERT_EQ(ceph_mount(cb, NULL), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo5.%d", getpid());
+
+  int fda = ceph_open(ca, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fda);
+
+  int fdb = ceph_open(cb, name, O_RDWR, 0644);
+  ASSERT_LE(0, fdb);
+
+  char out_buf_w[] = "1234567890";
+  /* Doing some non lazyio writes and read*/
+  ASSERT_EQ((int)sizeof(out_buf_w), ceph_write(ca, fda, out_buf_w, sizeof(out_buf_w), 0));
+
+  ASSERT_EQ((int)sizeof(out_buf_w), ceph_write(cb, fdb, out_buf_w, sizeof(out_buf_w), 10));
+
+  char in_buf_w[30];
+  ASSERT_EQ(ceph_read(ca, fda, in_buf_w, sizeof(in_buf_w), 0), 2*strlen(out_buf_w)+1);
+
+  /* Enable lazyio*/
+  ASSERT_EQ(0, ceph_lazyio(ca, fda, 1));
+  ASSERT_EQ(0, ceph_lazyio(cb, fdb, 1));
+
+  char out_buf[] = "fooooooooo";
+
+  /* Client a issues a write and propagates/flushes the buffer*/
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(ca, fda, out_buf, sizeof(out_buf), 20));
+  ASSERT_EQ(0, ceph_lazyio_propagate(ca, fda, 0, 0));
+  
+  /* Client b issues a write and propagates/flushes the buffer*/
+  ASSERT_EQ((int)sizeof(out_buf), ceph_write(cb, fdb, out_buf, sizeof(out_buf), 30));
+  ASSERT_EQ(0, ceph_lazyio_propagate(cb, fdb, 0, 0));
+
+  char in_buf[50];
+  /* Calling ceph_lazyio_synchronize here will invalidate client a's cache and hence enable client a to fetch the propagated writes of client b in the subsequent read */
+  ASSERT_EQ(0, ceph_lazyio_synchronize(ca, fda, 0, 0));
+  ASSERT_EQ(ceph_read(ca, fda, in_buf, sizeof(in_buf), 0), (2*(strlen(out_buf)))+(2*(strlen(out_buf_w)))+1);
+  ASSERT_STREQ(in_buf, "12345678901234567890fooooooooofooooooooo");
+
+  /* Client b does not need to call ceph_lazyio_synchronize here because it is the latest writer and the writes before it have already been propagated*/
+  ASSERT_EQ(ceph_read(cb, fdb, in_buf, sizeof(in_buf), 0), (2*(strlen(out_buf)))+(2*(strlen(out_buf_w)))+1);
+  ASSERT_STREQ(in_buf, "12345678901234567890fooooooooofooooooooo");
+
+  ceph_close(ca, fda);
+  ceph_close(cb, fdb);
+
+  ceph_shutdown(ca);
+  ceph_shutdown(cb);
+}
+
+static int update_root_mode()
+{
+  struct ceph_mount_info *admin;
+  int r = ceph_create(&admin, NULL);
+  if (r < 0)
+    return r;
+  ceph_conf_read_file(admin, NULL);
+  ceph_conf_parse_env(admin, NULL);
+  ceph_conf_set(admin, "client_permissions", "false");
+  r = ceph_mount(admin, "/");
+  if (r < 0)
+    goto out;
+  r = ceph_chmod(admin, "/", 0777);
+out:
+  ceph_shutdown(admin);
+  return r;
+}
+
+int main(int argc, char **argv)
+{
+  int r = update_root_mode();
+  if (r < 0)
+    exit(1);
+
+  ::testing::InitGoogleTest(&argc, argv);
+
+  srand(getpid());
+
+  r = rados_create(&cluster, NULL);
+  if (r < 0)
+    exit(1);
+
+  r = rados_conf_read_file(cluster, NULL);
+  if (r < 0)
+    exit(1);
+
+  rados_conf_parse_env(cluster, NULL);
+  r = rados_connect(cluster);
+  if (r < 0)
+    exit(1);
+
+  r = RUN_ALL_TESTS();
+
+  rados_shutdown(cluster);
+
+  return r;
+}
diff --git a/src/test/libcephfs/main.cc b/src/test/libcephfs/main.cc
new file mode 100644
index 000000000..5def83c43
--- /dev/null
+++ b/src/test/libcephfs/main.cc
@@ -0,0 +1,50 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ * Copyright (C) 2016 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+
+static int update_root_mode()
+{
+  struct ceph_mount_info *admin;
+  int r = ceph_create(&admin, NULL);
+  if (r < 0)
+    return r;
+  ceph_conf_read_file(admin, NULL);
+  ceph_conf_parse_env(admin, NULL);
+  ceph_conf_set(admin, "client_permissions", "false");
+  r = ceph_mount(admin, "/");
+  if (r < 0)
+    goto out;
+  r = ceph_chmod(admin, "/", 01777);
+out:
+  ceph_shutdown(admin);
+  return r;
+}
+
+
+int main(int argc, char **argv)
+{
+  int r = update_root_mode();
+  if (r < 0)
+    exit(1);
+
+  ::testing::InitGoogleTest(&argc, argv);
+
+  srand(getpid());
+
+  return RUN_ALL_TESTS();
+}
diff --git a/src/test/libcephfs/monconfig.cc b/src/test/libcephfs/monconfig.cc
new file mode 100644
index 000000000..178278c6b
--- /dev/null
+++ b/src/test/libcephfs/monconfig.cc
@@ -0,0 +1,101 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2020 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include "common/ceph_context.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+class MonConfig : public ::testing::Test
+{
+  protected:
+    struct ceph_mount_info *ca;
+
+    void SetUp() override {
+      ASSERT_EQ(0, ceph_create(&ca, NULL));
+      ASSERT_EQ(0, ceph_conf_read_file(ca, NULL));
+      ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+    }
+
+    void TearDown() override {
+      ceph_shutdown(ca);
+    }
+
+    // Helper to remove/unset all possible mon information from ConfigProxy
+    void clear_mon_config(CephContext *cct) {
+      auto& conf = cct->_conf;
+      // Clear safe_to_start_threads, allowing updates to config values
+      conf._clear_safe_to_start_threads();
+      ASSERT_EQ(0, conf.set_val("monmap", "", nullptr));
+      ASSERT_EQ(0, conf.set_val("mon_host", "", nullptr));
+      ASSERT_EQ(0, conf.set_val("mon_dns_srv_name", "", nullptr));
+      conf.set_safe_to_start_threads();
+    }
+
+    // Helper to test basic operation on a mount
+    void use_mount(struct ceph_mount_info *mnt, std::string name_prefix) {
+      char name[20];
+      snprintf(name, sizeof(name), "%s.%d", name_prefix.c_str(), getpid());
+      int fd = ceph_open(mnt, name, O_CREAT|O_RDWR, 0644);
+      ASSERT_LE(0, fd);
+
+      ceph_close(mnt, fd);
+    }
+};
+
+TEST_F(MonConfig, MonAddrsMissing) {
+  CephContext *cct;
+
+  // Test mount failure when there is no known mon config source
+  cct = ceph_get_mount_context(ca);
+  ASSERT_NE(nullptr, cct);
+  clear_mon_config(cct);
+
+  ASSERT_EQ(-CEPHFS_ENOENT, ceph_mount(ca, NULL));
+}
+
+TEST_F(MonConfig, MonAddrsInConfigProxy) {
+  // Test a successful mount with default mon config source in ConfigProxy
+  ASSERT_EQ(0, ceph_mount(ca, NULL));
+
+  use_mount(ca, "foo");
+}
+
+TEST_F(MonConfig, MonAddrsInCct) {
+  struct ceph_mount_info *cb;
+  CephContext *cct;
+
+  // Perform mount to bootstrap mon addrs in CephContext
+  ASSERT_EQ(0, ceph_mount(ca, NULL));
+
+  // Reuse bootstrapped CephContext, clearing ConfigProxy mon addr sources
+  cct = ceph_get_mount_context(ca);
+  ASSERT_NE(nullptr, cct);
+  clear_mon_config(cct);
+  ASSERT_EQ(0, ceph_create_with_context(&cb, cct));
+
+  // Test a successful mount with only mon values in CephContext
+  ASSERT_EQ(0, ceph_mount(cb, NULL));
+
+  use_mount(ca, "bar");
+  use_mount(cb, "bar");
+
+  ceph_shutdown(cb);
+}
diff --git a/src/test/libcephfs/multiclient.cc b/src/test/libcephfs/multiclient.cc
new file mode 100644
index 000000000..c9fc038bc
--- /dev/null
+++ b/src/test/libcephfs/multiclient.cc
@@ -0,0 +1,180 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <thread>
+#ifdef __linux__
+#include <sys/xattr.h>
+#endif
+
+TEST(LibCephFS, MulticlientSimple) {
+  struct ceph_mount_info *ca, *cb;
+  ASSERT_EQ(ceph_create(&ca, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(ca, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+  ASSERT_EQ(ceph_mount(ca, NULL), 0);
+
+  ASSERT_EQ(ceph_create(&cb, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cb, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cb, NULL));
+  ASSERT_EQ(ceph_mount(cb, NULL), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo.%d", getpid());
+  int fda = ceph_open(ca, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fda);
+  int fdb = ceph_open(cb, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fdb);
+
+  char bufa[4] = "foo";
+  char bufb[4];
+
+  for (int i=0; i<10; i++) {
+    strcpy(bufa, "foo");
+    ASSERT_EQ((int)sizeof(bufa), ceph_write(ca, fda, bufa, sizeof(bufa), i*6));
+    ASSERT_EQ((int)sizeof(bufa), ceph_read(cb, fdb, bufb, sizeof(bufa), i*6));
+    ASSERT_EQ(0, memcmp(bufa, bufb, sizeof(bufa)));
+    strcpy(bufb, "bar");
+    ASSERT_EQ((int)sizeof(bufb), ceph_write(cb, fdb, bufb, sizeof(bufb), i*6+3));
+    ASSERT_EQ((int)sizeof(bufb), ceph_read(ca, fda, bufa, sizeof(bufb), i*6+3));
+    ASSERT_EQ(0, memcmp(bufa, bufb, sizeof(bufa)));
+  }
+
+  ceph_close(ca, fda);
+  ceph_close(cb, fdb);
+
+  ceph_shutdown(ca);
+  ceph_shutdown(cb);
+}
+
+TEST(LibCephFS, MulticlientHoleEOF) {
+  struct ceph_mount_info *ca, *cb;
+  ASSERT_EQ(ceph_create(&ca, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(ca, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(ca, NULL));
+  ASSERT_EQ(ceph_mount(ca, NULL), 0);
+
+  ASSERT_EQ(ceph_create(&cb, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cb, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cb, NULL));
+  ASSERT_EQ(ceph_mount(cb, NULL), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo.%d", getpid());
+  int fda = ceph_open(ca, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fda);
+  int fdb = ceph_open(cb, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fdb);
+
+  ASSERT_EQ(3, ceph_write(ca, fda, "foo", 3, 0));
+  ASSERT_EQ(0, ceph_ftruncate(ca, fda, 1000000));
+
+  char buf[4];
+  ASSERT_EQ(2, ceph_read(cb, fdb, buf, sizeof(buf), 1000000-2));
+  ASSERT_EQ(0, buf[0]);
+  ASSERT_EQ(0, buf[1]);
+
+  ceph_close(ca, fda);
+  ceph_close(cb, fdb);
+
+  ceph_shutdown(ca);
+  ceph_shutdown(cb);
+}
+
+static void write_func(bool *stop)
+{
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo.%d", getpid());
+  int fd = ceph_open(cmount, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fd);
+
+  int buf_size = 4096;
+  char *buf = (char *)malloc(buf_size);
+  if (!buf) {
+    *stop = true;
+    printf("write_func failed to allocate buffer!");
+    return;
+  }
+  memset(buf, 1, buf_size);
+
+  while (!(*stop)) {
+    int i;
+
+    // truncate the file size to 4096 will set the max_size to 4MB.
+    ASSERT_EQ(0, ceph_ftruncate(cmount, fd, 4096));
+
+    // write 4MB + extra 64KB data will make client to trigger to
+    // call check_cap() to report new size. And if MDS is revoking
+    // the Fsxrw caps and we are still holding the Fw caps and will
+    // trigger tracker#57244.
+    for (i = 0; i < 1040; i++) {
+      ASSERT_EQ(ceph_write(cmount, fd, buf, buf_size, 0), buf_size);
+    }
+  }
+
+  ceph_shutdown(cmount);
+}
+
+static void setattr_func(bool *stop)
+{
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char name[20];
+  snprintf(name, sizeof(name), "foo.%d", getpid());
+  int fd = ceph_open(cmount, name, O_CREAT|O_RDWR, 0644);
+  ASSERT_LE(0, fd);
+
+  while (!(*stop)) {
+    // setattr will make the MDS to acquire xlock for the filelock and
+    // force to revoke caps from clients
+    struct ceph_statx stx = {.stx_size = 0};
+    ASSERT_EQ(ceph_fsetattrx(cmount, fd, &stx, CEPH_SETATTR_SIZE), 0);
+  }
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, MulticlientRevokeCaps) {
+  std::thread thread1, thread2;
+  bool stop = false;
+  int wait = 60; // in second
+
+  thread1 = std::thread(write_func, &stop);
+  thread2 = std::thread(setattr_func, &stop);
+
+  printf(" Will run test for %d seconds!\n", wait);
+  sleep(wait);
+  stop = true;
+
+  thread1.join();
+  thread2.join();
+}
diff --git a/src/test/libcephfs/newops.cc b/src/test/libcephfs/newops.cc
new file mode 100644
index 000000000..2a4573b9f
--- /dev/null
+++ b/src/test/libcephfs/newops.cc
@@ -0,0 +1,87 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2021 Red Hat Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "gtest/gtest-spi.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock-more-matchers.h"
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include "mds/mdstypes.h"
+#include "include/stat.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#endif
+
+#include <fmt/format.h>
+#include <map>
+#include <vector>
+#include <thread>
+#include <regex>
+#include <string>
+
+using ::testing::AnyOf;
+using ::testing::Gt;
+using ::testing::Eq;
+using namespace std;
+
+/*
+ * Test this with different ceph versions
+ */
+
+TEST(LibCephFS, NewOPs)
+{
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  const char *test_path = "test_newops_dir";
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, test_path, 0777));
+
+  {
+    char value[1024] = "";
+    int r = ceph_getxattr(cmount, test_path, "ceph.dir.pin.random", (void*)value, sizeof(value));
+    // Clients will return -CEPHFS_ENODATA if new getvxattr op not support yet.
+    EXPECT_THAT(r, AnyOf(Gt(0), Eq(-CEPHFS_ENODATA)));
+  }
+
+  {
+    double val = (double)1.0/(double)128.0;
+    std::stringstream ss;
+    ss << val;
+    int r = ceph_setxattr(cmount, test_path, "ceph.dir.pin.random", (void*)ss.str().c_str(), strlen(ss.str().c_str()), XATTR_CREATE);
+    // Old cephs will return -CEPHFS_EINVAL if not support "ceph.dir.pin.random" yet.
+    EXPECT_THAT(r, AnyOf(Eq(0), Eq(-CEPHFS_EINVAL)));
+
+    char value[1024] = "";
+    r = ceph_getxattr(cmount, test_path, "ceph.dir.pin.random", (void*)value, sizeof(value));
+    // Clients will return -CEPHFS_ENODATA if new getvxattr op not support yet.
+    EXPECT_THAT(r, AnyOf(Gt(0), Eq(-CEPHFS_ENODATA)));
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, test_path));
+
+  ceph_shutdown(cmount);
+}
diff --git a/src/test/libcephfs/quota.cc b/src/test/libcephfs/quota.cc
new file mode 100644
index 000000000..00afb3d09
--- /dev/null
+++ b/src/test/libcephfs/quota.cc
@@ -0,0 +1,167 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ * Copyright (C) 2022 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/compat.h"
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include "mds/mdstypes.h"
+#include "include/stat.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#endif
+
+TEST(LibCephFS, SnapQuota) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_snap_dir_quota_xattr[256];
+  char test_snap_subdir_quota_xattr[256];
+  char test_snap_subdir_noquota_xattr[256];
+  char xattrk[128];
+  char xattrv[128];
+  char c_temp[PATH_MAX];
+  char gxattrv[128];
+  int xbuflen = sizeof(gxattrv);
+  pid_t mypid = getpid();
+
+  // create dir and set quota
+  sprintf(test_snap_dir_quota_xattr, "test_snap_dir_quota_xattr_%d", mypid);
+  ASSERT_EQ(0, ceph_mkdir(cmount, test_snap_dir_quota_xattr, 0777));
+
+  sprintf(xattrk, "ceph.quota.max_bytes");
+  sprintf(xattrv, "65536");
+  ASSERT_EQ(0, ceph_setxattr(cmount, test_snap_dir_quota_xattr, xattrk, (void *)xattrv, 5, XATTR_CREATE));
+
+  // create subdir and set quota
+  sprintf(test_snap_subdir_quota_xattr, "test_snap_dir_quota_xattr_%d/subdir_quota", mypid);
+  ASSERT_EQ(0, ceph_mkdirs(cmount, test_snap_subdir_quota_xattr, 0777));
+
+  sprintf(xattrk, "ceph.quota.max_bytes");
+  sprintf(xattrv, "32768");
+  ASSERT_EQ(0, ceph_setxattr(cmount, test_snap_subdir_quota_xattr, xattrk, (void *)xattrv, 5, XATTR_CREATE));
+
+  // create subdir with no quota
+  sprintf(test_snap_subdir_noquota_xattr, "test_snap_dir_quota_xattr_%d/subdir_noquota", mypid);
+  ASSERT_EQ(0, ceph_mkdirs(cmount, test_snap_subdir_noquota_xattr, 0777));
+
+  // snapshot dir
+  sprintf(c_temp, "/.snap/test_snap_dir_quota_xattr_snap_%d", mypid);
+  ASSERT_EQ(0, ceph_mkdirs(cmount, c_temp, 0777));
+
+  // check dir quota under snap
+  sprintf(c_temp, "/.snap/test_snap_dir_quota_xattr_snap_%d/test_snap_dir_quota_xattr_%d", mypid, mypid);
+  int alen = ceph_getxattr(cmount, c_temp, "ceph.quota.max_bytes", (void *)gxattrv, xbuflen);
+  ASSERT_LT(0, alen);
+  ASSERT_LT(alen, xbuflen);
+  gxattrv[alen] = '\0';
+  ASSERT_STREQ(gxattrv, "65536");
+
+  // check subdir quota under snap
+  sprintf(c_temp, "/.snap/test_snap_dir_quota_xattr_snap_%d/test_snap_dir_quota_xattr_%d/subdir_quota", mypid, mypid);
+  alen = ceph_getxattr(cmount, c_temp, "ceph.quota.max_bytes", (void *)gxattrv, xbuflen);
+  ASSERT_LT(0, alen);
+  ASSERT_LT(alen, xbuflen);
+  gxattrv[alen] = '\0';
+  ASSERT_STREQ(gxattrv, "32768");
+
+  // ensure subdir noquota xattr under snap
+  sprintf(c_temp, "/.snap/test_snap_dir_quota_xattr_snap_%d/test_snap_dir_quota_xattr_%d/subdir_noquota", mypid, mypid);
+  EXPECT_EQ(-CEPHFS_ENODATA, ceph_getxattr(cmount, c_temp, "ceph.quota.max_bytes", (void *)gxattrv, xbuflen));
+
+  // listxattr() shouldn't return ceph.quota.max_bytes vxattr
+  sprintf(c_temp, "/.snap/test_snap_dir_quota_xattr_snap_%d/test_snap_dir_quota_xattr_%d", mypid, mypid);
+  char xattrlist[512];
+  int len = ceph_listxattr(cmount, c_temp, xattrlist, sizeof(xattrlist));
+  ASSERT_GE(sizeof(xattrlist), (size_t)len);
+  char *p = xattrlist;
+  int found = 0;
+  while (len > 0) {
+    if (strcmp(p, "ceph.quota.max_bytes") == 0)
+      found++;
+    len -= strlen(p) + 1;
+    p += strlen(p) + 1;
+  }
+  ASSERT_EQ(found, 0);
+
+  ceph_shutdown(cmount);
+}
+
+void statfs_quota_size_check(struct ceph_mount_info *cmount, const char *path,
+                             int blocks, int bsize)
+{
+  struct statvfs stvfs;
+
+  ASSERT_EQ(0, ceph_statfs(cmount, path, &stvfs));
+  ASSERT_EQ(blocks, stvfs.f_blocks);
+  ASSERT_EQ(bsize, stvfs.f_bsize);
+  ASSERT_EQ(bsize, stvfs.f_frsize);
+}
+
+TEST(LibCephFS, QuotaRealm) {
+  struct ceph_mount_info *cmount, *pmount1, *pmount2;
+  char test_quota_realm_pdir[128];
+  char test_quota_realm_cdir[256];
+  char xattrk[32];
+  char xattrv[16];
+
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  pid_t mypid = getpid();
+
+  // create parent directory and set quota size
+  sprintf(test_quota_realm_pdir, "/test_quota_realm_pdir_%d", mypid);
+  ASSERT_EQ(0, ceph_mkdir(cmount, test_quota_realm_pdir, 0777));
+  sprintf(xattrk, "ceph.quota.max_bytes");
+  sprintf(xattrv, "8388608"); // 8MB
+  ASSERT_EQ(0, ceph_setxattr(cmount, test_quota_realm_pdir, xattrk, (void *)xattrv, 7, XATTR_CREATE));
+
+  // create child directory and set quota file
+  sprintf(test_quota_realm_cdir, "%s/test_quota_realm_cdir", test_quota_realm_pdir);
+  ASSERT_EQ(0, ceph_mkdir(cmount, test_quota_realm_cdir, 0777));
+  sprintf(xattrk, "ceph.quota.max_files");
+  sprintf(xattrv, "1024"); // 1K files
+  ASSERT_EQ(0, ceph_setxattr(cmount, test_quota_realm_cdir, xattrk, (void *)xattrv, 4, XATTR_CREATE));
+
+  ASSERT_EQ(ceph_create(&pmount1, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(pmount1, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(pmount1, NULL));
+  ASSERT_EQ(ceph_mount(pmount1, test_quota_realm_pdir), 0);
+  statfs_quota_size_check(pmount1, "/", 2, 4194304); // 8MB
+
+  ASSERT_EQ(ceph_create(&pmount2, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(pmount2, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(pmount2, NULL));
+  ASSERT_EQ(ceph_mount(pmount2, test_quota_realm_cdir), 0);
+  statfs_quota_size_check(pmount2, "/", 2, 4194304); // 8MB
+
+  ceph_shutdown(pmount1);
+  ceph_shutdown(pmount2);
+  ceph_shutdown(cmount);
+}
diff --git a/src/test/libcephfs/readdir_r_cb.cc b/src/test/libcephfs/readdir_r_cb.cc
new file mode 100644
index 000000000..959c276f1
--- /dev/null
+++ b/src/test/libcephfs/readdir_r_cb.cc
@@ -0,0 +1,65 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include <errno.h>
+#include <fcntl.h>
+
+TEST(LibCephFS, ReaddirRCB) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  char c_dir[256];
+  sprintf(c_dir, "/readdir_r_cb_tests_%d", getpid());
+  struct ceph_dir_result *dirp;
+  ASSERT_EQ(0, ceph_mkdirs(cmount, c_dir, 0777));
+  ASSERT_LE(0, ceph_opendir(cmount, c_dir, &dirp));
+
+  // dir is empty, check that it only contains . and ..
+  int buflen = 100;
+  char *buf = new char[buflen];
+  // . is 2, .. is 3 (for null terminators)
+  ASSERT_EQ(5, ceph_getdnames(cmount, dirp, buf, buflen));
+  char c_file[256];
+  sprintf(c_file, "/readdir_r_cb_tests_%d/foo", getpid());
+  int fd = ceph_open(cmount, c_file, O_CREAT, 0777);
+  ASSERT_LT(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  // check correctness with one entry
+  ASSERT_LE(0, ceph_closedir(cmount, dirp));
+  ASSERT_LE(0, ceph_opendir(cmount, c_dir, &dirp));
+  ASSERT_EQ(9, ceph_getdnames(cmount, dirp, buf, buflen)); // ., .., foo
+
+  // check correctness if buffer is too small
+  ASSERT_LE(0, ceph_closedir(cmount, dirp));
+  ASSERT_GE(0, ceph_opendir(cmount, c_dir, &dirp));
+  ASSERT_EQ(-CEPHFS_ERANGE, ceph_getdnames(cmount, dirp, buf, 1));
+
+  //check correctness if it needs to split listing
+  ASSERT_LE(0, ceph_closedir(cmount, dirp));
+  ASSERT_LE(0, ceph_opendir(cmount, c_dir, &dirp));
+  ASSERT_EQ(5, ceph_getdnames(cmount, dirp, buf, 6));
+  ASSERT_EQ(4, ceph_getdnames(cmount, dirp, buf, 6));
+
+  // free cmount after finishing testing
+  ASSERT_LE(0, ceph_closedir(cmount, dirp));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ASSERT_EQ(0, ceph_release(cmount));
+}
diff --git a/src/test/libcephfs/reclaim.cc b/src/test/libcephfs/reclaim.cc
new file mode 100644
index 000000000..736ed5759
--- /dev/null
+++ b/src/test/libcephfs/reclaim.cc
@@ -0,0 +1,163 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Tests for Ceph delegation handling
+ *
+ * (c) 2017, Jeff Layton <jlayton@redhat.com>
+ */
+
+#include "gtest/gtest.h"
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include "include/stat.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <sys/uio.h>
+#include <libgen.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/xattr.h>
+#include <limits.h>
+#endif
+
+#ifdef __FreeBSD__
+#include <sys/wait.h>
+#endif
+
+
+#include <map>
+#include <vector>
+#include <thread>
+#include <atomic>
+
+#define	CEPHFS_RECLAIM_TIMEOUT		60
+
+static int dying_client(int argc, char **argv)
+{
+  struct ceph_mount_info *cmount;
+
+  /* Caller must pass in the uuid */
+  if (argc < 2)
+    return 1;
+
+  if (ceph_create(&cmount, nullptr) != 0)
+    return 1;
+
+  if (ceph_conf_read_file(cmount, nullptr) != 0)
+    return 1;
+
+  if (ceph_conf_parse_env(cmount, nullptr) != 0)
+    return 1;
+
+  if (ceph_init(cmount) != 0)
+    return 1;
+
+  ceph_set_session_timeout(cmount, CEPHFS_RECLAIM_TIMEOUT);
+
+  if (ceph_start_reclaim(cmount, argv[1], CEPH_RECLAIM_RESET) != -CEPHFS_ENOENT)
+    return 1;
+
+  ceph_set_uuid(cmount, argv[1]);
+
+  if (ceph_mount(cmount, "/") != 0)
+    return 1;
+
+  Inode *root, *file;
+  if (ceph_ll_lookup_root(cmount, &root) != 0)
+    return 1;
+
+  Fh *fh;
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  if (ceph_ll_create(cmount, root, argv[1], 0666, O_RDWR|O_CREAT|O_EXCL,
+		      &file, &fh, &stx, 0, 0, perms) != 0)
+    return 1;
+
+  return 0;
+}
+
+TEST(LibCephFS, ReclaimReset) {
+  pid_t		pid;
+  char		uuid[256];
+  const char	*exe = "/proc/self/exe";
+
+  sprintf(uuid, "simplereclaim:%x", getpid());
+
+  pid = fork();
+  ASSERT_GE(pid, 0);
+  if (pid == 0) {
+    errno = 0;
+    execl(exe, exe, uuid, nullptr);
+    /* It won't be zero of course, which is sort of the point... */
+    ASSERT_EQ(errno, 0);
+  }
+
+  /* parent - wait for child to exit */
+  int ret;
+  pid_t wp = wait(&ret);
+  ASSERT_GE(wp, 0);
+  ASSERT_EQ(WIFEXITED(ret), true);
+  ASSERT_EQ(WEXITSTATUS(ret), 0);
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, nullptr), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, nullptr), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, nullptr));
+  ASSERT_EQ(ceph_init(cmount), 0);
+  ceph_set_session_timeout(cmount, CEPHFS_RECLAIM_TIMEOUT);
+  ASSERT_EQ(ceph_start_reclaim(cmount, uuid, CEPH_RECLAIM_RESET), 0);
+  ceph_set_uuid(cmount, uuid);
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  Inode *root, *file;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+  UserPerm *perms = ceph_mount_perms(cmount);
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_ll_lookup(cmount, root, uuid, &file, &stx, 0, 0, perms), 0);
+  Fh *fh;
+  ASSERT_EQ(ceph_ll_open(cmount, file, O_WRONLY, &fh, perms), 0);
+
+  ceph_unmount(cmount);
+  ceph_release(cmount);
+}
+
+static int update_root_mode()
+{
+  struct ceph_mount_info *admin;
+  int r = ceph_create(&admin, nullptr);
+  if (r < 0)
+    return r;
+  ceph_conf_read_file(admin, nullptr);
+  ceph_conf_parse_env(admin, nullptr);
+  ceph_conf_set(admin, "client_permissions", "false");
+  r = ceph_mount(admin, "/");
+  if (r < 0)
+    goto out;
+  r = ceph_chmod(admin, "/", 01777);
+out:
+  ceph_shutdown(admin);
+  return r;
+}
+
+int main(int argc, char **argv)
+{
+  int r = update_root_mode();
+  if (r < 0)
+    exit(1);
+
+  ::testing::InitGoogleTest(&argc, argv);
+
+  if (argc > 1)
+    return dying_client(argc, argv);
+
+  srand(getpid());
+
+  return RUN_ALL_TESTS();
+}
diff --git a/src/test/libcephfs/recordlock.cc b/src/test/libcephfs/recordlock.cc
new file mode 100644
index 000000000..ad108b69c
--- /dev/null
+++ b/src/test/libcephfs/recordlock.cc
@@ -0,0 +1,1105 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *               2016 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <pthread.h>
+#include "gtest/gtest.h"
+#ifndef GTEST_IS_THREADSAFE
+#error "!GTEST_IS_THREADSAFE"
+#endif
+
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include <errno.h>
+#include <sys/fcntl.h>
+#include <unistd.h>
+#include <sys/file.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+
+#include <stdlib.h>
+#include <semaphore.h>
+#include <time.h>
+
+#ifndef _WIN32
+#include <sys/mman.h>
+#endif
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#elif __FreeBSD__
+#include <sys/types.h>
+#include <sys/wait.h>
+#endif
+
+#include "include/ceph_assert.h"
+#include "ceph_pthread_self.h"
+
+// Startup common: create and mount ceph fs
+#define STARTUP_CEPH() do {				\
+    ASSERT_EQ(0, ceph_create(&cmount, NULL));		\
+    ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));	\
+    ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));	\
+    ASSERT_EQ(0, ceph_mount(cmount, NULL));		\
+  } while(0)
+
+// Cleanup common: unmount and release ceph fs
+#define CLEANUP_CEPH() do {			\
+    ASSERT_EQ(0, ceph_unmount(cmount));		\
+    ASSERT_EQ(0, ceph_release(cmount));		\
+  } while(0)
+
+static const mode_t fileMode = S_IRWXU | S_IRWXG | S_IRWXO;
+
+// Default wait time for normal and "slow" operations
+// (5" should be enough in case of network congestion)
+static const long waitMs = 10;
+static const long waitSlowMs = 5000;
+
+// Get the absolute struct timespec reference from now + 'ms' milliseconds
+static const struct timespec* abstime(struct timespec &ts, long ms) {
+  if (clock_gettime(CLOCK_REALTIME, &ts) == -1) {
+    ceph_abort();
+  }
+  ts.tv_nsec += ms * 1000000;
+  ts.tv_sec += ts.tv_nsec / 1000000000;
+  ts.tv_nsec %= 1000000000;
+  return &ts;
+}
+
+/* Basic locking */
+
+TEST(LibCephFS, BasicRecordLocking) {
+  struct ceph_mount_info *cmount = NULL;
+  STARTUP_CEPH();
+
+  char c_file[1024];
+  sprintf(c_file, "recordlock_test_%d", getpid());
+  Fh *fh = NULL;
+  Inode *root = NULL, *inode = NULL;
+  struct ceph_statx stx;
+  int rc;
+  struct flock lock1, lock2;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  // Get the root inode
+  rc = ceph_ll_lookup_root(cmount, &root);
+  ASSERT_EQ(rc, 0); 
+
+  // Get the inode and Fh corresponding to c_file
+  rc = ceph_ll_create(cmount, root, c_file, fileMode, O_RDWR | O_CREAT,
+		      &inode, &fh, &stx, 0, 0, perms);
+  ASSERT_EQ(rc, 0); 
+
+  // write lock twice
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, 42, false));
+
+  lock2.l_type = F_WRLCK;
+  lock2.l_whence = SEEK_SET;
+  lock2.l_start = 0;
+  lock2.l_len = 1024;
+  lock2.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock2, 43, false));
+
+  // Now try a conflicting read lock
+  lock2.l_type = F_RDLCK;
+  lock2.l_whence = SEEK_SET;
+  lock2.l_start = 100;
+  lock2.l_len = 100;
+  lock2.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock2, 43, false));
+
+  // Now do a getlk
+  ASSERT_EQ(0, ceph_ll_getlk(cmount, fh, &lock2, 43));
+  ASSERT_EQ(lock2.l_type, F_WRLCK);
+  ASSERT_EQ(lock2.l_start, 0);
+  ASSERT_EQ(lock2.l_len, 1024);
+  ASSERT_EQ(lock2.l_pid, getpid());
+
+  // Extend the range of the write lock
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 1024;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, 42, false));
+
+  // Now do a getlk
+  lock2.l_type = F_RDLCK;
+  lock2.l_whence = SEEK_SET;
+  lock2.l_start = 100;
+  lock2.l_len = 100;
+  lock2.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_getlk(cmount, fh, &lock2, 43));
+  ASSERT_EQ(lock2.l_type, F_WRLCK);
+  ASSERT_EQ(lock2.l_start, 0);
+  ASSERT_EQ(lock2.l_len, 2048);
+  ASSERT_EQ(lock2.l_pid, getpid());
+
+  // Now release part of the range
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 512;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, 42, false));
+
+  // Now do a getlk to check 1st part
+  lock2.l_type = F_RDLCK;
+  lock2.l_whence = SEEK_SET;
+  lock2.l_start = 100;
+  lock2.l_len = 100;
+  lock2.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_getlk(cmount, fh, &lock2, 43));
+  ASSERT_EQ(lock2.l_type, F_WRLCK);
+  ASSERT_EQ(lock2.l_start, 0);
+  ASSERT_EQ(lock2.l_len, 512);
+  ASSERT_EQ(lock2.l_pid, getpid());
+
+  // Now do a getlk to check 2nd part
+  lock2.l_type = F_RDLCK;
+  lock2.l_whence = SEEK_SET;
+  lock2.l_start = 2000;
+  lock2.l_len = 100;
+  lock2.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_getlk(cmount, fh, &lock2, 43));
+  ASSERT_EQ(lock2.l_type, F_WRLCK);
+  ASSERT_EQ(lock2.l_start, 1536);
+  ASSERT_EQ(lock2.l_len, 512);
+  ASSERT_EQ(lock2.l_pid, getpid());
+
+  // Now do a getlk to check released part
+  lock2.l_type = F_RDLCK;
+  lock2.l_whence = SEEK_SET;
+  lock2.l_start = 512;
+  lock2.l_len = 1024;
+  lock2.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_getlk(cmount, fh, &lock2, 43));
+  ASSERT_EQ(lock2.l_type, F_UNLCK);
+  ASSERT_EQ(lock2.l_start, 512);
+  ASSERT_EQ(lock2.l_len, 1024);
+  ASSERT_EQ(lock2.l_pid, getpid());
+
+  // Now downgrade the 1st part of the lock
+  lock1.l_type = F_RDLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 512;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, 42, false));
+
+  // Now do a getlk to check 1st part
+  lock2.l_type = F_WRLCK;
+  lock2.l_whence = SEEK_SET;
+  lock2.l_start = 100;
+  lock2.l_len = 100;
+  lock2.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_getlk(cmount, fh, &lock2, 43));
+  ASSERT_EQ(lock2.l_type, F_RDLCK);
+  ASSERT_EQ(lock2.l_start, 0);
+  ASSERT_EQ(lock2.l_len, 512);
+  ASSERT_EQ(lock2.l_pid, getpid());
+
+  // Now upgrade the 1st part of the lock
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 512;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, 42, false));
+
+  // Now do a getlk to check 1st part
+  lock2.l_type = F_WRLCK;
+  lock2.l_whence = SEEK_SET;
+  lock2.l_start = 100;
+  lock2.l_len = 100;
+  lock2.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_getlk(cmount, fh, &lock2, 43));
+  ASSERT_EQ(lock2.l_type, F_WRLCK);
+  ASSERT_EQ(lock2.l_start, 0);
+  ASSERT_EQ(lock2.l_len, 512);
+  ASSERT_EQ(lock2.l_pid, getpid());
+
+  ASSERT_EQ(0, ceph_ll_close(cmount, fh));
+  ASSERT_EQ(0, ceph_ll_unlink(cmount, root, c_file, perms));
+  CLEANUP_CEPH();
+}
+
+/* Locking in different threads */
+
+// Used by ConcurrentLocking test
+struct str_ConcurrentRecordLocking {
+  const char *file;
+  struct ceph_mount_info *cmount;  // !NULL if shared
+  sem_t sem[2];
+  sem_t semReply[2];
+  void sem_init(int pshared) {
+    ASSERT_EQ(0, ::sem_init(&sem[0], pshared, 0));
+    ASSERT_EQ(0, ::sem_init(&sem[1], pshared, 0));
+    ASSERT_EQ(0, ::sem_init(&semReply[0], pshared, 0));
+    ASSERT_EQ(0, ::sem_init(&semReply[1], pshared, 0));
+  }
+  void sem_destroy() {
+    ASSERT_EQ(0, ::sem_destroy(&sem[0]));
+    ASSERT_EQ(0, ::sem_destroy(&sem[1]));
+    ASSERT_EQ(0, ::sem_destroy(&semReply[0]));
+    ASSERT_EQ(0, ::sem_destroy(&semReply[1]));
+  }
+};
+
+// Wakeup main (for (N) steps)
+#define PING_MAIN(n) ASSERT_EQ(0, sem_post(&s.sem[n%2]))
+// Wait for main to wake us up (for (RN) steps)
+#define WAIT_MAIN(n) \
+  ASSERT_EQ(0, sem_timedwait(&s.semReply[n%2], abstime(ts, waitSlowMs)))
+
+// Wakeup worker (for (RN) steps)
+#define PING_WORKER(n) ASSERT_EQ(0, sem_post(&s.semReply[n%2]))
+// Wait for worker to wake us up (for (N) steps)
+#define WAIT_WORKER(n) \
+  ASSERT_EQ(0, sem_timedwait(&s.sem[n%2], abstime(ts, waitSlowMs)))
+// Worker shall not wake us up (for (N) steps)
+#define NOT_WAIT_WORKER(n) \
+  ASSERT_EQ(-1, sem_timedwait(&s.sem[n%2], abstime(ts, waitMs)))
+
+// Do twice an operation
+#define TWICE(EXPR) do {			\
+    EXPR;					\
+    EXPR;					\
+  } while(0)
+
+/* Locking in different threads */
+
+// Used by ConcurrentLocking test
+static void thread_ConcurrentRecordLocking(str_ConcurrentRecordLocking& s) {
+  struct ceph_mount_info *const cmount = s.cmount;
+  Fh *fh = NULL;
+  Inode *root = NULL, *inode = NULL;
+  struct ceph_statx stx;
+  struct flock lock1;
+  int rc;
+  struct timespec ts;
+
+  // Get the root inode
+  rc = ceph_ll_lookup_root(cmount, &root);
+  ASSERT_EQ(rc, 0); 
+
+  // Get the inode and Fh corresponding to c_file
+  rc = ceph_ll_create(cmount, root, s.file, fileMode, O_RDWR | O_CREAT,
+		      &inode, &fh, &stx, 0, 0, ceph_mount_perms(cmount));
+  ASSERT_EQ(rc, 0); 
+
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+
+  PING_MAIN(1); // (1)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), true));
+  PING_MAIN(2); // (2)
+
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  PING_MAIN(3); // (3)
+
+  lock1.l_type = F_RDLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), true));
+  PING_MAIN(4); // (4)
+
+  WAIT_MAIN(1); // (R1)
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  PING_MAIN(5); // (5)
+
+  WAIT_MAIN(2); // (R2)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), true));
+  PING_MAIN(6); // (6)
+
+  WAIT_MAIN(3); // (R3)
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  PING_MAIN(7); // (7)
+
+  ASSERT_EQ(0, ceph_ll_close(cmount, fh));
+}
+
+// Used by ConcurrentRecordLocking test
+static void* thread_ConcurrentRecordLocking_(void *arg) {
+  str_ConcurrentRecordLocking *const s =
+    reinterpret_cast<str_ConcurrentRecordLocking*>(arg);
+  thread_ConcurrentRecordLocking(*s);
+  return NULL;
+}
+
+TEST(LibCephFS, ConcurrentRecordLocking) {
+  const pid_t mypid = getpid();
+  struct ceph_mount_info *cmount;
+  STARTUP_CEPH();
+
+  char c_file[1024];
+  sprintf(c_file, "recordlock_test_%d", mypid);
+  Fh *fh = NULL;
+  Inode *root = NULL, *inode = NULL;
+  struct ceph_statx stx;
+  struct flock lock1;
+  int rc;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  // Get the root inode
+  rc = ceph_ll_lookup_root(cmount, &root);
+  ASSERT_EQ(rc, 0); 
+
+  // Get the inode and Fh corresponding to c_file
+  rc = ceph_ll_create(cmount, root, c_file, fileMode, O_RDWR | O_CREAT,
+		      &inode, &fh, &stx, 0, 0, perms);
+  ASSERT_EQ(rc, 0); 
+
+  // Lock
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), true));
+
+  // Start locker thread
+  pthread_t thread;
+  struct timespec ts;
+  str_ConcurrentRecordLocking s = { c_file, cmount };
+  s.sem_init(0);
+  ASSERT_EQ(0, pthread_create(&thread, NULL, thread_ConcurrentRecordLocking_, &s));
+  // Synchronization point with thread (failure: thread is dead)
+  WAIT_WORKER(1); // (1)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(2); // (2)
+
+  // Unlock
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+
+  // Shall have lock
+  // Synchronization point with thread (failure: thread is dead)
+  WAIT_WORKER(2); // (2)
+
+  // Synchronization point with thread (failure: thread is dead)
+  WAIT_WORKER(3); // (3)
+
+  // Wait for thread to share lock
+  WAIT_WORKER(4); // (4)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  lock1.l_type = F_RDLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+
+  // Wake up thread to unlock shared lock
+  PING_WORKER(1); // (R1)
+  WAIT_WORKER(5); // (5)
+
+  // Now we can lock exclusively
+  // Upgrade to exclusive lock (as per POSIX)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), true));
+
+  // Wake up thread to lock shared lock
+  PING_WORKER(2); // (R2)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(6); // (6)
+
+  // Release lock ; thread will get it
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  WAIT_WORKER(6); // (6)
+
+  // We no longer have the lock
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  lock1.l_type = F_RDLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+
+  // Wake up thread to unlock exclusive lock
+  PING_WORKER(3); // (R3)
+  WAIT_WORKER(7); // (7)
+
+  // We can lock it again
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+
+  // Cleanup
+  void *retval = (void*) (uintptr_t) -1;
+  ASSERT_EQ(0, pthread_join(thread, &retval));
+  ASSERT_EQ(NULL, retval);
+  s.sem_destroy();
+  ASSERT_EQ(0, ceph_ll_close(cmount, fh));
+  ASSERT_EQ(0, ceph_ll_unlink(cmount, root, c_file, perms));
+  CLEANUP_CEPH();
+}
+
+TEST(LibCephFS, ThreesomeRecordLocking) {
+  const pid_t mypid = getpid();
+  struct ceph_mount_info *cmount;
+  STARTUP_CEPH();
+
+  char c_file[1024];
+  sprintf(c_file, "recordlock_test_%d", mypid);
+  Fh *fh = NULL;
+  Inode *root = NULL, *inode = NULL;
+  struct ceph_statx stx;
+  struct flock lock1;
+  int rc;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  // Get the root inode
+  rc = ceph_ll_lookup_root(cmount, &root);
+  ASSERT_EQ(rc, 0); 
+
+  // Get the inode and Fh corresponding to c_file
+  rc = ceph_ll_create(cmount, root, c_file, fileMode, O_RDWR | O_CREAT,
+		      &inode, &fh, &stx, 0, 0, perms);
+  ASSERT_EQ(rc, 0); 
+
+  // Lock
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), true));
+
+  // Start locker thread
+  pthread_t thread[2];
+  struct timespec ts;
+  str_ConcurrentRecordLocking s = { c_file, cmount };
+  s.sem_init(0);
+  ASSERT_EQ(0, pthread_create(&thread[0], NULL, thread_ConcurrentRecordLocking_, &s));
+  ASSERT_EQ(0, pthread_create(&thread[1], NULL, thread_ConcurrentRecordLocking_, &s));
+  // Synchronization point with thread (failure: thread is dead)
+  TWICE(WAIT_WORKER(1)); // (1)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(2); // (2)
+
+  // Unlock
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+
+  // Shall have lock
+  TWICE(// Synchronization point with thread (failure: thread is dead)
+	WAIT_WORKER(2); // (2)
+	
+	// Synchronization point with thread (failure: thread is dead)
+	WAIT_WORKER(3)); // (3)
+  
+  // Wait for thread to share lock
+  TWICE(WAIT_WORKER(4)); // (4)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  lock1.l_type = F_RDLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+
+  // Wake up thread to unlock shared lock
+  TWICE(PING_WORKER(1); // (R1)
+	WAIT_WORKER(5)); // (5)
+
+  // Now we can lock exclusively
+  // Upgrade to exclusive lock (as per POSIX)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), true));
+
+  TWICE(  // Wake up thread to lock shared lock
+	PING_WORKER(2); // (R2)
+	
+	// Shall not have lock immediately
+	NOT_WAIT_WORKER(6)); // (6)
+  
+  // Release lock ; thread will get it
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  TWICE(WAIT_WORKER(6); // (6)
+	
+	// We no longer have the lock
+	lock1.l_type = F_WRLCK;
+	lock1.l_whence = SEEK_SET;
+	lock1.l_start = 0;
+	lock1.l_len = 1024;
+	lock1.l_pid = getpid();
+	ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+	lock1.l_type = F_RDLCK;
+	lock1.l_whence = SEEK_SET;
+	lock1.l_start = 0;
+	lock1.l_len = 1024;
+	lock1.l_pid = getpid();
+	ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+	
+	// Wake up thread to unlock exclusive lock
+	PING_WORKER(3); // (R3)
+	WAIT_WORKER(7); // (7)
+	);
+  
+  // We can lock it again
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+
+  // Cleanup
+  void *retval = (void*) (uintptr_t) -1;
+  ASSERT_EQ(0, pthread_join(thread[0], &retval));
+  ASSERT_EQ(NULL, retval);
+  ASSERT_EQ(0, pthread_join(thread[1], &retval));
+  ASSERT_EQ(NULL, retval);
+  s.sem_destroy();
+  ASSERT_EQ(0, ceph_ll_close(cmount, fh));
+  ASSERT_EQ(0, ceph_ll_unlink(cmount, root, c_file, perms));
+  CLEANUP_CEPH();
+}
+
+/* Locking in different processes */
+
+#define PROCESS_SLOW_MS() \
+  static const long waitMs = 100; \
+  (void) waitMs
+
+// Used by ConcurrentLocking test
+static void process_ConcurrentRecordLocking(str_ConcurrentRecordLocking& s) {
+  const pid_t mypid = getpid();
+  PROCESS_SLOW_MS();
+
+  struct ceph_mount_info *cmount = NULL;
+  struct timespec ts;
+  Fh *fh = NULL;
+  Inode *root = NULL, *inode = NULL;
+  struct ceph_statx stx;
+  int rc;
+  struct flock lock1;
+
+  STARTUP_CEPH();
+  s.cmount = cmount;
+
+  // Get the root inode
+  rc = ceph_ll_lookup_root(cmount, &root);
+  ASSERT_EQ(rc, 0); 
+
+  // Get the inode and Fh corresponding to c_file
+  rc = ceph_ll_create(cmount, root, s.file, fileMode, O_RDWR | O_CREAT,
+		      &inode, &fh, &stx, 0, 0, ceph_mount_perms(cmount));
+  ASSERT_EQ(rc, 0); 
+
+  WAIT_MAIN(1); // (R1)
+
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  PING_MAIN(1); // (1)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, true));
+  PING_MAIN(2); // (2)
+
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  PING_MAIN(3); // (3)
+
+  lock1.l_type = F_RDLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, true));
+  PING_MAIN(4); // (4)
+
+  WAIT_MAIN(2); // (R2)
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  PING_MAIN(5); // (5)
+
+  WAIT_MAIN(3); // (R3)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, true));
+  PING_MAIN(6); // (6)
+
+  WAIT_MAIN(4); // (R4)
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  PING_MAIN(7); // (7)
+
+  ASSERT_EQ(0, ceph_ll_close(cmount, fh));
+  CLEANUP_CEPH();
+
+  s.sem_destroy();
+  exit(EXIT_SUCCESS);
+}
+
+// Disabled because of fork() issues (http://tracker.ceph.com/issues/16556)
+#ifndef _WIN32
+TEST(LibCephFS, DISABLED_InterProcessRecordLocking) {
+  PROCESS_SLOW_MS();
+  // Process synchronization
+  char c_file[1024];
+  const pid_t mypid = getpid();
+  sprintf(c_file, "recordlock_test_%d", mypid);
+  Fh *fh = NULL;
+  Inode *root = NULL, *inode = NULL;
+  struct ceph_statx stx;
+  struct flock lock1;
+  int rc;
+
+  // Note: the semaphores MUST be on a shared memory segment
+  str_ConcurrentRecordLocking *const shs =
+    reinterpret_cast<str_ConcurrentRecordLocking*>
+    (mmap(0, sizeof(*shs), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
+	  -1, 0));
+  str_ConcurrentRecordLocking &s = *shs;
+  s.file = c_file;
+  s.sem_init(1);
+
+  // Start locker process
+  const pid_t pid = fork();
+  ASSERT_GE(pid, 0);
+  if (pid == 0) {
+    process_ConcurrentRecordLocking(s);
+    exit(EXIT_FAILURE);
+  }
+
+  struct timespec ts;
+  struct ceph_mount_info *cmount;
+  STARTUP_CEPH();
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  // Get the root inode
+  rc = ceph_ll_lookup_root(cmount, &root);
+  ASSERT_EQ(rc, 0); 
+
+  // Get the inode and Fh corresponding to c_file
+  rc = ceph_ll_create(cmount, root, c_file, fileMode, O_RDWR | O_CREAT,
+		      &inode, &fh, &stx, 0, 0, perms);
+  ASSERT_EQ(rc, 0); 
+
+  // Lock
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, true));
+
+  // Synchronization point with process (failure: process is dead)
+  PING_WORKER(1); // (R1)
+  WAIT_WORKER(1); // (1)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(2); // (2)
+
+  // Unlock
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+
+  // Shall have lock
+  // Synchronization point with process (failure: process is dead)
+  WAIT_WORKER(2); // (2)
+
+  // Synchronization point with process (failure: process is dead)
+  WAIT_WORKER(3); // (3)
+
+  // Wait for process to share lock
+  WAIT_WORKER(4); // (4)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  lock1.l_type = F_RDLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+
+  // Wake up process to unlock shared lock
+  PING_WORKER(2); // (R2)
+  WAIT_WORKER(5); // (5)
+
+  // Now we can lock exclusively
+  // Upgrade to exclusive lock (as per POSIX)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, true));
+
+  // Wake up process to lock shared lock
+  PING_WORKER(3); // (R3)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(6); // (6)
+
+  // Release lock ; process will get it
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  WAIT_WORKER(6); // (6)
+
+  // We no longer have the lock
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  lock1.l_type = F_RDLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+
+  // Wake up process to unlock exclusive lock
+  PING_WORKER(4); // (R4)
+  WAIT_WORKER(7); // (7)
+
+  // We can lock it again
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+
+  // Wait pid
+  int status;
+  ASSERT_EQ(pid, waitpid(pid, &status, 0));
+  ASSERT_EQ(EXIT_SUCCESS, status);
+
+  // Cleanup
+  s.sem_destroy();
+  ASSERT_EQ(0, munmap(shs, sizeof(*shs)));
+  ASSERT_EQ(0, ceph_ll_close(cmount, fh));
+  ASSERT_EQ(0, ceph_ll_unlink(cmount, root, c_file, perms));
+  CLEANUP_CEPH();
+}
+#endif
+
+#ifndef _WIN32
+// Disabled because of fork() issues (http://tracker.ceph.com/issues/16556)
+TEST(LibCephFS, DISABLED_ThreesomeInterProcessRecordLocking) {
+  PROCESS_SLOW_MS();
+  // Process synchronization
+  char c_file[1024];
+  const pid_t mypid = getpid();
+  sprintf(c_file, "recordlock_test_%d", mypid);
+  Fh *fh = NULL;
+  Inode *root = NULL, *inode = NULL;
+  struct ceph_statx stx;
+  struct flock lock1;
+  int rc;
+
+  // Note: the semaphores MUST be on a shared memory segment
+  str_ConcurrentRecordLocking *const shs =
+    reinterpret_cast<str_ConcurrentRecordLocking*>
+    (mmap(0, sizeof(*shs), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
+	  -1, 0));
+  str_ConcurrentRecordLocking &s = *shs;
+  s.file = c_file;
+  s.sem_init(1);
+
+  // Start locker processes
+  pid_t pid[2];
+  pid[0] = fork();
+  ASSERT_GE(pid[0], 0);
+  if (pid[0] == 0) {
+    process_ConcurrentRecordLocking(s);
+    exit(EXIT_FAILURE);
+  }
+  pid[1] = fork();
+  ASSERT_GE(pid[1], 0);
+  if (pid[1] == 0) {
+    process_ConcurrentRecordLocking(s);
+    exit(EXIT_FAILURE);
+  }
+
+  struct timespec ts;
+  struct ceph_mount_info *cmount;
+  STARTUP_CEPH();
+
+  // Get the root inode
+  rc = ceph_ll_lookup_root(cmount, &root);
+  ASSERT_EQ(rc, 0); 
+
+  // Get the inode and Fh corresponding to c_file
+  UserPerm *perms = ceph_mount_perms(cmount);
+  rc = ceph_ll_create(cmount, root, c_file, fileMode, O_RDWR | O_CREAT,
+		      &inode, &fh, &stx, 0, 0, perms);
+  ASSERT_EQ(rc, 0); 
+
+  // Lock
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, true));
+
+  // Synchronization point with process (failure: process is dead)
+  TWICE(PING_WORKER(1)); // (R1)
+  TWICE(WAIT_WORKER(1)); // (1)
+
+  // Shall not have lock immediately
+  NOT_WAIT_WORKER(2); // (2)
+
+  // Unlock
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+
+  // Shall have lock
+  TWICE(// Synchronization point with process (failure: process is dead)
+	WAIT_WORKER(2); // (2)
+	
+	// Synchronization point with process (failure: process is dead)
+	WAIT_WORKER(3)); // (3)
+  
+  // Wait for process to share lock
+  TWICE(WAIT_WORKER(4)); // (4)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  lock1.l_type = F_RDLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+
+  // Wake up process to unlock shared lock
+  TWICE(PING_WORKER(2); // (R2)
+	WAIT_WORKER(5)); // (5)
+
+  // Now we can lock exclusively
+  // Upgrade to exclusive lock (as per POSIX)
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, true));
+
+  TWICE(  // Wake up process to lock shared lock
+	PING_WORKER(3); // (R3)
+	
+	// Shall not have lock immediately
+	NOT_WAIT_WORKER(6)); // (6)
+  
+  // Release lock ; process will get it
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  TWICE(WAIT_WORKER(6); // (6)
+	
+	// We no longer have the lock
+	lock1.l_type = F_WRLCK;
+	lock1.l_whence = SEEK_SET;
+	lock1.l_start = 0;
+	lock1.l_len = 1024;
+	lock1.l_pid = getpid();
+	ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+	lock1.l_type = F_RDLCK;
+	lock1.l_whence = SEEK_SET;
+	lock1.l_start = 0;
+	lock1.l_len = 1024;
+	lock1.l_pid = getpid();
+	ASSERT_EQ(-CEPHFS_EAGAIN, ceph_ll_setlk(cmount, fh, &lock1, ceph_pthread_self(), false));
+	
+	// Wake up process to unlock exclusive lock
+	PING_WORKER(4); // (R4)
+	WAIT_WORKER(7); // (7)
+	);
+  
+  // We can lock it again
+  lock1.l_type = F_WRLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+  lock1.l_type = F_UNLCK;
+  lock1.l_whence = SEEK_SET;
+  lock1.l_start = 0;
+  lock1.l_len = 1024;
+  lock1.l_pid = getpid();
+  ASSERT_EQ(0, ceph_ll_setlk(cmount, fh, &lock1, mypid, false));
+
+  // Wait pids
+  int status;
+  ASSERT_EQ(pid[0], waitpid(pid[0], &status, 0));
+  ASSERT_EQ(EXIT_SUCCESS, status);
+  ASSERT_EQ(pid[1], waitpid(pid[1], &status, 0));
+  ASSERT_EQ(EXIT_SUCCESS, status);
+
+  // Cleanup
+  s.sem_destroy();
+  ASSERT_EQ(0, munmap(shs, sizeof(*shs)));
+  ASSERT_EQ(0, ceph_ll_close(cmount, fh));
+  ASSERT_EQ(0, ceph_ll_unlink(cmount, root, c_file, perms));
+  CLEANUP_CEPH();
+}
+#endif
diff --git a/src/test/libcephfs/snapdiff.cc b/src/test/libcephfs/snapdiff.cc
new file mode 100644
index 000000000..2320bf58b
--- /dev/null
+++ b/src/test/libcephfs/snapdiff.cc
@@ -0,0 +1,1684 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/stat.h"
+#include "include/ceph_assert.h"
+#include "include/object.h"
+#include "include/stringify.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <string>
+#include <vector>
+#include <algorithm>
+#include <limits.h>
+#include <dirent.h>
+
+using namespace std;
+class TestMount {
+  ceph_mount_info* cmount = nullptr;
+  char dir_path[64];
+
+public:
+  TestMount( const char* root_dir_name = "dir0") {
+    ceph_create(&cmount, NULL);
+    ceph_conf_read_file(cmount, NULL);
+    ceph_conf_parse_env(cmount, NULL);
+    ceph_assert(0 == ceph_mount(cmount, NULL));
+
+    sprintf(dir_path, "/%s_%d", root_dir_name, getpid());
+    ceph_assert(0 == ceph_mkdir(cmount, dir_path, 0777));
+  }
+  ~TestMount()
+  {
+    if (cmount) {
+      ceph_assert(0 == purge_dir(""));
+    }
+    ceph_rmdir(cmount, dir_path);
+    ceph_shutdown(cmount);
+  }
+
+  int conf_get(const char *option, char *buf, size_t len) {
+    return ceph_conf_get(cmount, option, buf, len);
+  }
+
+  string make_file_path(const char* relpath) {
+    char path[PATH_MAX];
+    sprintf(path, "%s/%s", dir_path, relpath);
+    return path;
+  }
+
+  string make_snap_name(const char* name) {
+    char snap_name[64];
+    if (name && *name) {
+      sprintf(snap_name, "%s_%d", name, getpid());
+    } else {
+      // just simulate empty snapname
+      snap_name[0] = 0;
+    }
+    return snap_name;
+  }
+  string make_snap_path(const char* sname, const char* subdir = nullptr) {
+    char snap_path[PATH_MAX];
+    string snap_name = subdir ?
+      concat_path(make_snap_name(sname), subdir) :
+      make_snap_name(sname);
+    sprintf(snap_path, ".snap/%s", snap_name.c_str());
+    return snap_path;
+  }
+
+  int mksnap(const char* name) {
+    string snap_name = make_snap_name(name);
+    return ceph_mksnap(cmount, dir_path, snap_name.c_str(),
+      0755, nullptr, 0);
+  }
+  int rmsnap(const char* name) {
+    string snap_name = make_snap_name(name);
+    return ceph_rmsnap(cmount, dir_path, snap_name.c_str());
+  }
+  int get_snapid(const char* name, uint64_t* res)
+  {
+    ceph_assert(res);
+    snap_info snap_info;
+
+    char snap_path[PATH_MAX];
+    string snap_name = make_snap_name(name);
+    sprintf(snap_path, "%s/.snap/%s", dir_path, snap_name.c_str());
+    int r = ceph_get_snap_info(cmount, snap_path, &snap_info);
+    if (r >= 0) {
+      *res = snap_info.id;
+      r = 0;
+    }
+    return r;
+  }
+
+  int write_full(const char* relpath, const string& data)
+  {
+    auto file_path = make_file_path(relpath);
+    int fd = ceph_open(cmount, file_path.c_str(), O_WRONLY | O_CREAT, 0666);
+    if (fd < 0) {
+      return -EACCES;
+    }
+    int r = ceph_write(cmount, fd, data.c_str(), data.size(), 0);
+    if (r >= 0) {
+      ceph_truncate(cmount, file_path.c_str(), data.size());
+      ceph_fsync(cmount, fd, 0);
+    }
+    ceph_close(cmount, fd);
+    return r;
+  }
+  string concat_path(string_view path, string_view name) {
+    string s(path);
+    if (s.empty() || s.back() != '/') {
+      s += '/';
+    }
+    s += name;
+    return s;
+  }
+  int unlink(const char* relpath)
+  {
+    auto file_path = make_file_path(relpath);
+    return ceph_unlink(cmount, file_path.c_str());
+  }
+
+  int test_open(const char* relpath)
+  {
+    auto subdir_path = make_file_path(relpath);
+    struct ceph_dir_result* ls_dir;
+    int r = ceph_opendir(cmount, subdir_path.c_str(), &ls_dir);
+    if (r != 0) {
+      return r;
+    }
+    ceph_assert(0 == ceph_closedir(cmount, ls_dir));
+    return r;
+  }
+
+  int for_each_readdir(const char* relpath,
+    std::function<bool(const dirent*, const struct ceph_statx*)> fn)
+  {
+    auto subdir_path = make_file_path(relpath);
+    struct ceph_dir_result* ls_dir;
+    int r = ceph_opendir(cmount, subdir_path.c_str(), &ls_dir);
+    if (r != 0) {
+      return r;
+    }
+
+    while (1) {
+      struct dirent result;
+      struct ceph_statx stx;
+
+      r = ceph_readdirplus_r(
+        cmount, ls_dir, &result, &stx, CEPH_STATX_BASIC_STATS,
+        0,
+        NULL);
+      if (!r)
+        break;
+      if (r < 0) {
+        std::cerr << "ceph_readdirplus_r failed, error: "
+                  << r << std::endl;
+        return r;
+      }
+
+      if (strcmp(result.d_name, ".") == 0 ||
+          strcmp(result.d_name, "..") == 0) {
+        continue;
+      }
+      if (!fn(&result, &stx)) {
+        r = -EINTR;
+        break;
+      }
+    }
+    ceph_assert(0 == ceph_closedir(cmount, ls_dir));
+    return r;
+  }
+  int readdir_and_compare(const char* relpath,
+    const vector<string>& expected0)
+  {
+    vector<string> expected(expected0);
+    auto end = expected.end();
+    int r = for_each_readdir(relpath,
+      [&](const dirent* dire, const struct ceph_statx* stx) {
+
+        std::string name(dire->d_name);
+        auto it = std::find(expected.begin(), end, name);
+        if (it == end) {
+          std::cerr << "readdir_and_compare error: unexpected name:"
+                    << name << std::endl;
+          return false;
+        }
+        expected.erase(it);
+        return true;
+      });
+    if (r == 0 && !expected.empty()) {
+      std::cerr << __func__ << " error: left entries:" << std::endl;
+      for (auto& e : expected) {
+        std::cerr << e << std::endl;
+      }
+      std::cerr << __func__ << " ************" << std::endl;
+      r = -ENOTEMPTY;
+    }
+    return r;
+  }
+  int for_each_readdir_snapdiff(const char* relpath,
+    const char* snap1,
+    const char* snap2,
+    std::function<bool(const dirent*, uint64_t)> fn)
+  {
+    auto s1 = make_snap_name(snap1);
+    auto s2 = make_snap_name(snap2);
+    ceph_snapdiff_info info;
+    ceph_snapdiff_entry_t res_entry;
+    int r = ceph_open_snapdiff(cmount,
+                               dir_path,
+                               relpath,
+                               s1.c_str(),
+                               s2.c_str(),
+                               &info);
+    if (r != 0) {
+      std::cerr << " Failed to open snapdiff, ret:" << r << std::endl;
+      return r;
+    }
+    while (0 < (r = ceph_readdir_snapdiff(&info,
+                                          &res_entry))) {
+      if (strcmp(res_entry.dir_entry.d_name, ".") == 0 ||
+        strcmp(res_entry.dir_entry.d_name, "..") == 0) {
+        continue;
+      }
+      if (!fn(&res_entry.dir_entry, res_entry.snapid)) {
+        r = -EINTR;
+        break;
+      }
+    }
+    ceph_assert(0 == ceph_close_snapdiff(&info));
+    if (r != 0) {
+      std::cerr << " Failed to readdir snapdiff, ret:" << r
+                << " " << relpath << ", " << snap1 << " vs. " << snap2
+                << std::endl;
+    }
+    return r;
+  }
+  int readdir_snapdiff_and_compare(const char* relpath,
+    const char* snap1,
+    const char* snap2,
+    const vector<pair<string, uint64_t>>& expected0)
+  {
+    vector<pair<string, uint64_t>> expected(expected0);
+    auto end = expected.end();
+    int r = for_each_readdir_snapdiff(relpath, snap1, snap2,
+      [&](const dirent* dire, uint64_t snapid) {
+
+        pair<string, uint64_t> p = std::make_pair(dire->d_name, snapid);
+        auto it = std::find(expected.begin(), end, p);
+        if (it == end) {
+          std::cerr << "readdir_snapdiff_and_compare error: unexpected name:"
+            << dire->d_name << "/" << snapid << std::endl;
+          return false;
+        }
+        expected.erase(it);
+        return true;
+      });
+    if (r == 0 && !expected.empty()) {
+      std::cerr << __func__ << " error: left entries:" << std::endl;
+      for (auto& e : expected) {
+        std::cerr << e.first << "/" << e.second << std::endl;
+      }
+      std::cerr << __func__ << " ************" << std::endl;
+      r = -ENOTEMPTY;
+    }
+    return r;
+  }
+
+  int mkdir(const char* relpath)
+  {
+    auto path = make_file_path(relpath);
+    return ceph_mkdir(cmount, path.c_str(), 0777);
+  }
+  int rmdir(const char* relpath)
+  {
+    auto path = make_file_path(relpath);
+    return ceph_rmdir(cmount, path.c_str());
+  }
+  int purge_dir(const char* relpath0, bool inclusive = true)
+  {
+    int r =
+      for_each_readdir(relpath0,
+        [&](const dirent* dire, const struct ceph_statx* stx) {
+          string relpath = concat_path(relpath0, dire->d_name);
+	  if (S_ISDIR(stx->stx_mode)) {
+            purge_dir(relpath.c_str());
+            rmdir(relpath.c_str());
+          } else {
+            unlink(relpath.c_str());
+          }
+          return true;
+        });
+    if (r != 0) {
+      return r;
+    }
+    if (*relpath0 != 0) {
+      r = rmdir(relpath0);
+    }
+    return r;
+  }
+
+  void remove_all() {
+    purge_dir("/", false);
+  }
+
+  ceph_mount_info* get_cmount() {
+    return cmount;
+  }
+
+  void verify_snap_diff(vector<pair<string, uint64_t>>& expected,
+                        const char* relpath,
+                        const char* snap1,
+                        const char* snap2);
+  void print_snap_diff(const char* relpath,
+		       const char* snap1,
+                       const char* snap2);
+
+  void prepareSnapDiffLib1Cases();
+  void prepareSnapDiffLib2Cases();
+  void prepareSnapDiffLib3Cases();
+  void prepareHugeSnapDiff(const std::string& name_prefix_start,
+                           const std::string& name_prefix_bulk,
+                           const std::string& name_prefix_end,
+                           size_t file_count,
+                           bool bulk_diff);
+};
+
+// Helper function to verify readdir_snapdiff returns expected results
+void TestMount::verify_snap_diff(vector<pair<string, uint64_t>>& expected,
+                                 const char* relpath,
+                                 const char* snap1,
+                                 const char* snap2)
+{
+  std::cout << "---------" << snap1 << " vs. " << snap2
+            << " diff listing verification for /" << (relpath ? relpath : "")
+            << std::endl;
+  ASSERT_EQ(0,
+    readdir_snapdiff_and_compare(relpath, snap1, snap2, expected));
+};
+
+// Helper function to print readdir_snapdiff results
+void TestMount::print_snap_diff(const char* relpath,
+				const char* snap1,
+                                const char* snap2)
+{
+  std::cout << "---------" << snap1 << " vs. " << snap2
+            << " diff listing for /" << (relpath ? relpath : "")
+            << std::endl;
+  ASSERT_EQ(0, for_each_readdir_snapdiff(relpath, snap1, snap2,
+    [&](const dirent* dire, uint64_t snapid) {
+      std::cout << dire->d_name << " snap " << snapid << std::endl;
+      return true;
+    }));
+};
+
+
+/* The following method creates some files/folders/snapshots layout,
+   described in the sheet below.
+   We're to test SnapDiff readdir API against that structure.
+
+* where:
+  - xN denotes file 'x' version N.
+  - X denotes folder name
+  - * denotes no/removed file/folder
+
+#    snap1         snap2
+# fileA1      | fileA2      |
+# *           | fileB2      |
+# fileC1      | *           |
+# fileD1      | fileD1      |
+# dirA        | dirA        |
+# dirA/fileA1 | dirA/fileA2 |
+# *           | dirB        |
+# *           | dirB/fileb2 |
+# dirC        | *           |
+# dirC/filec1 | *           |
+# dirD        | dirD        |
+# dirD/fileD1 | dirD/fileD1 |
+*/
+void TestMount::prepareSnapDiffLib1Cases()
+{
+  //************ snap1 *************
+  ASSERT_LE(0, write_full("fileA", "hello world"));
+  ASSERT_LE(0, write_full("fileC", "hello world to be removed"));
+  ASSERT_LE(0, write_full("fileD", "hello world unmodified"));
+  ASSERT_EQ(0, mkdir("dirA"));
+  ASSERT_LE(0, write_full("dirA/fileA", "file 'A/a' v1"));
+  ASSERT_EQ(0, mkdir("dirC"));
+  ASSERT_LE(0, write_full("dirC/filec", "file 'C/c' v1"));
+  ASSERT_EQ(0, mkdir("dirD"));
+  ASSERT_LE(0, write_full("dirD/filed", "file 'D/d' v1"));
+
+  ASSERT_EQ(0, mksnap("snap1"));
+
+  //************ snap2 *************
+  ASSERT_LE(0, write_full("fileA", "hello world again in A"));
+  ASSERT_LE(0, write_full("fileB", "hello world in B"));
+  ASSERT_EQ(0, unlink("fileC"));
+
+  ASSERT_LE(0, write_full("dirA/fileA", "file 'A/a' v2"));
+  ASSERT_EQ(0, purge_dir("dirC"));
+  ASSERT_EQ(0, mkdir("dirB"));
+  ASSERT_LE(0, write_full("dirB/fileb", "file 'B/b' v2"));
+
+  ASSERT_EQ(0, mksnap("snap2"));
+}
+
+/*
+* Basic functionality testing for the SnapDiff readdir API
+*/
+TEST(LibCephFS, SnapDiffLib)
+{
+  TestMount test_mount;
+
+  // Create simple directory tree with a couple of snapshots
+  // to test against
+  test_mount.prepareSnapDiffLib1Cases();
+
+  uint64_t snapid1;
+  uint64_t snapid2;
+
+  // learn snapshot ids and do basic verification
+  ASSERT_EQ(0, test_mount.get_snapid("snap1", &snapid1));
+  ASSERT_EQ(0, test_mount.get_snapid("snap2", &snapid2));
+  ASSERT_GT(snapid1, 0);
+  ASSERT_GT(snapid2, 0);
+  ASSERT_GT(snapid2, snapid1);
+  std::cout << snapid1 << " vs. " << snapid2 << std::endl;
+
+  //
+  // Make sure root listing for snapshot snap1 is as expected
+  //
+  {
+    std::cout << "---------snap1 listing verification---------" << std::endl;
+    string snap_path = test_mount.make_snap_path("snap1");
+    vector<string> expected;
+    expected.push_back("fileA");
+    expected.push_back("fileC");
+    expected.push_back("fileD");
+    expected.push_back("dirA");
+    expected.push_back("dirC");
+    expected.push_back("dirD");
+    ASSERT_EQ(0,
+      test_mount.readdir_and_compare(snap_path.c_str(), expected));
+  }
+
+  //
+  // Make sure root listing for snapshot snap2 is as expected
+  //
+  {
+    std::cout << "---------snap2 listing verification---------" << std::endl;
+    string snap_path = test_mount.make_snap_path("snap2");
+    vector<string> expected;
+    expected.push_back("fileA");
+    expected.push_back("fileB");
+    expected.push_back("fileD");
+    expected.push_back("dirA");
+    expected.push_back("dirB");
+    expected.push_back("dirD");
+    ASSERT_EQ(0,
+      test_mount.readdir_and_compare(snap_path.c_str(), expected));
+  }
+
+  //
+  // Print snap1 vs. snap2 delta for the root
+  //
+  test_mount.print_snap_diff("", "snap1", "snap2");
+
+  //
+  // Make sure snap1 vs. snap2 delta for the root is as expected
+  //
+  {
+    vector<pair<string, uint64_t>> expected;
+    expected.emplace_back("fileA", snapid2);
+    expected.emplace_back("fileB", snapid2);
+    expected.emplace_back("fileC", snapid1);
+    expected.emplace_back("dirA", snapid2);
+    expected.emplace_back("dirB", snapid2);
+    expected.emplace_back("dirC", snapid1);
+    expected.emplace_back("dirD", snapid2);
+    test_mount.verify_snap_diff(expected, "", "snap1", "snap2");
+  }
+
+  //
+  // Make sure snap1 vs. snap2 delta for /dirA is as expected
+  //
+  {
+    vector<pair<string, uint64_t>> expected;
+    expected.emplace_back("fileA", snapid2);
+    test_mount.verify_snap_diff(expected, "dirA", "snap1", "snap2");
+  }
+
+  //
+  // Make sure snap1 vs. snap2 delta for /dirB is as expected
+  //
+  {
+    vector<pair<string, uint64_t>> expected;
+    expected.emplace_back("fileb", snapid2);
+    test_mount.verify_snap_diff(expected, "dirB", "snap1", "snap2");
+  }
+
+  //
+  // Make sure snap1 vs. snap2 delta for /dirC is as expected
+  //
+  {
+    vector<pair<string, uint64_t>> expected;
+    expected.emplace_back("filec", snapid1);
+    test_mount.verify_snap_diff(expected, "dirC", "snap2", "snap1");
+  }
+
+  //
+  // Make sure snap1 vs. snap2 delta for /dirD is as expected
+  //
+  {
+    vector<pair<string, uint64_t>> expected;
+    test_mount.verify_snap_diff(expected, "dirD", "snap1", "snap2");
+  }
+
+  // Make sure SnapDiff returns an error when provided with the same
+  // snapshot name for both parties A and B.
+  {
+    string snap_path = test_mount.make_snap_path("snap2");
+    string snap_other_path = snap_path;
+    std::cout << "---------invalid snapdiff params, the same snaps---------" << std::endl;
+    ASSERT_EQ(-EINVAL, test_mount.for_each_readdir_snapdiff(
+      "",
+      "snap2",
+      "snap2",
+      [&](const dirent* dire, uint64_t snapid) {
+        return true;
+      }));
+  }
+  // Make sure SnapDiff returns an error when provided with an empty
+  // snapshot name for one of the parties
+  {
+    std::cout << "---------invalid snapdiff params, no snap_other ---------" << std::endl;
+    string snap_path = test_mount.make_snap_path("snap2");
+    string snap_other_path;
+    ASSERT_EQ(-EINVAL, test_mount.for_each_readdir_snapdiff(
+      "",
+      "snap2",
+      "",
+      [&](const dirent* dire, uint64_t snapid) {
+        return true;
+      }));
+  }
+
+  std::cout << "------------- closing -------------" << std::endl;
+  ASSERT_EQ(0, test_mount.purge_dir(""));
+  ASSERT_EQ(0, test_mount.rmsnap("snap1"));
+  ASSERT_EQ(0, test_mount.rmsnap("snap2"));
+}
+
+/* The following method creates some files/folders/snapshots layout,
+   described in the sheet below.
+   We're to test SnapDiff readdir API against that structure.
+
+* where:
+  - xN denotes file 'x' version N.
+  - X denotes folder name
+  - * denotes no/removed file/folder
+
+#     snap1        snap2      snap3      head
+# fileA1      | fileA2      | fileA2
+# *           | fileB2      | fileB2
+# fileC1      | *           | fileC3
+# fileD1      | fileD1      | fileD3
+# *           | *           | fileE3
+# fileF1      | *           | *
+# fileG1      | fileG2      | *
+# dirA        | dirA        | *
+# dirA/fileA1 | dirA/fileA2 | *
+# *           | dirB        | *
+# *           | dirB/fileb2 | *
+# dirC        | *           | *
+# dirC/filec1 | *           | *
+# dirD        | dirD        | dirD
+# dirD/filed1 | dirD/filed1 | dirD/filed1
+*/
+void TestMount::prepareSnapDiffLib2Cases()
+{
+  //************ snap1 *************
+  ASSERT_LE(0, write_full("fileA", "hello world"));
+  ASSERT_LE(0, write_full("fileC", "hello world to be removed temporarily"));
+  ASSERT_LE(0, write_full("fileD", "hello world unmodified"));
+  ASSERT_LE(0, write_full("fileF", "hello world to be removed completely"));
+  ASSERT_LE(0, write_full("fileG", "hello world to be overwritten at snap2"));
+  ASSERT_EQ(0, mkdir("dirA"));
+  ASSERT_LE(0, write_full("dirA/fileA", "file 'A/a' v1"));
+  ASSERT_EQ(0, mkdir("dirC"));
+  ASSERT_LE(0, write_full("dirC/filec", "file 'C/c' v1"));
+  ASSERT_EQ(0, mkdir("dirD"));
+  ASSERT_LE(0, write_full("dirD/filed", "file 'D/d' v1"));
+
+  ASSERT_EQ(0, mksnap("snap1"));
+
+  //************ snap2 *************
+  ASSERT_LE(0, write_full("fileA", "hello world again in A"));
+  ASSERT_LE(0, write_full("fileB", "hello world in B"));
+  ASSERT_LE(0, write_full("fileG", "hello world to be removed at snap3"));
+  ASSERT_EQ(0, unlink("fileC"));
+  ASSERT_EQ(0, unlink("fileF"));
+
+  ASSERT_LE(0, write_full("dirA/fileA", "file 'A/a' v2"));
+  ASSERT_EQ(0, mkdir("dirB"));
+  ASSERT_LE(0, write_full("dirB/fileb", "file 'B/b' v2"));
+  ASSERT_EQ(0, purge_dir("dirC"));
+
+  ASSERT_EQ(0, mksnap("snap2"));
+
+  //************ snap3 *************
+  ASSERT_LE(0, write_full("fileC", "hello world in C recovered"));
+  ASSERT_LE(0, write_full("fileD", "hello world in D now modified"));
+  ASSERT_LE(0, write_full("fileE", "file 'E' created at snap3"));
+  ASSERT_EQ(0, unlink("fileG"));
+  ASSERT_EQ(0, purge_dir("dirA"));
+  ASSERT_EQ(0, purge_dir("dirB"));
+  ASSERT_EQ(0, mksnap("snap3"));
+}
+
+/* The following method creates a folder with tons of file
+   updated between two snapshots
+   We're to test SnapDiff readdir API against that structure.
+
+* where:
+  - xN denotes file 'x' version N.
+  - X denotes folder name
+  - * denotes no/removed file/folder
+
+#    snap1         snap2
+* aaaaA1     | aaaaA1    |
+* aaaaB1     |    *      |
+* *          | aaaaC2    |
+* aaaaD1     | aaaaD2    |
+# file<NNN>1 | file<NNN>2|
+* fileZ1     | fileA1    |
+* zzzzA1     | zzzzA1    |
+* zzzzB1     |    *      |
+* *          | zzzzC2    |
+* zzzzD1     | zzzzD2    |
+*/
+
+void TestMount::prepareHugeSnapDiff(const std::string& name_prefix_start,
+                                    const std::string& name_prefix_bulk,
+                                    const std::string& name_prefix_end,
+                                    size_t file_count,
+                                    bool bulk_diff)
+{
+  //************ snap1 *************
+  std::string startA = name_prefix_start + "A";
+  std::string startB = name_prefix_start + "B";
+  std::string startC = name_prefix_start + "C";
+  std::string startD = name_prefix_start + "D";
+  std::string endA = name_prefix_end + "A";
+  std::string endB = name_prefix_end + "B";
+  std::string endC = name_prefix_end + "C";
+  std::string endD = name_prefix_end + "D";
+
+  ASSERT_LE(0, write_full(startA.c_str(), "hello world"));
+  ASSERT_LE(0, write_full(startB.c_str(), "hello world"));
+  ASSERT_LE(0, write_full(startD.c_str(), "hello world"));
+  for(size_t i = 0; i < file_count; i++) {
+    auto s = name_prefix_bulk + stringify(i);
+    ASSERT_LE(0, write_full(s.c_str(), "hello world"));
+  }
+  ASSERT_LE(0, write_full(endA.c_str(), "hello world"));
+  ASSERT_LE(0, write_full(endB.c_str(), "hello world"));
+  ASSERT_LE(0, write_full(endD.c_str(), "hello world"));
+
+  ASSERT_EQ(0, mksnap("snap1"));
+
+  ASSERT_LE(0, unlink(startB.c_str()));
+  ASSERT_LE(0, write_full(startC.c_str(), "hello world2"));
+  ASSERT_LE(0, write_full(startD.c_str(), "hello world2"));
+  if (bulk_diff) {
+    for(size_t i = 0; i < file_count; i++) {
+      auto s = std::string(name_prefix_bulk) + stringify(i);
+      ASSERT_LE(0, write_full(s.c_str(), "hello world2"));
+    }
+  }
+  ASSERT_LE(0, unlink(endB.c_str()));
+  ASSERT_LE(0, write_full(endC.c_str(), "hello world2"));
+  ASSERT_LE(0, write_full(endD.c_str(), "hello world2"));
+  ASSERT_EQ(0, mksnap("snap2"));
+}
+
+/*
+* More versatile SnapDiff readdir API verification,
+* includes 3 different snapshots and interleaving/repetitive calls to make sure
+* the results aren't spoiled due to caching.
+*/
+TEST(LibCephFS, SnapDiffLib2)
+{
+  TestMount test_mount;
+
+  test_mount.prepareSnapDiffLib2Cases();
+
+  // Create simple directory tree with a couple of snapshots to test against
+  uint64_t snapid1;
+  uint64_t snapid2;
+  uint64_t snapid3;
+  ASSERT_EQ(0, test_mount.get_snapid("snap1", &snapid1));
+  ASSERT_EQ(0, test_mount.get_snapid("snap2", &snapid2));
+  ASSERT_EQ(0, test_mount.get_snapid("snap3", &snapid3));
+  std::cout << snapid1 << " vs. " << snapid2 << " vs. " << snapid3 << std::endl;
+  ASSERT_GT(snapid1, 0);
+  ASSERT_GT(snapid2, 0);
+  ASSERT_GT(snapid3, 0);
+  ASSERT_GT(snapid2, snapid1);
+  ASSERT_GT(snapid3, snapid2);
+
+  // define a labda which verifies snap1/snap2/snap3 listings
+  auto verify_snap_listing = [&]()
+  {
+    {
+      string snap_path = test_mount.make_snap_path("snap1");
+
+      std::cout << "---------snap1 listing verification---------" << std::endl;
+      vector<string> expected;
+      expected.push_back("fileA");
+      expected.push_back("fileC");
+      expected.push_back("fileD");
+      expected.push_back("fileF");
+      expected.push_back("fileG");
+      expected.push_back("dirA");
+      expected.push_back("dirC");
+      expected.push_back("dirD");
+      ASSERT_EQ(0,
+        test_mount.readdir_and_compare(snap_path.c_str(), expected));
+    }
+    {
+      std::cout << "---------snap2 listing verification---------" << std::endl;
+      string snap_path = test_mount.make_snap_path("snap2");
+      vector<string> expected;
+      expected.push_back("fileA");
+      expected.push_back("fileB");
+      expected.push_back("fileD");
+      expected.push_back("fileG");
+      expected.push_back("dirA");
+      expected.push_back("dirB");
+      expected.push_back("dirD");
+      ASSERT_EQ(0,
+        test_mount.readdir_and_compare(snap_path.c_str(), expected));
+    }
+    {
+      std::cout << "---------snap3 listing verification---------" << std::endl;
+      string snap_path = test_mount.make_snap_path("snap3");
+      vector<string> expected;
+      expected.push_back("fileA");
+      expected.push_back("fileB");
+      expected.push_back("fileC");
+      expected.push_back("fileD");
+      expected.push_back("fileE");
+      expected.push_back("dirD");
+      ASSERT_EQ(0,
+        test_mount.readdir_and_compare(snap_path.c_str(), expected));
+    }
+  };
+  // Prepare expected delta for snap1 vs. snap2
+  vector<pair<string, uint64_t>> snap1_2_diff_expected;
+  snap1_2_diff_expected.emplace_back("fileA", snapid2);
+  snap1_2_diff_expected.emplace_back("fileB", snapid2);
+  snap1_2_diff_expected.emplace_back("fileC", snapid1);
+  snap1_2_diff_expected.emplace_back("fileF", snapid1);
+  snap1_2_diff_expected.emplace_back("fileG", snapid2);
+  snap1_2_diff_expected.emplace_back("dirA", snapid2);
+  snap1_2_diff_expected.emplace_back("dirB", snapid2);
+  snap1_2_diff_expected.emplace_back("dirC", snapid1);
+  snap1_2_diff_expected.emplace_back("dirD", snapid2);
+
+  // Prepare expected delta for snap1 vs. snap3
+  vector<pair<string, uint64_t>> snap1_3_diff_expected;
+  snap1_3_diff_expected.emplace_back("fileA", snapid3);
+  snap1_3_diff_expected.emplace_back("fileB", snapid3);
+  snap1_3_diff_expected.emplace_back("fileC", snapid3);
+  snap1_3_diff_expected.emplace_back("fileD", snapid3);
+  snap1_3_diff_expected.emplace_back("fileE", snapid3);
+  snap1_3_diff_expected.emplace_back("fileF", snapid1);
+  snap1_3_diff_expected.emplace_back("fileG", snapid1);
+  snap1_3_diff_expected.emplace_back("dirA", snapid1);
+  snap1_3_diff_expected.emplace_back("dirC", snapid1);
+  snap1_3_diff_expected.emplace_back("dirD", snapid3);
+
+  // Prepare expected delta for snap2 vs. snap3
+  vector<pair<string, uint64_t>> snap2_3_diff_expected;
+  snap2_3_diff_expected.emplace_back("fileC", snapid3);
+  snap2_3_diff_expected.emplace_back("fileD", snapid3);
+  snap2_3_diff_expected.emplace_back("fileE", snapid3);
+  snap2_3_diff_expected.emplace_back("fileG", snapid2);
+  snap2_3_diff_expected.emplace_back("dirA", snapid2);
+  snap2_3_diff_expected.emplace_back("dirB", snapid2);
+  snap2_3_diff_expected.emplace_back("dirD", snapid3);
+
+  // Check snapshot listings on a cold cache
+  verify_snap_listing();
+
+  // Check snapshot listings on a warm cache
+  verify_snap_listing(); // served from cache
+
+  // Print snap1 vs. snap2 delta against the root folder
+  test_mount.print_snap_diff("", "snap1", "snap2");
+
+  // Verify snap1 vs. snap2 delta for the root
+  test_mount.verify_snap_diff(snap1_2_diff_expected, "", "snap1", "snap2");
+
+  // Check snapshot listings on a warm cache once again
+  // to make sure it wasn't spoiled by SnapDiff
+  verify_snap_listing(); // served from cache
+
+  // Verify snap2 vs. snap1 delta
+  test_mount.verify_snap_diff(snap1_2_diff_expected, "", "snap2", "snap1");
+
+  // Check snapshot listings on a warm cache once again
+  // to make sure it wasn't spoiled by SnapDiff
+  verify_snap_listing(); // served from cache
+
+  // Verify snap1 vs. snap3 delta for the root
+  test_mount.verify_snap_diff(snap1_3_diff_expected, "", "snap1", "snap3");
+
+  // Verify snap2 vs. snap3 delta for the root
+  test_mount.verify_snap_diff(snap2_3_diff_expected, "", "snap2", "snap3");
+
+  // Check snapshot listings on a warm cache once again
+  // to make sure it wasn't spoiled by SnapDiff
+  verify_snap_listing(); // served from cache
+
+  // Print snap1 vs. snap2 delta against /dirA folder
+  test_mount.print_snap_diff("dirA", "snap1", "snap2");
+
+  // Verify snap1 vs. snap2 delta for /dirA
+  {
+    vector<pair<string, uint64_t>> expected;
+    expected.emplace_back("fileA", snapid2);
+    test_mount.verify_snap_diff(expected, "dirA", "snap1", "snap2");
+  }
+
+  // Print snap1 vs. snap2 delta against /dirB folder
+  test_mount.print_snap_diff("dirB", "snap1", "snap2");
+
+  // Verify snap1 vs. snap2 delta for /dirB
+  {
+    vector<pair<string, uint64_t>> expected;
+    expected.emplace_back("fileb", snapid2);
+    test_mount.verify_snap_diff(expected, "dirB", "snap1", "snap2");
+  }
+
+  // Print snap1 vs. snap2 delta against /dirD folder
+  test_mount.print_snap_diff("dirD", "snap1", "snap2");
+
+  // Verify snap1 vs. snap2 delta for /dirD
+  {
+    vector<pair<string, uint64_t>> expected;
+    test_mount.verify_snap_diff(expected, "dirD", "snap1", "snap2");
+  }
+
+  // Check snapshot listings on a warm cache once again
+  // to make sure it wasn't spoiled by SnapDiff
+  verify_snap_listing(); // served from cache
+
+  // Verify snap1 vs. snap2 delta for the root once again
+  test_mount.verify_snap_diff(snap1_2_diff_expected, "", "snap1", "snap2");
+
+  // Verify snap2 vs. snap3 delta for the root once again
+  test_mount.verify_snap_diff(snap2_3_diff_expected, "", "snap3", "snap2");
+
+  // Verify snap1 vs. snap3 delta for the root once again
+  test_mount.verify_snap_diff(snap1_3_diff_expected, "", "snap1", "snap3");
+
+  std::cout << "------------- closing -------------" << std::endl;
+  ASSERT_EQ(0, test_mount.purge_dir(""));
+  ASSERT_EQ(0, test_mount.rmsnap("snap1"));
+  ASSERT_EQ(0, test_mount.rmsnap("snap2"));
+  ASSERT_EQ(0, test_mount.rmsnap("snap3"));
+}
+
+/* The following method creates some files/folders/snapshots layout,
+   described in the sheet below.
+   We're to test SnapDiff against that structure.
+
+* where:
+  - xN denotes file 'x' version N.
+  - X denotes folder name
+  - * denotes no/removed file/folder
+
+#     snap1        snap2      snap3      head
+# a1     |     a1     |    a3    |    a4
+# b1     |     b2     |    b3    |    b3
+# c1     |     *      |    *     |    *
+# *      |     d2     |    d3    |    d3
+# f1     |     f2     |    *     |    *
+# ff1    |     ff1    |    *     |    *
+# g1     |     *      |    g3    |    g3
+# *      |     *      |    *     |    h4
+# i1     |     i1     |    i1    |    i1
+# S      |     S      |    S     |    S
+# S/sa1  |     S/sa2  |    S/sa3 |    S/sa3
+# *      |     *      |    *     |    S/sh4
+# *      |     T      |    T     |    T
+# *      |     T/td2  |    T/td3 |    T/td3
+# C      |     *      |    *     |    *
+# C/cc1  |     *      |    *     |    *
+# C/C1   |     *      |    *     |    *
+# C/C1/c1|     *      |    *     |    *
+# G      |     *      |    G     |    G
+# G/gg1  |     *      |    G/gg3 |    G/gg3
+# *      |     k2     |    *     |    *
+# *      |     l2     |    l2    |    *
+# *      |     K      |    *     |    *
+# *      |     K/kk2  |    *     |    *
+# *      |     *      |    H     |    H
+# *      |     *      |    H/hh3 |    H/hh3
+# I      |     I      |    I     |    *
+# I/ii1  |     I/ii2  |    I/ii3 |    *
+# I/iii1 |     I/iii1 |    I/iii3|    *
+# *      |     *      |   I/iiii3|    *
+# *      |    I/J     |  I/J     |    *
+# *      |   I/J/i2   |  I/J/i3  |    *
+# *      |   I/J/j2   |  I/J/j2  |    *
+# *      |   I/J/k2   |    *     |    *
+# *      |     *      |  I/J/l3  |    *
+# L      |     L      |    L     |    L
+# L/ll1  |    L/ll1   |   L/ll3  |    L/ll3
+# L/LL   |    L/LL    |  L/LL    |    L/LL
+# *      |    L/LL/ll2|  L/LL/ll3|    L/LL/ll4
+# *      |    L/LM    |    *     |    *
+# *      |    L/LM/lm2|    *     |    *
+# *      |    L/LN    |    L/LN  |    *
+*/
+void TestMount::prepareSnapDiffLib3Cases()
+{
+  //************ snap1 *************
+  ASSERT_LE(0, write_full("a", "file 'a' v1"));
+  ASSERT_LE(0, write_full("b", "file 'b' v1"));
+  ASSERT_LE(0, write_full("c", "file 'c' v1"));
+  ASSERT_LE(0, write_full("e", "file 'e' v1"));
+  ASSERT_LE(0, write_full("~e", "file '~e' v1"));
+  ASSERT_LE(0, write_full("f", "file 'f' v1"));
+  ASSERT_LE(0, write_full("ff", "file 'ff' v1"));
+  ASSERT_LE(0, write_full("g", "file 'g' v1"));
+  ASSERT_LE(0, write_full("i", "file 'i' v1"));
+
+  ASSERT_EQ(0, mkdir("S"));
+  ASSERT_LE(0, write_full("S/sa", "file 'S/sa' v1"));
+
+  ASSERT_EQ(0, mkdir("C"));
+  ASSERT_LE(0, write_full("C/cc", "file 'C/cc' v1"));
+
+  ASSERT_EQ(0, mkdir("C/CC"));
+  ASSERT_LE(0, write_full("C/CC/c", "file 'C/CC/c' v1"));
+
+  ASSERT_EQ(0, mkdir("G"));
+  ASSERT_LE(0, write_full("G/gg", "file 'G/gg' v1"));
+
+  ASSERT_EQ(0, mkdir("I"));
+  ASSERT_LE(0, write_full("I/ii", "file 'I/ii' v1"));
+  ASSERT_LE(0, write_full("I/iii", "file 'I/iii' v1"));
+
+  ASSERT_EQ(0, mkdir("L"));
+  ASSERT_LE(0, write_full("L/ll", "file 'L/ll' v1"));
+  ASSERT_EQ(0, mkdir("L/LL"));
+
+  ASSERT_EQ(0, mksnap("snap1"));
+  //************ snap2 *************
+
+  ASSERT_LE(0, write_full("b", "file 'b' v2"));
+  ASSERT_EQ(0, unlink("c"));
+  ASSERT_LE(0, write_full("d", "file 'd' v2"));
+  ASSERT_LE(0, write_full("e", "file 'e' v2"));
+  ASSERT_LE(0, write_full("~e", "file '~e' v2"));
+  ASSERT_LE(0, write_full("f", "file 'f' v2"));
+  ASSERT_EQ(0, unlink("g"));
+
+  ASSERT_LE(0, write_full("S/sa", "file 'S/sa' v2"));
+
+  ASSERT_EQ(0, mkdir("T"));
+  ASSERT_LE(0, write_full("T/td", "file 'T/td' v2"));
+
+  ASSERT_EQ(0, purge_dir("C"));
+  ASSERT_EQ(0, purge_dir("G"));
+
+  ASSERT_LE(0, write_full("k", "file 'k' v2"));
+  ASSERT_LE(0, write_full("l", "file 'l' v2"));
+
+  ASSERT_EQ(0, mkdir("K"));
+  ASSERT_LE(0, write_full("K/kk", "file 'K/kk' v2"));
+
+  ASSERT_LE(0, write_full("I/ii", "file 'I/ii' v2"));
+
+  ASSERT_EQ(0, mkdir("I/J"));
+  ASSERT_LE(0, write_full("I/J/i", "file 'I/J/i' v2"));
+  ASSERT_LE(0, write_full("I/J/j", "file 'I/J/j' v2"));
+  ASSERT_LE(0, write_full("I/J/k", "file 'I/J/k' v2"));
+
+  ASSERT_LE(0, write_full("L/LL/ll", "file 'L/LL/ll' v2"));
+
+  ASSERT_EQ(0, mkdir("L/LM"));
+  ASSERT_LE(0, write_full("L/LM/lm", "file 'L/LM/lm' v2"));
+
+  ASSERT_EQ(0, mkdir("L/LN"));
+
+  ASSERT_EQ(0, mksnap("snap2"));
+    //************ snap3 *************
+
+  ASSERT_LE(0, write_full("a", "file 'a' v3"));
+  ASSERT_LE(0, write_full("b", "file 'b' v3"));
+  ASSERT_LE(0, write_full("d", "file 'd' v3"));
+  ASSERT_EQ(0, unlink("e"));
+  ASSERT_EQ(0, unlink("~e"));
+  ASSERT_EQ(0, unlink("f"));
+  ASSERT_EQ(0, unlink("ff"));
+  ASSERT_LE(0, write_full("g", "file 'g' v3"));
+
+  ASSERT_LE(0, write_full("S/sa", "file 'S/sa' v3"));
+
+  ASSERT_LE(0, write_full("T/td", "file 'T/td' v3"));
+
+  ASSERT_EQ(0, mkdir("G"));
+  ASSERT_LE(0, write_full("G/gg", "file 'G/gg' v3"));
+
+  ASSERT_EQ(0, unlink("k"));
+
+  ASSERT_EQ(0, purge_dir("K"));
+
+  ASSERT_EQ(0, mkdir("H"));
+  ASSERT_LE(0, write_full("H/hh", "file 'H/hh' v3"));
+
+  ASSERT_LE(0, write_full("I/ii", "file 'I/ii' v3"));
+  ASSERT_LE(0, write_full("I/iii", "file 'I/iii' v3"));
+  ASSERT_LE(0, write_full("I/iiii", "file 'I/iiii' v3"));
+
+  ASSERT_LE(0, write_full("I/J/i", "file 'I/J/i' v3"));
+  ASSERT_EQ(0, unlink("I/J/k"));
+  ASSERT_LE(0, write_full("I/J/l", "file 'I/J/l' v3"));
+
+  ASSERT_LE(0, write_full("L/ll", "file 'L/ll' v3"));
+
+  ASSERT_LE(0, write_full("L/LL/ll", "file 'L/LL/ll' v3"));
+
+  ASSERT_EQ(0, purge_dir("L/LM"));
+
+  ASSERT_EQ(0, mksnap("snap3"));
+  //************ head *************
+  ASSERT_LE(0, write_full("a", "file 'a' head"));
+
+  ASSERT_LE(0, write_full("h", "file 'h' head"));
+
+  ASSERT_LE(0, write_full("S/sh", "file 'S/sh' head"));
+
+  ASSERT_EQ(0, unlink("l"));
+
+  ASSERT_EQ(0, purge_dir("I"));
+
+  ASSERT_LE(0, write_full("L/LL/ll", "file 'L/LL/ll' head"));
+
+  ASSERT_EQ(0, purge_dir("L/LN"));
+}
+
+//
+// This case tests SnapDiff functionality for snap1/snap2 snapshot delta
+// It operates against FS layout created by prepareSnapDiffCases() method,
+// see relevant table before that function for FS state overview.
+//
+TEST(LibCephFS, SnapDiffCases1_2)
+{
+  TestMount test_mount;
+
+  // Create directory tree evolving through a bunch of snapshots
+  test_mount.prepareSnapDiffLib3Cases();
+
+  uint64_t snapid1;
+  uint64_t snapid2;
+  ASSERT_EQ(0, test_mount.get_snapid("snap1", &snapid1));
+  ASSERT_EQ(0, test_mount.get_snapid("snap2", &snapid2));
+  std::cout << snapid1 << " vs. " << snapid2 << std::endl;
+  ASSERT_GT(snapid1, 0);
+  ASSERT_GT(snapid2, 0);
+  ASSERT_GT(snapid2, snapid1);
+
+  // Print snapshot delta (snap1 vs. snap2) results for root in a
+  // human-readable form.
+  test_mount.print_snap_diff("", "snap1", "snap2");
+
+  {
+    // Make sure the root delta is as expected
+    // One should use columns snap1 and snap2 from
+    // the table preceeding prepareSnapDiffCases() function
+    // to learn which names to expect in the delta.
+    //
+    //  - file 'a' is unchanged hence not present in delta
+    //  - file 'ff' is unchanged hence not present in delta
+    //  - file 'i' is unchanged hence not present in delta
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("b", snapid2);  // file 'b' is updated in snap2
+    expected.emplace_back("c", snapid1);  // file 'c' is removed in snap2
+    expected.emplace_back("d", snapid2);  // file 'd' is created in snap2
+    expected.emplace_back("e", snapid2);  // file 'e' is updated in snap2
+    expected.emplace_back("~e", snapid2); // file '~e' is updated in snap2
+    expected.emplace_back("f", snapid2);  // file 'f' is updated in snap2
+    expected.emplace_back("g", snapid1);  // file 'g' is removed in snap2
+    expected.emplace_back("S", snapid2);  // folder 'S' is present in snap2 hence reported
+    expected.emplace_back("T", snapid2);  // folder 'T' is created in snap2
+    expected.emplace_back("C", snapid1);  // folder 'C' is removed in snap2
+    expected.emplace_back("G", snapid1);  // folder 'G' is removed in snap2
+    expected.emplace_back("k", snapid2);  // file 'k' is created in snap2
+    expected.emplace_back("l", snapid2);  // file 'l' is created in snap2
+    expected.emplace_back("K", snapid2);  // folder 'K' is created in snap2
+    expected.emplace_back("I", snapid2);  // folder 'I' is created in snap2
+    expected.emplace_back("L", snapid2);  // folder 'L' is present in snap2 but got more
+                                       // subfolders
+    test_mount.verify_snap_diff(expected, "", "snap1", "snap2");
+  }
+  {
+
+    //
+    // Make sure snapshot delta for /S (existed at both snap1 and snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("sa", snapid2);
+    test_mount.verify_snap_diff(expected, "S", "snap1", "snap2");
+  }
+  {
+    //
+    // Make sure snapshot delta for /T (created at snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("td", snapid2);
+    test_mount.verify_snap_diff(expected, "T", "snap1", "snap2");
+  }
+  {
+    //
+    // Make sure snapshot delta for /C (removed at snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("cc", snapid1);
+    expected.emplace_back("CC", snapid1);
+    test_mount.verify_snap_diff(expected, "C", "snap2", "snap1");
+  }
+  {
+    //
+    // Make sure snapshot delta for /C/CC (removed at snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("c", snapid1);
+    test_mount.verify_snap_diff(expected, "C/CC", "snap2", "snap1");
+  }
+  {
+    //
+    // Make sure snapshot delta for /I (created at snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("ii", snapid2);
+    expected.emplace_back("J", snapid2);
+    test_mount.verify_snap_diff(expected, "I", "snap1", "snap2");
+  }
+  {
+    //
+    // Make sure snapshot delta for /I/J (created at snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("i", snapid2);
+    expected.emplace_back("j", snapid2);
+    expected.emplace_back("k", snapid2);
+    test_mount.verify_snap_diff(expected, "I/J", "snap1", "snap2");
+  }
+  {
+    //
+    // Make sure snapshot delta for /L (extended at snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("LL", snapid2);
+    expected.emplace_back("LM", snapid2);
+    expected.emplace_back("LN", snapid2);
+    test_mount.verify_snap_diff(expected, "L", "snap1", "snap2");
+  }
+  {
+    //
+    // Make sure snapshot delta for /L/LL (updated at snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("ll", snapid2);
+    test_mount.verify_snap_diff(expected, "L/LL", "snap1", "snap2");
+  }
+  {
+    //
+    // Make sure snapshot delta for /L/LN (created empty at snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    test_mount.verify_snap_diff(expected, "L/LN", "snap1", "snap2");
+  }
+
+  {
+    // Make sure snapshot delta for /L/LM (created at snap2)
+    // is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("lm", snapid2);
+    test_mount.verify_snap_diff(expected, "L/LM", "snap1", "snap2");
+  }
+  std::cout << "-------------" << std::endl;
+
+  test_mount.remove_all();
+  test_mount.rmsnap("snap1");
+  test_mount.rmsnap("snap2");
+  test_mount.rmsnap("snap3");
+}
+
+//
+// This case tests SnapDiff functionality for snap2/snap3 snapshot delta
+// retrieved through .snap path-based query API.
+// It operates against FS layout created by prepareSnapDiffCases() method,
+// see relevant table before that function for FS state overview.
+//
+TEST(LibCephFS, SnapDiffCases2_3)
+{
+  TestMount test_mount;
+
+  // Create directory tree evolving through a bunch of snapshots
+  test_mount.prepareSnapDiffLib3Cases();
+
+  uint64_t snapid2;
+  uint64_t snapid3;
+  ASSERT_EQ(0, test_mount.get_snapid("snap2", &snapid2));
+  ASSERT_EQ(0, test_mount.get_snapid("snap3", &snapid3));
+  std::cout << snapid2 << " vs. " << snapid3 << std::endl;
+  ASSERT_GT(snapid3, 0);
+  ASSERT_GT(snapid3, 0);
+  ASSERT_GT(snapid3, snapid2);
+
+  // Print snapshot delta (snap2 vs. snap3) results for root in a
+  // human-readable form.
+  test_mount.print_snap_diff("", "snap2", "snap3");
+
+  {
+    // Make sure the root delta is as expected
+    // One should use columns snap1 and snap2 from
+    // the table preceeding prepareSnapDiffCases() function
+    // to learn which names to expect in the delta.
+    //
+    //  - file 'c' is removed since snap1 hence not present in delta
+    //  - file 'l' is unchanged hence not present in delta
+    //  - file 'i' is unchanged hence not present in delta
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("a", snapid3);   // file 'a' is updated in snap3
+    expected.emplace_back("b", snapid3);   // file 'b' is updated in snap3
+    expected.emplace_back("d", snapid3);   // file 'd' is updated in snap3
+    expected.emplace_back("~e", snapid2);  // file '~e' is removed in snap3
+    expected.emplace_back("e", snapid2);   // file 'e' is removed in snap3
+    expected.emplace_back("f", snapid2);   // file 'f' is removed in snap3
+    expected.emplace_back("ff", snapid2);  // file 'ff' is removed in snap3
+    expected.emplace_back("g", snapid3);   // file 'g' re-appeared in snap3
+    expected.emplace_back("S", snapid3);   // folder 'S' is present in snap3 hence reported
+    expected.emplace_back("T", snapid3);   // folder 'T' is present in snap3 hence reported
+    expected.emplace_back("G", snapid3);   // folder 'G' re-appeared in snap3 hence reported
+    expected.emplace_back("k", snapid2);   // file 'k' is removed in snap3
+    expected.emplace_back("K", snapid2);   // folder 'K' is removed in snap3
+    expected.emplace_back("H", snapid3);   // folder 'H' is created in snap3 hence reported
+    expected.emplace_back("I", snapid3);   // folder 'I' is present in snap3 hence reported
+    expected.emplace_back("L", snapid3);   // folder 'L' is present in snap3 hence reported
+    test_mount.verify_snap_diff(expected, "", "snap2", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /S (children updated) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("sa", snapid3);
+    test_mount.verify_snap_diff(expected, "S", "snap2", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /T (children updated) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("td", snapid3);
+    test_mount.verify_snap_diff(expected, "T", "snap2", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /G (re-appeared) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("gg", snapid3);
+    test_mount.verify_snap_diff(expected, "G", "snap2", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /K (removed) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("kk", snapid2);
+    test_mount.verify_snap_diff(expected, "K", "snap3", "snap2");
+  }
+  {
+    //
+    // Make sure snapshot delta for /H (created) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("hh", snapid3);
+    test_mount.verify_snap_diff(expected, "H", "snap2", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /I (children updated) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("ii", snapid3);
+    expected.emplace_back("iii", snapid3);
+    expected.emplace_back("iiii", snapid3);
+    expected.emplace_back("J", snapid3);
+    test_mount.verify_snap_diff(expected, "I", "snap2", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /I/J (children updated/removed) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("i", snapid3);
+    expected.emplace_back("k", snapid2);
+        expected.emplace_back("l", snapid3);
+    test_mount.verify_snap_diff(expected, "I/J", "snap2", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /L (children updated/removed) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("ll", snapid3);
+    expected.emplace_back("LL", snapid3);
+    expected.emplace_back("LM", snapid2);
+    expected.emplace_back("LN", snapid3);
+    test_mount.verify_snap_diff(expected, "L", "snap2", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /L/LL (children updated) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("ll", snapid3);
+    test_mount.verify_snap_diff(expected, "L/LL", "snap2", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /L/LM (removed) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("lm", snapid2);
+    test_mount.verify_snap_diff(expected, "L/LM", "snap3", "snap2");
+  }
+  {
+    //
+    // Make sure snapshot delta for /L/LN (created empty) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    test_mount.verify_snap_diff(expected, "L/LN", "snap2", "snap3");
+  }
+  test_mount.remove_all();
+  test_mount.rmsnap("snap1");
+  test_mount.rmsnap("snap2");
+  test_mount.rmsnap("snap3");
+}
+
+//
+// This case tests SnapDiff functionality for snap1/snap3 snapshot delta
+// retrieved through .snap path-based query API.
+// It operates against FS layout created by prepareSnapDiffCases() method,
+// see relevant table before that function for FS state overview.
+//
+TEST(LibCephFS, SnapDiffCases1_3)
+{
+  TestMount test_mount;
+
+  // Create directory tree evolving through a bunch of snapshots
+  test_mount.prepareSnapDiffLib3Cases();
+
+  uint64_t snapid1;
+  uint64_t snapid3;
+  ASSERT_EQ(0, test_mount.get_snapid("snap1", &snapid1));
+  ASSERT_EQ(0, test_mount.get_snapid("snap3", &snapid3));
+  std::cout << snapid1 << " vs. " << snapid3 << std::endl;
+  ASSERT_GT(snapid3, 0);
+  ASSERT_GT(snapid3, 0);
+  ASSERT_GT(snapid3, snapid1);
+
+  // Print snapshot delta (snap2 vs. snap3) results for root in a
+  // human-readable form.
+  test_mount.print_snap_diff("", "snap1", "snap3");
+
+  {
+    // Make sure the root delta is as expected
+    // One should use columns snap1 and snap3 from
+    // the table preceeding prepareSnapDiffCases() function
+    // to learn which names to expect in the delta.
+    //
+    //  - file 'i' is unchanged hence not present in delta
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("a", snapid3);  // file 'a' is updated in snap3
+    expected.emplace_back("b", snapid3);  // file 'b' is updated in snap3
+    expected.emplace_back("c", snapid1); // file 'c' is removed in snap2
+    expected.emplace_back("d", snapid3);  // file 'd' is updated in snap3
+    expected.emplace_back("~e", snapid1); // file '~e' is removed in snap3
+    expected.emplace_back("e", snapid1);  // file 'e' is removed in snap3
+    expected.emplace_back("f", snapid1);  // file 'f' is removed in snap3
+    expected.emplace_back("ff", snapid1); // file 'ff' is removed in snap3
+    expected.emplace_back("g", snapid3);  // file 'g' removed in snap2 and
+                                          // re-appeared in snap3
+    expected.emplace_back("S", snapid3);  // folder 'S' is present in snap3 hence reported
+    expected.emplace_back("T", snapid3);  // folder 'T' is present in snap3 hence reported
+    expected.emplace_back("C", snapid1);  // folder 'C' is removed in snap2
+
+    // folder 'G' is removed in snap2 and re-appeared in snap3
+    // hence reporting it twice under different snapid
+    expected.emplace_back("G", snapid1);
+    expected.emplace_back("G", snapid3);
+
+    expected.emplace_back("l", snapid3);   // file 'l' is created in snap2
+    expected.emplace_back("H", snapid3);   // folder 'H' is created in snap3 hence reported
+    expected.emplace_back("I", snapid3);   // folder 'I' is created in snap3 hence reported
+    expected.emplace_back("L", snapid3);   // folder 'L' is created in snap3 hence reported
+    test_mount.verify_snap_diff(expected, "", "snap3", "snap1");
+  }
+  {
+    //
+    // Make sure snapshot delta for /S (children updated) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("sa", snapid3);
+    test_mount.verify_snap_diff(expected, "S", "snap3", "snap1");
+  }
+  {
+    //
+    // Make sure snapshot delta for /T (created and children updated) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("td", snapid3);
+    test_mount.verify_snap_diff(expected, "T", "snap3", "snap1");
+  }
+  {
+    //
+    // Make sure snapshot delta for /C (removed) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("cc", snapid1);
+    expected.emplace_back("CC", snapid1);
+    test_mount.verify_snap_diff(expected, "C", "snap3", "snap1");
+  }
+  {
+    //
+    // Make sure snapshot delta for /C/CC (removed) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("c", snapid1);
+    test_mount.verify_snap_diff(expected, "C/CC", "snap3", "snap1");
+  }
+  {
+    //
+    // Make sure snapshot delta for /G (removed) is as expected
+    // For this case (G@snap1 and G@snap3 are different entries)
+    // the order in which snapshot names are provided is crucial.
+    // Making  G@snap1 vs. snap3 delta returns everything from G@snap1
+    // but omits any entries from G/snap3 (since it's a different entry).
+    // And making  G@snap3 vs. snap1 delta returns everything from G@snap3
+    // but nothing from snap1,
+
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("gg", snapid1);
+    test_mount.verify_snap_diff(expected, "G", "snap1", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /G (re-created) is as expected
+    // The snapshot names order is important, see above.
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("gg", snapid3);
+    test_mount.verify_snap_diff(expected, "G", "snap3", "snap1");
+  }
+  {
+    //
+    // Make sure snapshot delta for /H (created) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("hh", snapid3);
+    test_mount.verify_snap_diff(expected, "H", "snap1", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /I (chinldren updated) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("ii", snapid3);
+    expected.emplace_back("iii", snapid3);
+    expected.emplace_back("iiii", snapid3);
+    expected.emplace_back("J", snapid3);
+    test_mount.verify_snap_diff(expected, "I", "snap1", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /I/J (created at snap2) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("i", snapid3);
+    expected.emplace_back("j", snapid3);
+    expected.emplace_back("l", snapid3);
+    test_mount.verify_snap_diff(expected, "I/J", "snap1", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /L is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("ll", snapid3);
+    expected.emplace_back("LL", snapid3);
+    expected.emplace_back("LN", snapid3);
+    test_mount.verify_snap_diff(expected, "L", "snap1", "snap3");
+  }
+  {
+    //
+    // Make sure snapshot delta for /L/LL (children updated) is as expected
+    //
+    vector<std::pair<string, uint64_t>> expected;
+    expected.emplace_back("ll", snapid3);
+    test_mount.verify_snap_diff(expected, "L/LL", "snap1", "snap3");
+  }
+  {
+    vector<std::pair<string, uint64_t>> expected;
+    test_mount.verify_snap_diff(expected, "L/LN", "snap1", "snap3");
+  }
+  std::cout << "-------------" << std::endl;
+
+  test_mount.remove_all();
+  test_mount.rmsnap("snap1");
+  test_mount.rmsnap("snap2");
+  test_mount.rmsnap("snap3");
+}
+
+/*
+* SnapDiff readdir API testing for huge dir
+* when delta is minor.
+*/
+TEST(LibCephFS, HugeSnapDiffSmallDelta)
+{
+  TestMount test_mount;
+
+  long int file_count = 10000;
+  printf("Seeding %ld files...\n", file_count);
+
+  // Create simple directory tree with a couple of snapshots
+  // to test against.
+  string name_prefix_start = "aaaa";
+  string name_prefix_bulk = "file";
+  string name_prefix_end = "zzzz";
+  test_mount.prepareHugeSnapDiff(name_prefix_start,
+                                 name_prefix_bulk,
+                                 name_prefix_end,
+                                 file_count,
+                                 false);
+
+  uint64_t snapid1;
+  uint64_t snapid2;
+
+  // learn snapshot ids and do basic verification
+  ASSERT_EQ(0, test_mount.get_snapid("snap1", &snapid1));
+  ASSERT_EQ(0, test_mount.get_snapid("snap2", &snapid2));
+  ASSERT_GT(snapid1, 0);
+  ASSERT_GT(snapid2, 0);
+  ASSERT_GT(snapid2, snapid1);
+  std::cout << snapid1 << " vs. " << snapid2 << std::endl;
+
+  //
+  // Make sure snap1 vs. snap2 delta for the root is as expected
+  //
+  {
+    vector<pair<string, uint64_t>> expected;
+    expected.emplace_back(name_prefix_start + "B", snapid1);
+    expected.emplace_back(name_prefix_start + "C", snapid2);
+    expected.emplace_back(name_prefix_start + "D", snapid2);
+
+    expected.emplace_back(name_prefix_end + "B", snapid1);
+    expected.emplace_back(name_prefix_end + "C", snapid2);
+    expected.emplace_back(name_prefix_end + "D", snapid2);
+    test_mount.verify_snap_diff(expected, "", "snap1", "snap2");
+  }
+
+  std::cout << "------------- closing -------------" << std::endl;
+  ASSERT_EQ(0, test_mount.purge_dir(""));
+  ASSERT_EQ(0, test_mount.rmsnap("snap1"));
+  ASSERT_EQ(0, test_mount.rmsnap("snap2"));
+}
+
+/*
+* SnapDiff readdir API testing for huge dir
+* when delta is large
+*/
+TEST(LibCephFS, HugeSnapDiffLargeDelta)
+{
+  TestMount test_mount;
+
+  // Calculate amount of files required to have multiple directory fragments
+  // using relevant config parameters.
+  // file_count = mds_bal_spli_size * mds_bal_fragment_fast_factor + 100
+  char buf[256];
+  int r = test_mount.conf_get("mds_bal_split_size", buf, sizeof(buf));
+  ASSERT_TRUE(r >= 0);
+  long int file_count = strtol(buf, nullptr, 10);
+  r = test_mount.conf_get("mds_bal_fragment_fast_factor ", buf, sizeof(buf));
+  ASSERT_TRUE(r >= 0);
+  double factor = strtod(buf, nullptr);
+  file_count *= factor;
+  file_count += 100;
+  printf("Seeding %ld files...\n", file_count);
+
+  // Create simple directory tree with a couple of snapshots
+  // to test against.
+
+  string name_prefix_start = "aaaa";
+  string name_prefix_bulk = "file";
+  string name_prefix_end = "zzzz";
+
+  test_mount.prepareHugeSnapDiff(name_prefix_start,
+                                 name_prefix_bulk,
+                                 name_prefix_end,
+                                 file_count,
+                                 true);
+  uint64_t snapid1;
+  uint64_t snapid2;
+
+  // learn snapshot ids and do basic verification
+  ASSERT_EQ(0, test_mount.get_snapid("snap1", &snapid1));
+  ASSERT_EQ(0, test_mount.get_snapid("snap2", &snapid2));
+  ASSERT_GT(snapid1, 0);
+  ASSERT_GT(snapid2, 0);
+  ASSERT_GT(snapid2, snapid1);
+  std::cout << snapid1 << " vs. " << snapid2 << std::endl;
+
+  //
+  // Make sure snap1 vs. snap2 delta for the root is as expected
+  //
+  {
+    vector<pair<string, uint64_t>> expected;
+    expected.emplace_back(name_prefix_start + "B", snapid1);
+    expected.emplace_back(name_prefix_start + "C", snapid2);
+    expected.emplace_back(name_prefix_start + "D", snapid2);
+    for (size_t i = 0; i < (size_t)file_count; i++) {
+      expected.emplace_back(name_prefix_bulk + stringify(i), snapid2);
+    }
+    expected.emplace_back(name_prefix_end + "B", snapid1);
+    expected.emplace_back(name_prefix_end + "C", snapid2);
+    expected.emplace_back(name_prefix_end + "D", snapid2);
+    test_mount.verify_snap_diff(expected, "", "snap1", "snap2");
+  }
+
+  std::cout << "------------- closing -------------" << std::endl;
+  ASSERT_EQ(0, test_mount.purge_dir(""));
+  ASSERT_EQ(0, test_mount.rmsnap("snap1"));
+  ASSERT_EQ(0, test_mount.rmsnap("snap2"));
+}
diff --git a/src/test/libcephfs/suidsgid.cc b/src/test/libcephfs/suidsgid.cc
new file mode 100644
index 000000000..d750613eb
--- /dev/null
+++ b/src/test/libcephfs/suidsgid.cc
@@ -0,0 +1,331 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2023 Red Hat, Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gtest/gtest.h"
+#include "common/ceph_argparse.h"
+#include "include/buffer.h"
+#include "include/fs_types.h"
+#include "include/stringify.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/rados/librados.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <sys/uio.h>
+#include <iostream>
+#include <vector>
+#include "json_spirit/json_spirit.h"
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#endif
+
+using namespace std;
+struct ceph_mount_info *admin;
+struct ceph_mount_info *cmount;
+char filename[128];
+
+void run_fallocate_test_case(int mode, int result, bool with_admin=false)
+{
+  struct ceph_statx stx;
+  int flags = FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE;
+
+  ASSERT_EQ(0, ceph_chmod(admin, filename, mode));
+
+  struct ceph_mount_info *_cmount = cmount;
+  if (with_admin) {
+    _cmount = admin;
+  }
+  int fd = ceph_open(_cmount, filename, O_RDWR, 0);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_fallocate(_cmount, fd, flags, 1024, 40960));
+  ASSERT_EQ(ceph_statx(_cmount, filename, &stx, CEPH_STATX_MODE, 0), 0);
+  std::cout << "After ceph_fallocate, mode: 0" << oct << mode << " -> 0"
+            << (stx.stx_mode & 07777) << dec << std::endl;
+  ASSERT_EQ(stx.stx_mode & (S_ISUID|S_ISGID), result);
+  ceph_close(_cmount, fd);
+}
+
+rados_t cluster;
+
+int do_mon_command(string s, string *key)
+{
+  char *outs, *outbuf;
+  size_t outs_len, outbuf_len;
+  const char *ss = s.c_str();
+  int r = rados_mon_command(cluster, (const char **)&ss, 1,
+			    0, 0,
+			    &outbuf, &outbuf_len,
+			    &outs, &outs_len);
+  if (outbuf_len) {
+    string s(outbuf, outbuf_len);
+    std::cout << "out: " << s << std::endl;
+
+    // parse out the key
+    json_spirit::mValue v, k;
+    json_spirit::read_or_throw(s, v);
+    k = v.get_array()[0].get_obj().find("key")->second;
+    *key = k.get_str();
+    std::cout << "key: " << *key << std::endl;
+    free(outbuf);
+  } else {
+    return -CEPHFS_EINVAL;
+  }
+  if (outs_len) {
+    string s(outs, outs_len);
+    std::cout << "outs: " << s << std::endl;
+    free(outs);
+  }
+  return r;
+}
+
+void run_write_test_case(int mode, int result, bool with_admin=false)
+{
+  struct ceph_statx stx;
+
+  ASSERT_EQ(0, ceph_chmod(admin, filename, mode));
+
+  struct ceph_mount_info *_cmount = cmount;
+  if (with_admin) {
+    _cmount = admin;
+  }
+  int fd = ceph_open(_cmount, filename, O_RDWR, 0);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(ceph_write(_cmount, fd, "foo", 3, 0), 3);
+  ASSERT_EQ(ceph_statx(_cmount, filename, &stx, CEPH_STATX_MODE, 0), 0);
+  std::cout << "After ceph_write, mode: 0" << oct << mode << " -> 0"
+            << (stx.stx_mode & 07777) << dec << std::endl;
+  ASSERT_EQ(stx.stx_mode & (S_ISUID|S_ISGID), result);
+  ceph_close(_cmount, fd);
+}
+
+void run_truncate_test_case(int mode, int result, size_t size, bool with_admin=false)
+{
+  struct ceph_statx stx;
+
+  ASSERT_EQ(0, ceph_chmod(admin, filename, mode));
+
+  struct ceph_mount_info *_cmount = cmount;
+  if (with_admin) {
+    _cmount = admin;
+  }
+  int fd = ceph_open(_cmount, filename, O_RDWR, 0);
+  ASSERT_LE(0, fd);
+  ASSERT_GE(ceph_ftruncate(_cmount, fd, size), 0);
+  ASSERT_EQ(ceph_statx(_cmount, filename, &stx, CEPH_STATX_MODE, 0), 0);
+  std::cout << "After ceph_truncate size " << size << " mode: 0" << oct
+            << mode << " -> 0" << (stx.stx_mode & 07777) << dec << std::endl;
+  ASSERT_EQ(stx.stx_mode & (S_ISUID|S_ISGID), result);
+  ceph_close(_cmount, fd);
+}
+
+TEST(SuidsgidTest, WriteClearSetuid) {
+  ASSERT_EQ(0, ceph_create(&admin, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(admin, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(admin, NULL));
+  ASSERT_EQ(0, ceph_mount(admin, "/"));
+
+  sprintf(filename, "/clear_suidsgid_file_%d", getpid());
+  int fd = ceph_open(admin, filename, O_CREAT|O_RDWR, 0766);
+  ASSERT_GE(ceph_ftruncate(admin, fd, 10000000), 0);
+  ceph_close(admin, fd);
+
+  string user = "clear_suidsgid_" + stringify(rand());
+  // create access key
+  string key;
+  ASSERT_EQ(0, do_mon_command(
+      "{\"prefix\": \"auth get-or-create\", \"entity\": \"client." + user + "\", "
+      "\"caps\": [\"mon\", \"allow *\", \"osd\", \"allow *\", \"mgr\", \"allow *\", "
+      "\"mds\", \"allow *\"], \"format\": \"json\"}", &key));
+
+  ASSERT_EQ(0, ceph_create(&cmount, user.c_str()));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_set(cmount, "key", key.c_str()));
+  ASSERT_EQ(ceph_init(cmount), 0);
+  UserPerm *perms = ceph_userperm_new(123, 456, 0, NULL);
+  ASSERT_NE(nullptr, perms);
+  ASSERT_EQ(0, ceph_mount_perms_set(cmount, perms));
+  ceph_userperm_destroy(perms);
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  // 1, Commit to a non-exec file by an unprivileged user clears suid and sgid.
+  run_fallocate_test_case(06666, 0); // a+rws
+
+  // 2, Commit to a group-exec file by an unprivileged user clears suid and sgid.
+  run_fallocate_test_case(06676, 0); // g+x,a+rws
+
+  // 3, Commit to a user-exec file by an unprivileged user clears suid and sgid.
+  run_fallocate_test_case(06766, 0); // u+x,a+rws,g-x
+
+  // 4, Commit to a all-exec file by an unprivileged user clears suid and sgid.
+  run_fallocate_test_case(06777, 0); // a+rwxs
+
+  // 5, Commit to a non-exec file by root leaves suid and sgid.
+  run_fallocate_test_case(06666, S_ISUID|S_ISGID, true); // a+rws
+
+  // 6, Commit to a group-exec file by root leaves suid and sgid.
+  run_fallocate_test_case(06676, S_ISUID|S_ISGID, true); // g+x,a+rws
+
+  // 7, Commit to a user-exec file by root leaves suid and sgid.
+  run_fallocate_test_case(06766, S_ISUID|S_ISGID, true); // u+x,a+rws,g-x
+
+  // 8, Commit to a all-exec file by root leaves suid and sgid.
+  run_fallocate_test_case(06777, S_ISUID|S_ISGID, true); // a+rwxs
+
+  // 9, Commit to a group-exec file by an unprivileged user clears sgid
+  run_fallocate_test_case(02676, 0); // a+rw,g+rwxs
+
+  // 10, Commit to a all-exec file by an unprivileged user clears sgid.
+  run_fallocate_test_case(02777, 0); // a+rwx,g+rwxs
+
+  // 11, Write by privileged user leaves the suid and sgid
+  run_write_test_case(06766, S_ISUID | S_ISGID, true);
+
+  // 12, Write by unprivileged user clears the suid and sgid
+  run_write_test_case(06766, 0);
+
+  // 13, Truncate by privileged user leaves the suid and sgid
+  run_truncate_test_case(06766, S_ISUID | S_ISGID, 10000, true);
+
+  // 14, Truncate by unprivileged user clears the suid and sgid
+  run_truncate_test_case(06766, 0, 100);
+
+  // clean up
+  ceph_shutdown(cmount);
+  ceph_shutdown(admin);
+}
+
+TEST(LibCephFS, ChownClearSetuid) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  Inode *root;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  char filename[32];
+  sprintf(filename, "clearsetuid%x", getpid());
+
+  Fh *fh;
+  Inode *in;
+  struct ceph_statx stx;
+  const mode_t after_mode = S_IRWXU;
+  const mode_t before_mode = S_IRWXU | S_ISUID | S_ISGID;
+  const unsigned want = CEPH_STATX_UID|CEPH_STATX_GID|CEPH_STATX_MODE;
+  UserPerm *usercred = ceph_mount_perms(cmount);
+
+  ceph_ll_unlink(cmount, root, filename, usercred);
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, before_mode,
+                          O_RDWR|O_CREAT|O_EXCL, &in, &fh, &stx, want, 0,
+                          usercred), 0);
+
+  ASSERT_EQ(stx.stx_mode & (mode_t)ALLPERMS, before_mode);
+
+  // chown  -- for this we need to be "root"
+  UserPerm *rootcred = ceph_userperm_new(0, 0, 0, NULL);
+  ASSERT_TRUE(rootcred);
+  stx.stx_uid++;
+  stx.stx_gid++;
+  ASSERT_EQ(ceph_ll_setattr(cmount, in, &stx, CEPH_SETATTR_UID|CEPH_SETATTR_GID, rootcred), 0);
+  ASSERT_EQ(ceph_ll_getattr(cmount, in, &stx, CEPH_STATX_MODE, 0, usercred), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_MODE);
+  ASSERT_EQ(stx.stx_mode & (mode_t)ALLPERMS, after_mode);
+
+  /* test chown with supplementary groups, and chown with/without exe bit */
+  uid_t u = 65534;
+  gid_t g = 65534;
+  gid_t gids[] = {65533,65532};
+  UserPerm *altcred = ceph_userperm_new(u, g, sizeof gids / sizeof gids[0], gids);
+  stx.stx_uid = u;
+  stx.stx_gid = g;
+  mode_t m = S_ISGID|S_ISUID|S_IRUSR|S_IWUSR;
+  stx.stx_mode = m;
+  ASSERT_EQ(ceph_ll_setattr(cmount, in, &stx, CEPH_SETATTR_MODE|CEPH_SETATTR_UID|CEPH_SETATTR_GID, rootcred), 0);
+  ASSERT_EQ(ceph_ll_getattr(cmount, in, &stx, CEPH_STATX_MODE, 0, altcred), 0);
+  ASSERT_EQ(stx.stx_mode&(mode_t)ALLPERMS, m);
+  /* not dropped without exe bit */
+  stx.stx_gid = gids[0];
+  ASSERT_EQ(ceph_ll_setattr(cmount, in, &stx, CEPH_SETATTR_GID, altcred), 0);
+  ASSERT_EQ(ceph_ll_getattr(cmount, in, &stx, CEPH_STATX_MODE, 0, altcred), 0);
+  ASSERT_EQ(stx.stx_mode&(mode_t)ALLPERMS, m);
+  /* now check dropped with exe bit */
+  m = S_ISGID|S_ISUID|S_IRWXU;
+  stx.stx_mode = m;
+  ASSERT_EQ(ceph_ll_setattr(cmount, in, &stx, CEPH_STATX_MODE, altcred), 0);
+  ASSERT_EQ(ceph_ll_getattr(cmount, in, &stx, CEPH_STATX_MODE, 0, altcred), 0);
+  ASSERT_EQ(stx.stx_mode&(mode_t)ALLPERMS, m);
+  stx.stx_gid = gids[1];
+  ASSERT_EQ(ceph_ll_setattr(cmount, in, &stx, CEPH_SETATTR_GID, altcred), 0);
+  ASSERT_EQ(ceph_ll_getattr(cmount, in, &stx, CEPH_STATX_MODE, 0, altcred), 0);
+  ASSERT_EQ(stx.stx_mode&(mode_t)ALLPERMS, m&(S_IRWXU|S_IRWXG|S_IRWXO));
+  ceph_userperm_destroy(altcred);
+
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ceph_shutdown(cmount);
+}
+
+static int update_root_mode()
+{
+  struct ceph_mount_info *admin;
+  int r = ceph_create(&admin, NULL);
+  if (r < 0)
+    return r;
+  ceph_conf_read_file(admin, NULL);
+  ceph_conf_parse_env(admin, NULL);
+  ceph_conf_set(admin, "client_permissions", "false");
+  r = ceph_mount(admin, "/");
+  if (r < 0)
+    goto out;
+  r = ceph_chmod(admin, "/", 0777);
+out:
+  ceph_shutdown(admin);
+  return r;
+}
+
+int main(int argc, char **argv)
+{
+  int r = update_root_mode();
+  if (r < 0)
+    exit(1);
+
+  ::testing::InitGoogleTest(&argc, argv);
+
+  srand(getpid());
+
+  r = rados_create(&cluster, NULL);
+  if (r < 0)
+    exit(1);
+
+  r = rados_conf_read_file(cluster, NULL);
+  if (r < 0)
+    exit(1);
+
+  rados_conf_parse_env(cluster, NULL);
+  r = rados_connect(cluster);
+  if (r < 0)
+    exit(1);
+
+  r = RUN_ALL_TESTS();
+
+  rados_shutdown(cluster);
+
+  return r;
+}
diff --git a/src/test/libcephfs/test.cc b/src/test/libcephfs/test.cc
new file mode 100644
index 000000000..c83ddccf9
--- /dev/null
+++ b/src/test/libcephfs/test.cc
@@ -0,0 +1,3775 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/compat.h"
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include "mds/mdstypes.h"
+#include "include/stat.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <sys/uio.h>
+#include <sys/time.h>
+
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+#include "common/Clock.h"
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#endif
+
+#include <fmt/format.h>
+#include <map>
+#include <vector>
+#include <thread>
+#include <regex>
+
+using namespace std;
+
+TEST(LibCephFS, OpenEmptyComponent) {
+
+  pid_t mypid = getpid();
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  char c_dir[1024];
+  sprintf(c_dir, "/open_test_%d", mypid);
+  struct ceph_dir_result *dirp;
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, c_dir, 0777));
+
+  ASSERT_EQ(0, ceph_opendir(cmount, c_dir, &dirp));
+
+  char c_path[1024];
+  sprintf(c_path, "/open_test_%d//created_file_%d", mypid, mypid);
+  int fd = ceph_open(cmount, c_path, O_RDONLY|O_CREAT, 0666);
+  ASSERT_LT(0, fd);
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_closedir(cmount, dirp));
+  ceph_shutdown(cmount);
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  fd = ceph_open(cmount, c_path, O_RDONLY, 0666);
+  ASSERT_LT(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  // cleanup
+  ASSERT_EQ(0, ceph_unlink(cmount, c_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, c_dir));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, OpenReadTruncate) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  auto path = fmt::format("test_open_rdt_{}", getpid());
+  int fd = ceph_open(cmount, path.c_str(), O_WRONLY|O_CREAT, 0666);
+  ASSERT_LE(0, fd);
+
+  auto data = std::string("hello world");
+  ASSERT_EQ(ceph_write(cmount, fd, data.c_str(), data.size(), 0), (int)data.size());
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  fd = ceph_open(cmount, path.c_str(), O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(ceph_ftruncate(cmount, fd, 0), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_ftruncate(cmount, fd, 1), -CEPHFS_EBADF);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, OpenReadWrite) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  char c_path[1024];
+  sprintf(c_path, "test_open_rdwr_%d", getpid());
+  int fd = ceph_open(cmount, c_path, O_WRONLY|O_CREAT, 0666);
+  ASSERT_LT(0, fd);
+
+  const char *out_buf = "hello world";
+  size_t size = strlen(out_buf);
+  char in_buf[100];
+  ASSERT_EQ(ceph_write(cmount, fd, out_buf, size, 0), (int)size);
+  ASSERT_EQ(ceph_read(cmount, fd, in_buf, sizeof(in_buf), 0), -CEPHFS_EBADF);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  fd = ceph_open(cmount, c_path, O_RDONLY, 0);
+  ASSERT_LT(0, fd);
+  ASSERT_EQ(ceph_write(cmount, fd, out_buf, size, 0), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_read(cmount, fd, in_buf, sizeof(in_buf), 0), (int)size);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  fd = ceph_open(cmount, c_path, O_RDWR, 0);
+  ASSERT_LT(0, fd);
+  ASSERT_EQ(ceph_write(cmount, fd, out_buf, size, 0), (int)size);
+  ASSERT_EQ(ceph_read(cmount, fd, in_buf, sizeof(in_buf), 0), (int)size);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, MountNonExist) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_NE(0, ceph_mount(cmount, "/non-exist"));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, MountDouble) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(-CEPHFS_EISCONN, ceph_mount(cmount, "/"));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, MountRemount) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+
+  CephContext *cct = ceph_get_mount_context(cmount);
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(cct, ceph_get_mount_context(cmount));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, UnmountUnmounted) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(-CEPHFS_ENOTCONN, ceph_unmount(cmount));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, ReleaseUnmounted) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_release(cmount));
+}
+
+TEST(LibCephFS, ReleaseMounted) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(-CEPHFS_EISCONN, ceph_release(cmount));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ASSERT_EQ(0, ceph_release(cmount));
+}
+
+TEST(LibCephFS, UnmountRelease) {
+
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ASSERT_EQ(0, ceph_release(cmount));
+}
+
+TEST(LibCephFS, Mount) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+  ceph_shutdown(cmount);
+
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, OpenLayout) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  /* valid layout */
+  char test_layout_file[256];
+  sprintf(test_layout_file, "test_layout_%d_b", getpid());
+  int fd = ceph_open_layout(cmount, test_layout_file, O_CREAT|O_WRONLY, 0666, (1<<20), 7, (1<<20), NULL);
+  ASSERT_GT(fd, 0);
+  char poolname[80];
+  ASSERT_LT(0, ceph_get_file_pool_name(cmount, fd, poolname, sizeof(poolname)));
+  ASSERT_LT(0, ceph_get_file_pool_name(cmount, fd, poolname, 0));
+
+  /* on already-written file (CEPHFS_ENOTEMPTY) */
+  ceph_write(cmount, fd, "hello world", 11, 0);
+  ceph_close(cmount, fd);
+
+  char xattrk[128];
+  char xattrv[128];
+  sprintf(xattrk, "ceph.file.layout.stripe_unit");
+  sprintf(xattrv, "65536");
+  ASSERT_EQ(-CEPHFS_ENOTEMPTY, ceph_setxattr(cmount, test_layout_file, xattrk, (void *)xattrv, 5, 0));
+
+  /* invalid layout */
+  sprintf(test_layout_file, "test_layout_%d_c", getpid());
+  fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 1, 19, NULL);
+  ASSERT_EQ(fd, -CEPHFS_EINVAL);
+
+  /* with data pool */
+  sprintf(test_layout_file, "test_layout_%d_d", getpid());
+  fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 7, (1<<20), poolname);
+  ASSERT_GT(fd, 0);
+  ceph_close(cmount, fd);
+
+  /* with metadata pool (invalid) */
+  sprintf(test_layout_file, "test_layout_%d_e", getpid());
+  fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 7, (1<<20), "metadata");
+  ASSERT_EQ(fd, -CEPHFS_EINVAL);
+
+  /* with metadata pool (does not exist) */
+  sprintf(test_layout_file, "test_layout_%d_f", getpid());
+  fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 7, (1<<20), "asdfjasdfjasdf");
+  ASSERT_EQ(fd, -CEPHFS_EINVAL);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DirLs) {
+
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  struct ceph_dir_result *ls_dir = NULL;
+  char foostr[256];
+  sprintf(foostr, "dir_ls%d", mypid);
+  ASSERT_EQ(ceph_opendir(cmount, foostr, &ls_dir), -CEPHFS_ENOENT);
+
+  ASSERT_EQ(ceph_mkdir(cmount, foostr, 0777), 0);
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_statx(cmount, foostr, &stx, 0, 0), 0);
+  ASSERT_NE(S_ISDIR(stx.stx_mode), 0);
+
+  char barstr[256];
+  sprintf(barstr, "dir_ls2%d", mypid);
+  ASSERT_EQ(ceph_statx(cmount, barstr, &stx, 0, AT_SYMLINK_NOFOLLOW), -CEPHFS_ENOENT);
+
+  // insert files into directory and test open
+  char bazstr[256];
+  int i = 0, r = rand() % 4096;
+  if (getenv("LIBCEPHFS_RAND")) {
+    r = atoi(getenv("LIBCEPHFS_RAND"));
+  }
+  printf("rand: %d\n", r);
+  for(; i < r; ++i) {
+
+    sprintf(bazstr, "dir_ls%d/dirf%d", mypid, i);
+    int fd  = ceph_open(cmount, bazstr, O_CREAT|O_RDONLY, 0666);
+    ASSERT_GT(fd, 0);
+    ASSERT_EQ(ceph_close(cmount, fd), 0);
+
+    // set file sizes for readdirplus
+    ceph_truncate(cmount, bazstr, i);
+  }
+
+  ASSERT_EQ(ceph_opendir(cmount, foostr, &ls_dir), 0);
+
+  // not guaranteed to get . and .. first, but its a safe assumption in this case
+  struct dirent *result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+
+  std::vector<std::string> entries;
+  std::map<std::string, int64_t> offset_map;
+  int64_t offset = ceph_telldir(cmount, ls_dir);
+  for(i = 0; i < r; ++i) {
+    result = ceph_readdir(cmount, ls_dir);
+    ASSERT_TRUE(result != NULL);
+    entries.push_back(result->d_name);
+    offset_map[result->d_name] = offset;
+    offset = ceph_telldir(cmount, ls_dir);
+  }
+
+  ASSERT_TRUE(ceph_readdir(cmount, ls_dir) == NULL);
+  offset = ceph_telldir(cmount, ls_dir);
+
+  ASSERT_EQ(offset_map.size(), entries.size());
+  for(i = 0; i < r; ++i) {
+    sprintf(bazstr, "dirf%d", i);
+    ASSERT_TRUE(offset_map.count(bazstr) == 1);
+  }
+
+  // test seekdir
+  ceph_seekdir(cmount, ls_dir, offset);
+  ASSERT_TRUE(ceph_readdir(cmount, ls_dir) == NULL);
+
+  for (auto p = offset_map.begin(); p != offset_map.end(); ++p) {
+    ceph_seekdir(cmount, ls_dir, p->second);
+    result = ceph_readdir(cmount, ls_dir);
+    ASSERT_TRUE(result != NULL);
+    std::string d_name(result->d_name);
+    ASSERT_EQ(p->first, d_name);
+  }
+
+  // test rewinddir
+  ceph_rewinddir(cmount, ls_dir);
+
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+
+  ceph_rewinddir(cmount, ls_dir);
+
+  int t = ceph_telldir(cmount, ls_dir);
+  ASSERT_GT(t, -1);
+
+  ASSERT_TRUE(ceph_readdir(cmount, ls_dir) != NULL);
+
+  // test seekdir - move back to the beginning
+  ceph_seekdir(cmount, ls_dir, t);
+
+  // test getdents
+  struct dirent *getdents_entries;
+  size_t getdents_entries_len = (r + 2) * sizeof(*getdents_entries);
+  getdents_entries = (struct dirent *)malloc(getdents_entries_len);
+
+  int count = 0;
+  std::vector<std::string> found;
+  while (true) {
+    int len = ceph_getdents(cmount, ls_dir, (char *)getdents_entries, getdents_entries_len);
+    if (len == 0)
+      break;
+    ASSERT_GT(len, 0);
+    ASSERT_TRUE((len % sizeof(*getdents_entries)) == 0);
+    int n = len / sizeof(*getdents_entries);
+    int j;
+    if (count == 0) {
+      ASSERT_STREQ(getdents_entries[0].d_name, ".");
+      ASSERT_STREQ(getdents_entries[1].d_name, "..");
+      j = 2;
+    } else {
+      j = 0;
+    }
+    count += n;
+    for(; j < n; ++i, ++j) {
+      const char *name = getdents_entries[j].d_name;
+      found.push_back(name);
+    }
+  }
+  ASSERT_EQ(found, entries);
+  free(getdents_entries);
+
+  // test readdir_r
+  ceph_rewinddir(cmount, ls_dir);
+
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+
+  found.clear();
+  while (true) {
+    struct dirent rdent;
+    int len = ceph_readdir_r(cmount, ls_dir, &rdent);
+    if (len == 0)
+      break;
+    ASSERT_EQ(len, 1);
+    found.push_back(rdent.d_name);
+  }
+  ASSERT_EQ(found, entries);
+
+  // test readdirplus
+  ceph_rewinddir(cmount, ls_dir);
+
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+
+  found.clear();
+  while (true) {
+    struct dirent rdent;
+    struct ceph_statx stx;
+    int len = ceph_readdirplus_r(cmount, ls_dir, &rdent, &stx,
+				 CEPH_STATX_SIZE, AT_STATX_DONT_SYNC, NULL);
+    if (len == 0)
+      break;
+    ASSERT_EQ(len, 1);
+    const char *name = rdent.d_name;
+    found.push_back(name);
+    int size;
+    sscanf(name, "dirf%d", &size);
+    ASSERT_TRUE(stx.stx_mask & CEPH_STATX_SIZE);
+    ASSERT_EQ(stx.stx_size, (size_t)size);
+    // On Windows, dirent uses long (4B) inodes, which get trimmed
+    // and can't be used.
+    // TODO: consider defining ceph_dirent.
+    #ifndef _WIN32
+    ASSERT_EQ(stx.stx_ino, rdent.d_ino);
+    #endif
+    //ASSERT_EQ(st.st_mode, (mode_t)0666);
+  }
+  ASSERT_EQ(found, entries);
+
+  ASSERT_EQ(ceph_closedir(cmount, ls_dir), 0);
+
+  // cleanup
+  for(i = 0; i < r; ++i) {
+    sprintf(bazstr, "dir_ls%d/dirf%d", mypid, i);
+    ASSERT_EQ(0, ceph_unlink(cmount, bazstr));
+  }
+  ASSERT_EQ(0, ceph_rmdir(cmount, foostr));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, ManyNestedDirs) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  const char *many_path = "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a";
+  ASSERT_EQ(ceph_mkdirs(cmount, many_path, 0755), 0);
+
+  int i = 0;
+
+  for(; i < 39; ++i) {
+    ASSERT_EQ(ceph_chdir(cmount, "a"), 0);
+
+    struct ceph_dir_result *dirp;
+    ASSERT_EQ(ceph_opendir(cmount, "a", &dirp), 0);
+    struct dirent *dent = ceph_readdir(cmount, dirp);
+    ASSERT_TRUE(dent != NULL);
+    ASSERT_STREQ(dent->d_name, ".");
+    dent = ceph_readdir(cmount, dirp);
+    ASSERT_TRUE(dent != NULL);
+    ASSERT_STREQ(dent->d_name, "..");
+    dent = ceph_readdir(cmount, dirp);
+    ASSERT_TRUE(dent != NULL);
+    ASSERT_STREQ(dent->d_name, "a");
+    ASSERT_EQ(ceph_closedir(cmount, dirp), 0);
+  }
+
+  ASSERT_STREQ(ceph_getcwd(cmount), "/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a");
+
+  ASSERT_EQ(ceph_chdir(cmount, "a/a/a"), 0);
+
+  for(i = 0; i < 39; ++i) {
+    ASSERT_EQ(ceph_chdir(cmount, ".."), 0);
+    ASSERT_EQ(ceph_rmdir(cmount, "a"), 0);
+  }
+
+  ASSERT_EQ(ceph_chdir(cmount, "/"), 0);
+
+  ASSERT_EQ(ceph_rmdir(cmount, "a/a/a"), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Xattrs) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_xattr_file[256];
+  sprintf(test_xattr_file, "test_xattr_%d", getpid());
+  int fd = ceph_open(cmount, test_xattr_file, O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+
+  // test removing non-existent xattr
+  ASSERT_EQ(-CEPHFS_ENODATA, ceph_removexattr(cmount, test_xattr_file, "user.nosuchxattr"));
+
+  char i = 'a';
+  char xattrk[128];
+  char xattrv[128];
+  for(; i < 'a'+26; ++i) {
+    sprintf(xattrk, "user.test_xattr_%c", i);
+    int len = sprintf(xattrv, "testxattr%c", i);
+    ASSERT_EQ(ceph_setxattr(cmount, test_xattr_file, xattrk, (void *) xattrv, len, XATTR_CREATE), 0);
+  }
+
+  // zero size should return required buffer length
+  int len_needed = ceph_listxattr(cmount, test_xattr_file, NULL, 0);
+  ASSERT_GT(len_needed, 0);
+
+  // buffer size smaller than needed should fail
+  char xattrlist[128*26];
+  ASSERT_GT(sizeof(xattrlist), (size_t)len_needed);
+  int len = ceph_listxattr(cmount, test_xattr_file, xattrlist, len_needed - 1);
+  ASSERT_EQ(-CEPHFS_ERANGE, len);
+
+  len = ceph_listxattr(cmount, test_xattr_file, xattrlist, sizeof(xattrlist));
+  ASSERT_EQ(len, len_needed);
+  char *p = xattrlist;
+  char *n;
+  i = 'a';
+  while (len > 0) {
+    // ceph.* xattrs should not be listed
+    ASSERT_NE(strncmp(p, "ceph.", 5), 0);
+
+    sprintf(xattrk, "user.test_xattr_%c", i);
+    ASSERT_STREQ(p, xattrk);
+
+    char gxattrv[128];
+    std::cout << "getting attr " << p << std::endl;
+    int alen = ceph_getxattr(cmount, test_xattr_file, p, (void *) gxattrv, 128);
+    ASSERT_GT(alen, 0);
+    sprintf(xattrv, "testxattr%c", i);
+    ASSERT_TRUE(!strncmp(xattrv, gxattrv, alen));
+
+    n = strchr(p, '\0');
+    n++;
+    len -= (n - p);
+    p = n;
+    ++i;
+  }
+
+  i = 'a';
+  for(i = 'a'; i < 'a'+26; ++i) {
+    sprintf(xattrk, "user.test_xattr_%c", i);
+    ASSERT_EQ(ceph_removexattr(cmount, test_xattr_file, xattrk), 0);
+  }
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+
+}
+
+TEST(LibCephFS, Xattrs_ll) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_xattr_file[256];
+  sprintf(test_xattr_file, "test_xattr_%d", getpid());
+  int fd = ceph_open(cmount, test_xattr_file, O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+  ceph_close(cmount, fd);
+
+  Inode *root = NULL;
+  Inode *existent_file_handle = NULL;
+
+  int res = ceph_ll_lookup_root(cmount, &root);
+  ASSERT_EQ(res, 0);
+
+  UserPerm *perms = ceph_mount_perms(cmount);
+  struct ceph_statx stx;
+
+  res = ceph_ll_lookup(cmount, root, test_xattr_file, &existent_file_handle,
+		       &stx, 0, 0, perms);
+  ASSERT_EQ(res, 0);
+
+  const char *valid_name = "user.attrname";
+  const char *value = "attrvalue";
+  char value_buf[256] = { 0 };
+
+  res = ceph_ll_setxattr(cmount, existent_file_handle, valid_name, value, strlen(value), 0, perms);
+  ASSERT_EQ(res, 0);
+
+  res = ceph_ll_getxattr(cmount, existent_file_handle, valid_name, value_buf, 256, perms);
+  ASSERT_EQ(res, (int)strlen(value));
+
+  value_buf[res] = '\0';
+  ASSERT_STREQ(value_buf, value);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LstatSlashdot) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_statx(cmount, "/.", &stx, 0, AT_SYMLINK_NOFOLLOW), 0);
+  ASSERT_EQ(ceph_statx(cmount, ".", &stx, 0, AT_SYMLINK_NOFOLLOW), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, StatDirNlink) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_dir1[256];
+  sprintf(test_dir1, "dir1_symlinks_%d", getpid());
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir1, 0700), 0);
+
+  int fd = ceph_open(cmount, test_dir1, O_DIRECTORY|O_RDONLY, 0);
+  ASSERT_GT(fd, 0);
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_fstatx(cmount, fd, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+  ASSERT_EQ(stx.stx_nlink, 2u);
+
+  {
+    char test_dir2[296];
+    sprintf(test_dir2, "%s/.", test_dir1);
+    ASSERT_EQ(ceph_statx(cmount, test_dir2, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+    ASSERT_EQ(stx.stx_nlink, 2u);
+  }
+
+  {
+    char test_dir2[296];
+    sprintf(test_dir2, "%s/1", test_dir1);
+    ASSERT_EQ(ceph_mkdir(cmount, test_dir2, 0700), 0);
+    ASSERT_EQ(ceph_statx(cmount, test_dir2, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+    ASSERT_EQ(stx.stx_nlink, 2u);
+      ASSERT_EQ(ceph_statx(cmount, test_dir1, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+      ASSERT_EQ(stx.stx_nlink, 3u);
+    sprintf(test_dir2, "%s/2", test_dir1);
+    ASSERT_EQ(ceph_mkdir(cmount, test_dir2, 0700), 0);
+      ASSERT_EQ(ceph_statx(cmount, test_dir1, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+      ASSERT_EQ(stx.stx_nlink, 4u);
+    sprintf(test_dir2, "%s/1/1", test_dir1);
+    ASSERT_EQ(ceph_mkdir(cmount, test_dir2, 0700), 0);
+      ASSERT_EQ(ceph_statx(cmount, test_dir1, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+      ASSERT_EQ(stx.stx_nlink, 4u);
+    ASSERT_EQ(ceph_rmdir(cmount, test_dir2), 0);
+      ASSERT_EQ(ceph_statx(cmount, test_dir1, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+      ASSERT_EQ(stx.stx_nlink, 4u);
+    sprintf(test_dir2, "%s/1", test_dir1);
+    ASSERT_EQ(ceph_rmdir(cmount, test_dir2), 0);
+      ASSERT_EQ(ceph_statx(cmount, test_dir1, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+      ASSERT_EQ(stx.stx_nlink, 3u);
+    sprintf(test_dir2, "%s/2", test_dir1);
+    ASSERT_EQ(ceph_rmdir(cmount, test_dir2), 0);
+      ASSERT_EQ(ceph_statx(cmount, test_dir1, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+      ASSERT_EQ(stx.stx_nlink, 2u);
+  }
+
+  ASSERT_EQ(ceph_rmdir(cmount, test_dir1), 0);
+  ASSERT_EQ(ceph_fstatx(cmount, fd, &stx, CEPH_STATX_NLINK, AT_SYMLINK_NOFOLLOW), 0);
+  ASSERT_EQ(stx.stx_nlink, 0u);
+
+  ceph_close(cmount, fd);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DoubleChmod) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_perms_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  // write some stuff
+  const char *bytes = "foobarbaz";
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+
+  ceph_close(cmount, fd);
+
+  // set perms to read but can't write
+  ASSERT_EQ(ceph_chmod(cmount, test_file, 0400), 0);
+
+  fd = ceph_open(cmount, test_file, O_RDWR, 0);
+  ASSERT_EQ(fd, -CEPHFS_EACCES);
+
+  fd = ceph_open(cmount, test_file, O_RDONLY, 0);
+  ASSERT_GT(fd, -1);
+
+  char buf[100];
+  int ret = ceph_read(cmount, fd, buf, 100, 0);
+  ASSERT_EQ(ret, (int)strlen(bytes));
+  buf[ret] = '\0';
+  ASSERT_STREQ(buf, bytes);
+
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), -CEPHFS_EBADF);
+
+  ceph_close(cmount, fd);
+
+  // reset back to writeable
+  ASSERT_EQ(ceph_chmod(cmount, test_file, 0600), 0);
+
+  // ensure perms are correct
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_statx(cmount, test_file, &stx, CEPH_STATX_MODE, AT_SYMLINK_NOFOLLOW), 0);
+  ASSERT_EQ(stx.stx_mode, 0100600U);
+
+  fd = ceph_open(cmount, test_file, O_RDWR, 0);
+  ASSERT_GT(fd, 0);
+
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+  ceph_close(cmount, fd);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Fchmod) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_perms_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  // write some stuff
+  const char *bytes = "foobarbaz";
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+
+  // set perms to read but can't write
+  ASSERT_EQ(ceph_fchmod(cmount, fd, 0400), 0);
+
+  char buf[100];
+  int ret = ceph_read(cmount, fd, buf, 100, 0);
+  ASSERT_EQ(ret, (int)strlen(bytes));
+  buf[ret] = '\0';
+  ASSERT_STREQ(buf, bytes);
+
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+
+  ceph_close(cmount, fd);
+
+  ASSERT_EQ(ceph_open(cmount, test_file, O_RDWR, 0), -CEPHFS_EACCES);
+
+  // reset back to writeable
+  ASSERT_EQ(ceph_chmod(cmount, test_file, 0600), 0);
+
+  fd = ceph_open(cmount, test_file, O_RDWR, 0);
+  ASSERT_GT(fd, 0);
+
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+  ceph_close(cmount, fd);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Lchmod) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_perms_lchmod_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  // write some stuff
+  const char *bytes = "foobarbaz";
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+  ceph_close(cmount, fd);
+
+  // Create symlink
+  char test_symlink[256];
+  sprintf(test_symlink, "test_lchmod_sym_%d", getpid());
+  ASSERT_EQ(ceph_symlink(cmount, test_file, test_symlink), 0);
+
+  // get symlink stat - lstat
+  struct ceph_statx stx_orig1;
+  ASSERT_EQ(ceph_statx(cmount, test_symlink, &stx_orig1, CEPH_STATX_ALL_STATS, AT_SYMLINK_NOFOLLOW), 0);
+
+  // Change mode on symlink file
+  ASSERT_EQ(ceph_lchmod(cmount, test_symlink, 0400), 0);
+  struct ceph_statx stx_orig2;
+  ASSERT_EQ(ceph_statx(cmount, test_symlink, &stx_orig2, CEPH_STATX_ALL_STATS, AT_SYMLINK_NOFOLLOW), 0);
+
+  // Compare modes
+  ASSERT_NE(stx_orig1.stx_mode, stx_orig2.stx_mode);
+  static const int permbits = S_IRWXU|S_IRWXG|S_IRWXO;
+  ASSERT_EQ(permbits&stx_orig1.stx_mode, 0777);
+  ASSERT_EQ(permbits&stx_orig2.stx_mode, 0400);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Fchown) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_fchown_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  // set perms to readable and writeable only by owner
+  ASSERT_EQ(ceph_fchmod(cmount, fd, 0600), 0);
+
+  // change ownership to nobody -- we assume nobody exists and id is always 65534
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "0"), 0);
+  ASSERT_EQ(ceph_fchown(cmount, fd, 65534, 65534), 0);
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "1"), 0);
+
+  ceph_close(cmount, fd);
+
+  // "nobody" will be ignored on Windows
+  #ifndef _WIN32
+  fd = ceph_open(cmount, test_file, O_RDWR, 0);
+  ASSERT_EQ(fd, -CEPHFS_EACCES);
+  #endif
+
+  ceph_shutdown(cmount);
+}
+
+#if defined(__linux__) && defined(O_PATH)
+TEST(LibCephFS, FlagO_PATH) {
+  struct ceph_mount_info *cmount;
+
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, NULL));
+
+  char test_file[PATH_MAX];
+  sprintf(test_file, "test_oflag_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR|O_PATH, 0666);
+  ASSERT_EQ(-CEPHFS_ENOENT, fd);
+
+  fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  // ok, the file has been created. perform real checks now
+  fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR|O_PATH, 0666);
+  ASSERT_GT(fd, 0);
+
+  char buf[128];
+  ASSERT_EQ(-CEPHFS_EBADF, ceph_read(cmount, fd, buf, sizeof(buf), 0));
+  ASSERT_EQ(-CEPHFS_EBADF, ceph_write(cmount, fd, buf, sizeof(buf), 0));
+
+  // set perms to readable and writeable only by owner
+  ASSERT_EQ(-CEPHFS_EBADF, ceph_fchmod(cmount, fd, 0600));
+
+  // change ownership to nobody -- we assume nobody exists and id is always 65534
+  ASSERT_EQ(-CEPHFS_EBADF, ceph_fchown(cmount, fd, 65534, 65534));
+
+  // try to sync
+  ASSERT_EQ(-CEPHFS_EBADF, ceph_fsync(cmount, fd, false));
+
+  struct ceph_statx stx;
+  ASSERT_EQ(0, ceph_fstatx(cmount, fd, &stx, 0, 0));
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ceph_shutdown(cmount);
+}
+#endif /* __linux */
+
+TEST(LibCephFS, Symlinks) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_symlinks_%d", getpid());
+
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  ceph_close(cmount, fd);
+
+  char test_symlink[256];
+  sprintf(test_symlink, "test_symlinks_sym_%d", getpid());
+
+  ASSERT_EQ(ceph_symlink(cmount, test_file, test_symlink), 0);
+
+  // test the O_NOFOLLOW case
+  fd = ceph_open(cmount, test_symlink, O_NOFOLLOW, 0);
+  ASSERT_EQ(fd, -CEPHFS_ELOOP);
+
+  // stat the original file
+  struct ceph_statx stx_orig;
+  ASSERT_EQ(ceph_statx(cmount, test_file, &stx_orig, CEPH_STATX_ALL_STATS, 0), 0);
+  // stat the symlink
+  struct ceph_statx stx_symlink_orig;
+  ASSERT_EQ(ceph_statx(cmount, test_symlink, &stx_symlink_orig, CEPH_STATX_ALL_STATS, 0), 0);
+  // ensure the statx bufs are equal
+  ASSERT_EQ(memcmp(&stx_orig, &stx_symlink_orig, sizeof(stx_orig)), 0);
+
+  sprintf(test_file, "/test_symlinks_abs_%d", getpid());
+
+  fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  ceph_close(cmount, fd);
+
+  sprintf(test_symlink, "/test_symlinks_abs_sym_%d", getpid());
+
+  ASSERT_EQ(ceph_symlink(cmount, test_file, test_symlink), 0);
+
+  // stat the original file
+  ASSERT_EQ(ceph_statx(cmount, test_file, &stx_orig, CEPH_STATX_ALL_STATS, 0), 0);
+  // stat the symlink
+  ASSERT_EQ(ceph_statx(cmount, test_symlink, &stx_symlink_orig, CEPH_STATX_ALL_STATS, 0), 0);
+  // ensure the statx bufs are equal
+  ASSERT_TRUE(!memcmp(&stx_orig, &stx_symlink_orig, sizeof(stx_orig)));
+
+  // test lstat
+  ASSERT_EQ(ceph_statx(cmount, test_symlink, &stx_orig, CEPH_STATX_ALL_STATS, AT_SYMLINK_NOFOLLOW), 0);
+  ASSERT_TRUE(S_ISLNK(stx_orig.stx_mode));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DirSyms) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_dir1[256];
+  sprintf(test_dir1, "dir1_symlinks_%d", getpid());
+
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir1, 0700), 0);
+
+  char test_symdir[256];
+  sprintf(test_symdir, "symdir_symlinks_%d", getpid());
+
+  ASSERT_EQ(ceph_symlink(cmount, test_dir1, test_symdir), 0);
+
+  char test_file[256];
+  sprintf(test_file, "/symdir_symlinks_%d/test_symdir_file", getpid());
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0600);
+  ASSERT_GT(fd, 0);
+  ceph_close(cmount, fd);
+
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_statx(cmount, test_file, &stx, 0, AT_SYMLINK_NOFOLLOW), 0);
+
+  // ensure that its a file not a directory we get back
+  ASSERT_TRUE(S_ISREG(stx.stx_mode));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LoopSyms) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_dir1[256];
+  sprintf(test_dir1, "dir1_loopsym_%d", getpid());
+
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir1, 0700), 0);
+
+  char test_dir2[256];
+  sprintf(test_dir2, "/dir1_loopsym_%d/loop_dir", getpid());
+
+  ASSERT_EQ(ceph_mkdir(cmount, test_dir2, 0700), 0);
+
+  // symlink it itself:  /path/to/mysym -> /path/to/mysym
+  char test_symdir[256];
+  sprintf(test_symdir, "/dir1_loopsym_%d/loop_dir/symdir", getpid());
+
+  ASSERT_EQ(ceph_symlink(cmount, test_symdir, test_symdir), 0);
+
+  char test_file[256];
+  sprintf(test_file, "/dir1_loopsym_%d/loop_dir/symdir/test_loopsym_file", getpid());
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0600);
+  ASSERT_EQ(fd, -CEPHFS_ELOOP);
+
+  // loop: /a -> /b, /b -> /c, /c -> /a
+  char a[264], b[264], c[264];
+  sprintf(a, "/%s/a", test_dir1);
+  sprintf(b, "/%s/b", test_dir1);
+  sprintf(c, "/%s/c", test_dir1);
+  ASSERT_EQ(ceph_symlink(cmount, a, b), 0);
+  ASSERT_EQ(ceph_symlink(cmount, b, c), 0);
+  ASSERT_EQ(ceph_symlink(cmount, c, a), 0);
+  ASSERT_EQ(ceph_open(cmount, a, O_RDWR, 0), -CEPHFS_ELOOP);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, HardlinkNoOriginal) {
+
+  int mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir[256];
+  sprintf(dir, "/test_rmdirfail%d", mypid);
+  ASSERT_EQ(ceph_mkdir(cmount, dir, 0777), 0);
+
+  ASSERT_EQ(ceph_chdir(cmount, dir), 0);
+
+  int fd = ceph_open(cmount, "f1", O_CREAT, 0644);
+  ASSERT_GT(fd, 0);
+
+  ceph_close(cmount, fd);
+
+  // create hard link
+  ASSERT_EQ(ceph_link(cmount, "f1", "hardl1"), 0);
+
+  // remove file link points to
+  ASSERT_EQ(ceph_unlink(cmount, "f1"), 0);
+
+  ceph_shutdown(cmount);
+
+  // now cleanup
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+  ASSERT_EQ(ceph_chdir(cmount, dir), 0);
+  ASSERT_EQ(ceph_unlink(cmount, "hardl1"), 0);
+  ASSERT_EQ(ceph_rmdir(cmount, dir), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, BadArgument) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  int fd = ceph_open(cmount, "test_file", O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+  char buf[100];
+  ASSERT_EQ(ceph_write(cmount, fd, buf, sizeof(buf), 0), (int)sizeof(buf));
+  ASSERT_EQ(ceph_read(cmount, fd, buf, 0, 5), 0);
+  ceph_close(cmount, fd);
+  ASSERT_EQ(ceph_unlink(cmount, "test_file"), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, BadFileDesc) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  ASSERT_EQ(ceph_fchmod(cmount, -1, 0655), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_close(cmount, -1), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_lseek(cmount, -1, 0, SEEK_SET), -CEPHFS_EBADF);
+
+  char buf[0];
+  ASSERT_EQ(ceph_read(cmount, -1, buf, 0, 0), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_write(cmount, -1, buf, 0, 0), -CEPHFS_EBADF);
+
+  ASSERT_EQ(ceph_ftruncate(cmount, -1, 0), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_fsync(cmount, -1, 0), -CEPHFS_EBADF);
+
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_fstatx(cmount, -1, &stx, 0, 0), -CEPHFS_EBADF);
+
+  struct sockaddr_storage addr;
+  ASSERT_EQ(ceph_get_file_stripe_address(cmount, -1, 0, &addr, 1), -CEPHFS_EBADF);
+
+  ASSERT_EQ(ceph_get_file_stripe_unit(cmount, -1), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_get_file_pool(cmount, -1), -CEPHFS_EBADF);
+  char poolname[80];
+  ASSERT_EQ(ceph_get_file_pool_name(cmount, -1, poolname, sizeof(poolname)), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_get_file_replication(cmount, -1), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_get_file_object_size(cmount, -1), -CEPHFS_EBADF);
+  int stripe_unit, stripe_count, object_size, pg_pool;
+  ASSERT_EQ(ceph_get_file_layout(cmount, -1, &stripe_unit, &stripe_count, &object_size, &pg_pool), -CEPHFS_EBADF);
+  ASSERT_EQ(ceph_get_file_stripe_count(cmount, -1), -CEPHFS_EBADF);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, ReadEmptyFile) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  // test the read_sync path in the client for zero files
+  ASSERT_EQ(ceph_conf_set(cmount, "client_debug_force_sync_read", "true"), 0);
+
+  int mypid = getpid();
+  char testf[256];
+
+  sprintf(testf, "test_reademptyfile%d", mypid);
+  int fd = ceph_open(cmount, testf, O_CREAT|O_TRUNC|O_WRONLY, 0644);
+  ASSERT_GT(fd, 0);
+
+  ceph_close(cmount, fd);
+
+  fd = ceph_open(cmount, testf, O_RDONLY, 0);
+  ASSERT_GT(fd, 0);
+
+  char buf[4096];
+  ASSERT_EQ(ceph_read(cmount, fd, buf, 4096, 0), 0);
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, PreadvPwritev) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  int mypid = getpid();
+  char testf[256];
+
+  sprintf(testf, "test_preadvpwritevfile%d", mypid);
+  int fd = ceph_open(cmount, testf, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  char out0[] = "hello ";
+  char out1[] = "world\n";
+  struct iovec iov_out[2] = {
+	{out0, sizeof(out0)},
+	{out1, sizeof(out1)},
+  };
+  char in0[sizeof(out0)];
+  char in1[sizeof(out1)];
+  struct iovec iov_in[2] = {
+	{in0, sizeof(in0)},
+	{in1, sizeof(in1)},
+  };
+  ssize_t nwritten = iov_out[0].iov_len + iov_out[1].iov_len; 
+  ssize_t nread = iov_in[0].iov_len + iov_in[1].iov_len; 
+
+  ASSERT_EQ(ceph_pwritev(cmount, fd, iov_out, 2, 0), nwritten);
+  ASSERT_EQ(ceph_preadv(cmount, fd, iov_in, 2, 0), nread);
+  ASSERT_EQ(0, strncmp((const char*)iov_in[0].iov_base, (const char*)iov_out[0].iov_base, iov_out[0].iov_len));
+  ASSERT_EQ(0, strncmp((const char*)iov_in[1].iov_base, (const char*)iov_out[1].iov_base, iov_out[1].iov_len));
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LlreadvLlwritev) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  int mypid = getpid();
+  char filename[256];
+
+  sprintf(filename, "test_llreadvllwritevfile%u", mypid);
+
+  Inode *root, *file;
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  Fh *fh;
+  struct ceph_statx stx;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
+		    O_RDWR|O_CREAT|O_TRUNC, &file, &fh, &stx, 0, 0, perms), 0);
+
+  /* Reopen read-only */
+  char out0[] = "hello ";
+  char out1[] = "world\n";
+  struct iovec iov_out[2] = {
+	{out0, sizeof(out0)},
+	{out1, sizeof(out1)},
+  };
+  char in0[sizeof(out0)];
+  char in1[sizeof(out1)];
+  struct iovec iov_in[2] = {
+	{in0, sizeof(in0)},
+	{in1, sizeof(in1)},
+  };
+  ssize_t nwritten = iov_out[0].iov_len + iov_out[1].iov_len;
+  ssize_t nread = iov_in[0].iov_len + iov_in[1].iov_len;
+
+  ASSERT_EQ(ceph_ll_writev(cmount, fh, iov_out, 2, 0), nwritten);
+  ASSERT_EQ(ceph_ll_readv(cmount, fh, iov_in, 2, 0), nread);
+  ASSERT_EQ(0, strncmp((const char*)iov_in[0].iov_base, (const char*)iov_out[0].iov_base, iov_out[0].iov_len));
+  ASSERT_EQ(0, strncmp((const char*)iov_in[1].iov_base, (const char*)iov_out[1].iov_base, iov_out[1].iov_len));
+
+  ceph_ll_close(cmount, fh);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, StripeUnitGran) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+  ASSERT_GT(ceph_get_stripe_unit_granularity(cmount), 0);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Rename) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  int mypid = getpid();
+  char path_src[256];
+  char path_dst[256];
+
+  /* make a source file */
+  sprintf(path_src, "test_rename_src%d", mypid);
+  int fd = ceph_open(cmount, path_src, O_CREAT|O_TRUNC|O_WRONLY, 0777);
+  ASSERT_GT(fd, 0);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  /* rename to a new dest path */
+  sprintf(path_dst, "test_rename_dst%d", mypid);
+  ASSERT_EQ(0, ceph_rename(cmount, path_src, path_dst));
+
+  /* test that dest path exists */
+  struct ceph_statx stx;
+  ASSERT_EQ(0, ceph_statx(cmount, path_dst, &stx, 0, 0));
+
+  /* test that src path doesn't exist */
+  ASSERT_EQ(-CEPHFS_ENOENT, ceph_statx(cmount, path_src, &stx, 0, AT_SYMLINK_NOFOLLOW));
+
+  /* rename with non-existent source path */
+  ASSERT_EQ(-CEPHFS_ENOENT, ceph_rename(cmount, path_src, path_dst));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, path_dst));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, UseUnmounted) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+
+  struct statvfs stvfs;
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_statfs(cmount, "/", &stvfs));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_local_osd(cmount));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_chdir(cmount, "/"));
+
+  struct ceph_dir_result *dirp;
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_opendir(cmount, "/", &dirp));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_closedir(cmount, dirp));
+
+  ceph_readdir(cmount, dirp);
+  EXPECT_EQ(CEPHFS_ENOTCONN, errno);
+
+  struct dirent rdent;
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_readdir_r(cmount, dirp, &rdent));
+
+  struct ceph_statx stx;
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_readdirplus_r(cmount, dirp, &rdent, &stx, 0, 0, NULL));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_getdents(cmount, dirp, NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_getdnames(cmount, dirp, NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_telldir(cmount, dirp));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_link(cmount, "/", "/link"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_unlink(cmount, "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_rename(cmount, "/path", "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_mkdir(cmount, "/", 0655));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_mkdirs(cmount, "/", 0655));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_rmdir(cmount, "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_readlink(cmount, "/path", NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_symlink(cmount, "/path", "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_statx(cmount, "/path", &stx, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_setattrx(cmount, "/path", &stx, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_getxattr(cmount, "/path", "name", NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_lgetxattr(cmount, "/path", "name", NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_listxattr(cmount, "/path", NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_llistxattr(cmount, "/path", NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_removexattr(cmount, "/path", "name"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_lremovexattr(cmount, "/path", "name"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_setxattr(cmount, "/path", "name", NULL, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_lsetxattr(cmount, "/path", "name", NULL, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_fsetattrx(cmount, 0, &stx, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_chmod(cmount, "/path", 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_fchmod(cmount, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_chown(cmount, "/path", 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_lchown(cmount, "/path", 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_fchown(cmount, 0, 0, 0));
+
+  struct utimbuf utb;
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_utime(cmount, "/path", &utb));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_truncate(cmount, "/path", 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_mknod(cmount, "/path", 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_open(cmount, "/path", 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_open_layout(cmount, "/path", 0, 0, 0, 0, 0, "pool"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_close(cmount, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_lseek(cmount, 0, 0, SEEK_SET));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_read(cmount, 0, NULL, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_write(cmount, 0, NULL, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_ftruncate(cmount, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_fsync(cmount, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_fstatx(cmount, 0, &stx, 0, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_sync_fs(cmount));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_file_stripe_unit(cmount, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_file_stripe_count(cmount, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_file_layout(cmount, 0, NULL, NULL ,NULL ,NULL));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_file_object_size(cmount, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_file_pool(cmount, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_file_pool_name(cmount, 0, NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_file_replication(cmount, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_path_replication(cmount, "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_path_layout(cmount, "/path", NULL, NULL, NULL, NULL));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_path_object_size(cmount, "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_path_stripe_count(cmount, "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_path_stripe_unit(cmount, "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_path_pool(cmount, "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_path_pool_name(cmount, "/path", NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_pool_name(cmount, 0, NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_file_stripe_address(cmount, 0, 0, NULL, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_localize_reads(cmount, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_debug_get_fd_caps(cmount, 0));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_debug_get_file_caps(cmount, "/path"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_stripe_unit_granularity(cmount));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_pool_id(cmount, "data"));
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_pool_replication(cmount, 1));
+
+  ceph_release(cmount);
+}
+
+TEST(LibCephFS, GetPoolId) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char name[80];
+  memset(name, 0, sizeof(name));
+  ASSERT_LE(0, ceph_get_path_pool_name(cmount, "/", name, sizeof(name)));
+  ASSERT_GE(ceph_get_pool_id(cmount, name), 0);
+  ASSERT_EQ(ceph_get_pool_id(cmount, "weflkjwelfjwlkejf"), -CEPHFS_ENOENT);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetPoolReplication) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  /* negative pools */
+  ASSERT_EQ(ceph_get_pool_replication(cmount, -10), -CEPHFS_ENOENT);
+
+  /* valid pool */
+  int pool_id;
+  int stripe_unit, stripe_count, object_size;
+  ASSERT_EQ(0, ceph_get_path_layout(cmount, "/", &stripe_unit, &stripe_count,
+				    &object_size, &pool_id));
+  ASSERT_GE(pool_id, 0);
+  ASSERT_GT(ceph_get_pool_replication(cmount, pool_id), 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetExtentOsds) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_file_extent_osds(cmount, 0, 0, NULL, NULL, 0));
+
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  int stripe_unit = (1<<18);
+
+  /* make a file! */
+  char test_file[256];
+  sprintf(test_file, "test_extent_osds_%d", getpid());
+  int fd = ceph_open_layout(cmount, test_file, O_CREAT|O_RDWR, 0666,
+      stripe_unit, 2, stripe_unit*2, NULL);
+  ASSERT_GT(fd, 0);
+
+  /* get back how many osds > 0 */
+  int ret = ceph_get_file_extent_osds(cmount, fd, 0, NULL, NULL, 0);
+  EXPECT_GT(ret, 0);
+
+  int64_t len;
+  int osds[ret];
+
+  /* full stripe extent */
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 0, &len, osds, ret));
+  EXPECT_EQ(len, (int64_t)stripe_unit);
+
+  /* half stripe extent */
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, stripe_unit/2, &len, osds, ret));
+  EXPECT_EQ(len, (int64_t)stripe_unit/2);
+
+  /* 1.5 stripe unit offset -1 byte */
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 3*stripe_unit/2-1, &len, osds, ret));
+  EXPECT_EQ(len, (int64_t)stripe_unit/2+1);
+
+  /* 1.5 stripe unit offset +1 byte */
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 3*stripe_unit/2+1, &len, osds, ret));
+  EXPECT_EQ(len, (int64_t)stripe_unit/2-1);
+
+  /* only when more than 1 osd */
+  if (ret > 1) {
+    EXPECT_EQ(-CEPHFS_ERANGE, ceph_get_file_extent_osds(cmount, fd, 0, NULL, osds, 1));
+  }
+
+  ceph_close(cmount, fd);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetOsdCrushLocation) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_osd_crush_location(cmount, 0, NULL, 0));
+
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  ASSERT_EQ(ceph_get_osd_crush_location(cmount, 0, NULL, 1), -CEPHFS_EINVAL);
+
+  char path[256];
+  ASSERT_EQ(ceph_get_osd_crush_location(cmount, 9999999, path, 0), -CEPHFS_ENOENT);
+  ASSERT_EQ(ceph_get_osd_crush_location(cmount, -1, path, 0), -CEPHFS_EINVAL);
+
+  char test_file[256];
+  sprintf(test_file, "test_osds_loc_%d", getpid());
+  int fd = ceph_open(cmount, test_file, O_CREAT|O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+
+  /* get back how many osds > 0 */
+  int ret = ceph_get_file_extent_osds(cmount, fd, 0, NULL, NULL, 0);
+  EXPECT_GT(ret, 0);
+
+  /* full stripe extent */
+  int osds[ret];
+  EXPECT_EQ(ret, ceph_get_file_extent_osds(cmount, fd, 0, NULL, osds, ret));
+
+  ASSERT_GT(ceph_get_osd_crush_location(cmount, 0, path, 0), 0);
+  ASSERT_EQ(ceph_get_osd_crush_location(cmount, 0, path, 1), -CEPHFS_ERANGE);
+
+  for (int i = 0; i < ret; i++) {
+    int len = ceph_get_osd_crush_location(cmount, osds[i], path, sizeof(path));
+    ASSERT_GT(len, 0);
+    int pos = 0;
+    while (pos < len) {
+      std::string type(path + pos);
+      ASSERT_GT((int)type.size(), 0);
+      pos += type.size() + 1;
+
+      std::string name(path + pos);
+      ASSERT_GT((int)name.size(), 0);
+      pos += name.size() + 1;
+    }
+  }
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetOsdAddr) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+
+  EXPECT_EQ(-CEPHFS_ENOTCONN, ceph_get_osd_addr(cmount, 0, NULL));
+
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  ASSERT_EQ(-CEPHFS_EINVAL, ceph_get_osd_addr(cmount, 0, NULL));
+
+  struct sockaddr_storage addr;
+  ASSERT_EQ(-CEPHFS_ENOENT, ceph_get_osd_addr(cmount, -1, &addr));
+  ASSERT_EQ(-CEPHFS_ENOENT, ceph_get_osd_addr(cmount, 9999999, &addr));
+
+  ASSERT_EQ(0, ceph_get_osd_addr(cmount, 0, &addr));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, OpenNoClose) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  pid_t mypid = getpid();
+  char str_buf[256];
+  sprintf(str_buf, "open_no_close_dir%d", mypid);
+  ASSERT_EQ(0, ceph_mkdirs(cmount, str_buf, 0777));
+
+  struct ceph_dir_result *ls_dir = NULL;
+  ASSERT_EQ(ceph_opendir(cmount, str_buf, &ls_dir), 0);
+
+  sprintf(str_buf, "open_no_close_file%d", mypid);
+  int fd = ceph_open(cmount, str_buf, O_RDONLY|O_CREAT, 0666);
+  ASSERT_LT(0, fd);
+
+  // shutdown should force close opened file/dir
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Nlink) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  Inode *root, *dir, *file;
+
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+
+  char dirname[32], filename[32], linkname[32];
+  sprintf(dirname, "nlinkdir%x", getpid());
+  sprintf(filename, "nlinkorig%x", getpid());
+  sprintf(linkname, "nlinklink%x", getpid());
+
+  struct ceph_statx stx;
+  Fh *fh;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  ASSERT_EQ(ceph_ll_mkdir(cmount, root, dirname, 0755, &dir, &stx, 0, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_create(cmount, dir, filename, 0666, O_RDWR|O_CREAT|O_EXCL,
+			   &file, &fh, &stx, CEPH_STATX_NLINK, 0, perms), 0);
+  ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
+  ASSERT_EQ(stx.stx_nlink, (nlink_t)1);
+
+  ASSERT_EQ(ceph_ll_link(cmount, file, dir, linkname, perms), 0);
+  ASSERT_EQ(ceph_ll_getattr(cmount, file, &stx, CEPH_STATX_NLINK, 0, perms), 0);
+  ASSERT_EQ(stx.stx_nlink, (nlink_t)2);
+
+  ASSERT_EQ(ceph_ll_unlink(cmount, dir, linkname, perms), 0);
+  ASSERT_EQ(ceph_ll_lookup(cmount, dir, filename, &file, &stx,
+			   CEPH_STATX_NLINK, 0, perms), 0);
+  ASSERT_EQ(stx.stx_nlink, (nlink_t)1);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SlashDotDot) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  struct ceph_statx	stx;
+  ASSERT_EQ(ceph_statx(cmount, "/.", &stx, CEPH_STATX_INO, 0), 0);
+
+  ino_t ino = stx.stx_ino;
+  ASSERT_EQ(ceph_statx(cmount, "/..", &stx, CEPH_STATX_INO, 0), 0);
+
+  /* At root, "." and ".." should be the same inode */
+  ASSERT_EQ(ino, stx.stx_ino);
+
+  /* Test accessing the parent of an unlinked directory */
+  char dir1[32], dir2[56];
+  sprintf(dir1, "/sldotdot%x", getpid());
+  sprintf(dir2, "%s/sub%x", dir1, getpid());
+
+  ASSERT_EQ(ceph_mkdir(cmount, dir1, 0755), 0);
+  ASSERT_EQ(ceph_mkdir(cmount, dir2, 0755), 0);
+
+  ASSERT_EQ(ceph_chdir(cmount, dir2), 0);
+
+  /* Test behavior when unlinking cwd */
+  struct ceph_dir_result *rdir;
+  ASSERT_EQ(ceph_opendir(cmount, ".", &rdir), 0);
+  ASSERT_EQ(ceph_rmdir(cmount, dir2), 0);
+
+  /* get "." entry */
+  struct dirent *result = ceph_readdir(cmount, rdir);
+  ino = result->d_ino;
+
+  /* get ".." entry */
+  result = ceph_readdir(cmount, rdir);
+  ASSERT_EQ(ino, result->d_ino);
+  ceph_closedir(cmount, rdir);
+
+  /* Make sure it works same way when mounting subtree */
+  ASSERT_EQ(ceph_unmount(cmount), 0);
+  ASSERT_EQ(ceph_mount(cmount, dir1), 0);
+  ASSERT_EQ(ceph_statx(cmount, "/..", &stx, CEPH_STATX_INO, 0), 0);
+
+  /* Test readdir behavior */
+  ASSERT_EQ(ceph_opendir(cmount, "/", &rdir), 0);
+  result = ceph_readdir(cmount, rdir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  ino = result->d_ino;
+  result = ceph_readdir(cmount, rdir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+  ASSERT_EQ(ino, result->d_ino);
+
+  ceph_shutdown(cmount);
+}
+
+static inline bool
+timespec_eq(timespec const& lhs, timespec const& rhs)
+{
+  return lhs.tv_sec == rhs.tv_sec && lhs.tv_nsec == rhs.tv_nsec;
+}
+
+TEST(LibCephFS, Btime) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char filename[32];
+  sprintf(filename, "/getattrx%x", getpid());
+
+  ceph_unlink(cmount, filename);
+  int fd = ceph_open(cmount, filename, O_RDWR|O_CREAT|O_EXCL, 0666);
+  ASSERT_LT(0, fd);
+
+  /* make sure fstatx works */
+  struct ceph_statx	stx;
+
+  ASSERT_EQ(ceph_fstatx(cmount, fd, &stx, CEPH_STATX_CTIME|CEPH_STATX_BTIME, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & (CEPH_STATX_CTIME|CEPH_STATX_BTIME));
+  ASSERT_TRUE(timespec_eq(stx.stx_ctime, stx.stx_btime));
+  ceph_close(cmount, fd);
+
+  ASSERT_EQ(ceph_statx(cmount, filename, &stx, CEPH_STATX_CTIME|CEPH_STATX_BTIME, 0), 0);
+  ASSERT_TRUE(timespec_eq(stx.stx_ctime, stx.stx_btime));
+  ASSERT_TRUE(stx.stx_mask & (CEPH_STATX_CTIME|CEPH_STATX_BTIME));
+
+  struct timespec old_btime = stx.stx_btime;
+
+  /* Now sleep, do a chmod and verify that the ctime changed, but btime didn't */
+  sleep(1);
+  ASSERT_EQ(ceph_chmod(cmount, filename, 0644), 0);
+  ASSERT_EQ(ceph_statx(cmount, filename, &stx, CEPH_STATX_CTIME|CEPH_STATX_BTIME, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_BTIME);
+  ASSERT_TRUE(timespec_eq(stx.stx_btime, old_btime));
+  ASSERT_FALSE(timespec_eq(stx.stx_ctime, stx.stx_btime));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SetBtime) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char filename[32];
+  sprintf(filename, "/setbtime%x", getpid());
+
+  ceph_unlink(cmount, filename);
+  int fd = ceph_open(cmount, filename, O_RDWR|O_CREAT|O_EXCL, 0666);
+  ASSERT_LT(0, fd);
+  ceph_close(cmount, fd);
+
+  struct ceph_statx stx;
+  struct timespec old_btime = { 1, 2 };
+
+  stx.stx_btime = old_btime;
+
+  ASSERT_EQ(ceph_setattrx(cmount, filename, &stx, CEPH_SETATTR_BTIME, 0), 0);
+
+  ASSERT_EQ(ceph_statx(cmount, filename, &stx, CEPH_STATX_BTIME, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_BTIME);
+  ASSERT_TRUE(timespec_eq(stx.stx_btime, old_btime));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LazyStatx) {
+  struct ceph_mount_info *cmount1, *cmount2;
+  ASSERT_EQ(ceph_create(&cmount1, NULL), 0);
+  ASSERT_EQ(ceph_create(&cmount2, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount1, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount2, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount1, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount2, NULL));
+  ASSERT_EQ(ceph_mount(cmount1, "/"), 0);
+  ASSERT_EQ(ceph_mount(cmount2, "/"), 0);
+
+  char filename[32];
+  sprintf(filename, "lazystatx%x", getpid());
+
+  Inode *root1, *file1, *root2, *file2;
+  struct ceph_statx stx;
+  Fh *fh;
+  UserPerm *perms1 = ceph_mount_perms(cmount1);
+  UserPerm *perms2 = ceph_mount_perms(cmount2);
+
+  ASSERT_EQ(ceph_ll_lookup_root(cmount1, &root1), 0);
+  ceph_ll_unlink(cmount1, root1, filename, perms1);
+  ASSERT_EQ(ceph_ll_create(cmount1, root1, filename, 0666, O_RDWR|O_CREAT|O_EXCL,
+			   &file1, &fh, &stx, 0, 0, perms1), 0);
+  ASSERT_EQ(ceph_ll_close(cmount1, fh), 0);
+
+  ASSERT_EQ(ceph_ll_lookup_root(cmount2, &root2), 0);
+
+  ASSERT_EQ(ceph_ll_lookup(cmount2, root2, filename, &file2, &stx, CEPH_STATX_CTIME, 0, perms2), 0);
+
+  struct timespec old_ctime = stx.stx_ctime;
+
+  /*
+   * Now sleep, do a chmod on the first client and the see whether we get a
+   * different ctime with a statx that uses AT_STATX_DONT_SYNC
+   */
+  sleep(1);
+  stx.stx_mode = 0644;
+  ASSERT_EQ(ceph_ll_setattr(cmount1, file1, &stx, CEPH_SETATTR_MODE, perms1), 0);
+
+  ASSERT_EQ(ceph_ll_getattr(cmount2, file2, &stx, CEPH_STATX_CTIME, AT_STATX_DONT_SYNC, perms2), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_CTIME);
+  ASSERT_TRUE(stx.stx_ctime.tv_sec == old_ctime.tv_sec &&
+	      stx.stx_ctime.tv_nsec == old_ctime.tv_nsec);
+
+  ceph_shutdown(cmount1);
+  ceph_shutdown(cmount2);
+}
+
+TEST(LibCephFS, ChangeAttr) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char filename[32];
+  sprintf(filename, "/changeattr%x", getpid());
+
+  ceph_unlink(cmount, filename);
+  int fd = ceph_open(cmount, filename, O_RDWR|O_CREAT|O_EXCL, 0666);
+  ASSERT_LT(0, fd);
+
+  struct ceph_statx	stx;
+  ASSERT_EQ(ceph_statx(cmount, filename, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+
+  uint64_t old_change_attr = stx.stx_version;
+
+  /* do chmod, and check whether change_attr changed */
+  ASSERT_EQ(ceph_chmod(cmount, filename, 0644), 0);
+  ASSERT_EQ(ceph_statx(cmount, filename, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+  ASSERT_NE(stx.stx_version, old_change_attr);
+  old_change_attr = stx.stx_version;
+
+  /* now do a write and see if it changed again */
+  ASSERT_EQ(3, ceph_write(cmount, fd, "foo", 3, 0));
+  ASSERT_EQ(ceph_statx(cmount, filename, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+  ASSERT_NE(stx.stx_version, old_change_attr);
+  old_change_attr = stx.stx_version;
+
+  /* Now truncate and check again */
+  ASSERT_EQ(0, ceph_ftruncate(cmount, fd, 0));
+  ASSERT_EQ(ceph_statx(cmount, filename, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+  ASSERT_NE(stx.stx_version, old_change_attr);
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DirChangeAttrCreateFile) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dirpath[32], filepath[56];
+  sprintf(dirpath, "/dirchange%x", getpid());
+  sprintf(filepath, "%s/foo", dirpath);
+
+  ASSERT_EQ(ceph_mkdir(cmount, dirpath, 0755), 0);
+
+  struct ceph_statx	stx;
+  ASSERT_EQ(ceph_statx(cmount, dirpath, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+
+  uint64_t old_change_attr = stx.stx_version;
+
+  /* Important: Follow an operation that changes the directory's ctime (setxattr)
+   * with one that changes the directory's mtime and ctime (create).
+   * Check that directory's change_attr is updated everytime ctime changes.
+   */
+
+  /* set xattr on dir, and check whether dir's change_attr is incremented */
+  ASSERT_EQ(ceph_setxattr(cmount, dirpath, "user.name", (void*)"bob", 3, XATTR_CREATE), 0);
+  ASSERT_EQ(ceph_statx(cmount, dirpath, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+  ASSERT_GT(stx.stx_version, old_change_attr);
+  old_change_attr = stx.stx_version;
+
+  /* create a file within dir, and check whether dir's change_attr is incremented */
+  int fd = ceph_open(cmount, filepath, O_RDWR|O_CREAT|O_EXCL, 0666);
+  ASSERT_LT(0, fd);
+  ceph_close(cmount, fd);
+  ASSERT_EQ(ceph_statx(cmount, dirpath, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+  ASSERT_GT(stx.stx_version, old_change_attr);
+
+  ASSERT_EQ(0, ceph_unlink(cmount, filepath));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dirpath));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DirChangeAttrRenameFile) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dirpath[32], filepath[56], newfilepath[56];
+  sprintf(dirpath, "/dirchange%x", getpid());
+  sprintf(filepath, "%s/foo", dirpath);
+
+  ASSERT_EQ(ceph_mkdir(cmount, dirpath, 0755), 0);
+
+  int fd = ceph_open(cmount, filepath, O_RDWR|O_CREAT|O_EXCL, 0666);
+  ASSERT_LT(0, fd);
+  ceph_close(cmount, fd);
+
+  /* Important: Follow an operation that changes the directory's ctime (setattr)
+   * with one that changes the directory's mtime and ctime (rename).
+   * Check that directory's change_attr is updated everytime ctime changes.
+   */
+  struct ceph_statx	stx;
+  ASSERT_EQ(ceph_statx(cmount, dirpath, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+
+  uint64_t old_change_attr = stx.stx_version;
+
+  /* chmod dir, and check whether dir's change_attr is incremented */
+  ASSERT_EQ(ceph_chmod(cmount, dirpath, 0777), 0);
+  ASSERT_EQ(ceph_statx(cmount, dirpath, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+  ASSERT_GT(stx.stx_version, old_change_attr);
+  old_change_attr = stx.stx_version;
+
+  /* rename a file within dir, and check whether dir's change_attr is incremented */
+  sprintf(newfilepath, "%s/bar", dirpath);
+  ASSERT_EQ(ceph_rename(cmount, filepath, newfilepath), 0);
+  ASSERT_EQ(ceph_statx(cmount, dirpath, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+  ASSERT_GT(stx.stx_version, old_change_attr);
+
+  ASSERT_EQ(0, ceph_unlink(cmount, newfilepath));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dirpath));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, DirChangeAttrRemoveFile) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dirpath[32], filepath[56];
+  sprintf(dirpath, "/dirchange%x", getpid());
+  sprintf(filepath, "%s/foo", dirpath);
+
+  ASSERT_EQ(ceph_mkdir(cmount, dirpath, 0755), 0);
+
+  ASSERT_EQ(ceph_setxattr(cmount, dirpath, "user.name", (void*)"bob", 3, XATTR_CREATE), 0);
+
+  int fd = ceph_open(cmount, filepath, O_RDWR|O_CREAT|O_EXCL, 0666);
+  ASSERT_LT(0, fd);
+  ceph_close(cmount, fd);
+
+  /* Important: Follow an operation that changes the directory's ctime (removexattr)
+   * with one that changes the directory's mtime and ctime (remove a file).
+   * Check that directory's change_attr is updated everytime ctime changes.
+   */
+  struct ceph_statx	stx;
+  ASSERT_EQ(ceph_statx(cmount, dirpath, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+
+  uint64_t old_change_attr = stx.stx_version;
+
+  /* remove xattr, and check whether dir's change_attr is incremented */
+  ASSERT_EQ(ceph_removexattr(cmount, dirpath, "user.name"), 0);
+  ASSERT_EQ(ceph_statx(cmount, dirpath, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+  ASSERT_GT(stx.stx_version, old_change_attr);
+  old_change_attr = stx.stx_version;
+
+  /* remove a file within dir, and check whether dir's change_attr is incremented */
+  ASSERT_EQ(0, ceph_unlink(cmount, filepath));
+  ASSERT_EQ(ceph_statx(cmount, dirpath, &stx, CEPH_STATX_VERSION, 0), 0);
+  ASSERT_TRUE(stx.stx_mask & CEPH_STATX_VERSION);
+  ASSERT_GT(stx.stx_version, old_change_attr);
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, dirpath));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SetSize) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char filename[32];
+  sprintf(filename, "/setsize%x", getpid());
+
+  ceph_unlink(cmount, filename);
+  int fd = ceph_open(cmount, filename, O_RDWR|O_CREAT|O_EXCL, 0666);
+  ASSERT_LT(0, fd);
+
+  struct ceph_statx stx;
+  uint64_t size = 8388608;
+  stx.stx_size = size;
+  ASSERT_EQ(ceph_fsetattrx(cmount, fd, &stx, CEPH_SETATTR_SIZE), 0);
+  ASSERT_EQ(ceph_fstatx(cmount, fd, &stx, CEPH_STATX_SIZE, 0), 0);
+  ASSERT_EQ(stx.stx_size, size);
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, OperationsOnRoot)
+{
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dirname[32];
+  sprintf(dirname, "/somedir%x", getpid());
+
+  ASSERT_EQ(ceph_mkdir(cmount, dirname, 0755), 0);
+
+  ASSERT_EQ(ceph_rmdir(cmount, "/"), -CEPHFS_EBUSY);
+
+  ASSERT_EQ(ceph_link(cmount, "/", "/"), -CEPHFS_EEXIST);
+  ASSERT_EQ(ceph_link(cmount, dirname, "/"), -CEPHFS_EEXIST);
+  ASSERT_EQ(ceph_link(cmount, "nonExisitingDir", "/"), -CEPHFS_ENOENT);
+
+  ASSERT_EQ(ceph_unlink(cmount, "/"), -CEPHFS_EISDIR);
+
+  ASSERT_EQ(ceph_rename(cmount, "/", "/"), -CEPHFS_EBUSY);
+  ASSERT_EQ(ceph_rename(cmount, dirname, "/"), -CEPHFS_EBUSY);
+  ASSERT_EQ(ceph_rename(cmount, "nonExistingDir", "/"), -CEPHFS_EBUSY);
+  ASSERT_EQ(ceph_rename(cmount, "/", dirname), -CEPHFS_EBUSY);
+  ASSERT_EQ(ceph_rename(cmount, "/", "nonExistingDir"), -CEPHFS_EBUSY);
+
+  ASSERT_EQ(ceph_mkdir(cmount, "/", 0777), -CEPHFS_EEXIST);
+
+  ASSERT_EQ(ceph_mknod(cmount, "/", 0, 0), -CEPHFS_EEXIST);
+
+  ASSERT_EQ(ceph_symlink(cmount, "/", "/"), -CEPHFS_EEXIST);
+  ASSERT_EQ(ceph_symlink(cmount, dirname, "/"), -CEPHFS_EEXIST);
+  ASSERT_EQ(ceph_symlink(cmount, "nonExistingDir", "/"), -CEPHFS_EEXIST);
+
+  ceph_shutdown(cmount);
+}
+
+// no rlimits on Windows
+#ifndef _WIN32
+static void shutdown_racer_func()
+{
+  const int niter = 32;
+  struct ceph_mount_info *cmount;
+  int i;
+
+  for (i = 0; i < niter; ++i) {
+    ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+    ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+    ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+    ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+    ceph_shutdown(cmount);
+  }
+}
+
+// See tracker #20988
+TEST(LibCephFS, ShutdownRace)
+{
+  const int nthreads = 32;
+  std::thread threads[nthreads];
+
+  // Need a bunch of fd's for this test
+  struct rlimit rold, rnew;
+  ASSERT_EQ(getrlimit(RLIMIT_NOFILE, &rold), 0);
+  rnew = rold;
+  rnew.rlim_cur = rnew.rlim_max;
+
+  cout << "Setting RLIMIT_NOFILE from " << rold.rlim_cur <<
+	  " to " << rnew.rlim_cur << std::endl;
+
+  ASSERT_EQ(setrlimit(RLIMIT_NOFILE, &rnew), 0);
+
+  for (int i = 0; i < nthreads; ++i)
+    threads[i] = std::thread(shutdown_racer_func);
+
+  for (int i = 0; i < nthreads; ++i)
+    threads[i].join();
+  /*
+   * Let's just ignore restoring the open files limit,
+   * the kernel will defer releasing the file descriptors
+   * and then the process will be possibly reachthe open
+   * files limit. More detail, please see tracer#43039
+   */
+//  ASSERT_EQ(setrlimit(RLIMIT_NOFILE, &rold), 0);
+}
+#endif
+
+static void get_current_time_utimbuf(struct utimbuf *utb)
+{
+  utime_t t = ceph_clock_now();
+  utb->actime = t.sec();
+  utb->modtime = t.sec();
+}
+
+static void get_current_time_timeval(struct timeval tv[2])
+{
+  utime_t t = ceph_clock_now();
+  t.copy_to_timeval(&tv[0]);
+  t.copy_to_timeval(&tv[1]);
+}
+
+static void get_current_time_timespec(struct timespec ts[2])
+{
+  utime_t t = ceph_clock_now();
+  t.to_timespec(&ts[0]);
+  t.to_timespec(&ts[1]);
+}
+
+TEST(LibCephFS, TestUtime) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  sprintf(test_file, "test_utime_file_%d", getpid());
+  int fd = ceph_open(cmount, test_file, O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+
+  struct utimbuf utb;
+  struct ceph_statx stx;
+
+  get_current_time_utimbuf(&utb);
+
+  // ceph_utime()
+  EXPECT_EQ(0, ceph_utime(cmount, test_file, &utb));
+  ASSERT_EQ(ceph_statx(cmount, test_file, &stx,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME, 0), 0);
+  ASSERT_EQ(utime_t(stx.stx_atime), utime_t(utb.actime, 0));
+  ASSERT_EQ(utime_t(stx.stx_mtime), utime_t(utb.modtime, 0));
+
+  get_current_time_utimbuf(&utb);
+
+  // ceph_futime()
+  EXPECT_EQ(0, ceph_futime(cmount, fd, &utb));
+  ASSERT_EQ(ceph_statx(cmount, test_file, &stx,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME, 0), 0);
+  ASSERT_EQ(utime_t(stx.stx_atime), utime_t(utb.actime, 0));
+  ASSERT_EQ(utime_t(stx.stx_mtime), utime_t(utb.modtime, 0));
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, TestUtimes) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+  char test_symlink[256];
+
+  sprintf(test_file, "test_utimes_file_%d", getpid());
+  sprintf(test_symlink, "test_utimes_symlink_%d", getpid());
+  int fd = ceph_open(cmount, test_file, O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+
+  ASSERT_EQ(ceph_symlink(cmount, test_file, test_symlink), 0);
+
+  struct timeval times[2];
+  struct ceph_statx stx;
+
+  get_current_time_timeval(times);
+
+  // ceph_utimes() on symlink, validate target file time
+  EXPECT_EQ(0, ceph_utimes(cmount, test_symlink, times));
+  ASSERT_EQ(ceph_statx(cmount, test_symlink, &stx,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME, 0), 0);
+  ASSERT_EQ(utime_t(stx.stx_atime), utime_t(times[0]));
+  ASSERT_EQ(utime_t(stx.stx_mtime), utime_t(times[1]));
+
+  get_current_time_timeval(times);
+
+  // ceph_lutimes() on symlink, validate symlink time
+  EXPECT_EQ(0, ceph_lutimes(cmount, test_symlink, times));
+  ASSERT_EQ(ceph_statx(cmount, test_symlink, &stx,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME, AT_SYMLINK_NOFOLLOW), 0);
+  ASSERT_EQ(utime_t(stx.stx_atime), utime_t(times[0]));
+  ASSERT_EQ(utime_t(stx.stx_mtime), utime_t(times[1]));
+
+  get_current_time_timeval(times);
+
+  // ceph_futimes()
+  EXPECT_EQ(0, ceph_futimes(cmount, fd, times));
+  ASSERT_EQ(ceph_statx(cmount, test_file, &stx,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME, 0), 0);
+  ASSERT_EQ(utime_t(stx.stx_atime), utime_t(times[0]));
+  ASSERT_EQ(utime_t(stx.stx_mtime), utime_t(times[1]));
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, TestFutimens) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_file[256];
+
+  sprintf(test_file, "test_futimens_file_%d", getpid());
+  int fd = ceph_open(cmount, test_file, O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+
+  struct timespec times[2];
+  struct ceph_statx stx;
+
+  get_current_time_timespec(times);
+
+  // ceph_futimens()
+  EXPECT_EQ(0, ceph_futimens(cmount, fd, times));
+  ASSERT_EQ(ceph_statx(cmount, test_file, &stx,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME, 0), 0);
+  ASSERT_EQ(utime_t(stx.stx_atime), utime_t(times[0]));
+  ASSERT_EQ(utime_t(stx.stx_mtime), utime_t(times[1]));
+
+  ceph_close(cmount, fd);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, OperationsOnDotDot) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char c_dir[512], c_dir_dot[1024], c_dir_dotdot[1024];
+  char c_non_existent_dir[1024], c_non_existent_dirs[1024];
+  char c_temp[1024];
+
+  pid_t mypid = getpid();
+  sprintf(c_dir, "/oodd_dir_%d", mypid);
+  sprintf(c_dir_dot, "/%s/.", c_dir);
+  sprintf(c_dir_dotdot, "/%s/..", c_dir);
+  sprintf(c_non_existent_dir, "/%s/../oodd_nonexistent/..", c_dir);
+  sprintf(c_non_existent_dirs,
+          "/%s/../ood_nonexistent1_%d/oodd_nonexistent2_%d", c_dir, mypid, mypid);
+  sprintf(c_temp, "/oodd_temp_%d", mypid);
+
+  ASSERT_EQ(0, ceph_mkdir(cmount, c_dir, 0777));
+  ASSERT_EQ(-CEPHFS_EEXIST, ceph_mkdir(cmount, c_dir_dot, 0777));
+  ASSERT_EQ(-CEPHFS_EEXIST, ceph_mkdir(cmount, c_dir_dotdot, 0777));
+  ASSERT_EQ(0, ceph_mkdirs(cmount, c_non_existent_dirs, 0777));
+
+  ASSERT_EQ(-CEPHFS_ENOTEMPTY, ceph_rmdir(cmount, c_dir_dot));
+  ASSERT_EQ(-CEPHFS_ENOTEMPTY, ceph_rmdir(cmount, c_dir_dotdot));
+  // non existent directory should return -CEPHFS_ENOENT
+  ASSERT_EQ(-CEPHFS_ENOENT, ceph_rmdir(cmount, c_non_existent_dir));
+
+  ASSERT_EQ(-CEPHFS_EBUSY, ceph_rename(cmount, c_dir_dot, c_temp));
+  ASSERT_EQ(0, ceph_chdir(cmount, c_dir));
+  ASSERT_EQ(0, ceph_mkdir(cmount, c_temp, 0777));
+  ASSERT_EQ(-CEPHFS_EBUSY, ceph_rename(cmount, c_temp, ".."));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Caps_vxattr) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_caps_vxattr_file[256];
+  char gxattrv[128];
+  int xbuflen = sizeof(gxattrv);
+  pid_t mypid = getpid();
+
+  sprintf(test_caps_vxattr_file, "test_caps_vxattr_%d", mypid);
+  int fd = ceph_open(cmount, test_caps_vxattr_file, O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+  ceph_close(cmount, fd);
+
+  int alen = ceph_getxattr(cmount, test_caps_vxattr_file, "ceph.caps", (void *)gxattrv, xbuflen);
+  ASSERT_GT(alen, 0);
+  gxattrv[alen] = '\0';
+
+  char caps_regex[] = "pA[sx]*L[sx]*X[sx]*F[sxcrwbal]*/0x[0-9a-fA-f]+";
+  ASSERT_TRUE(regex_match(gxattrv, regex(caps_regex)) == 1);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SnapXattrs) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_snap_xattr_file[256];
+  char c_temp[PATH_MAX];
+  char gxattrv[128];
+  char gxattrv2[128];
+  int xbuflen = sizeof(gxattrv);
+  pid_t mypid = getpid();
+
+  sprintf(test_snap_xattr_file, "test_snap_xattr_%d", mypid);
+  int fd = ceph_open(cmount, test_snap_xattr_file, O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+  ceph_close(cmount, fd);
+
+  sprintf(c_temp, "/.snap/test_snap_xattr_snap_%d", mypid);
+  ASSERT_EQ(0, ceph_mkdir(cmount, c_temp, 0777));
+
+  int alen = ceph_getxattr(cmount, c_temp, "ceph.snap.btime", (void *)gxattrv, xbuflen);
+  // xattr value is secs.nsecs (don't assume zero-term)
+  ASSERT_LT(0, alen);
+  ASSERT_LT(alen, xbuflen);
+  gxattrv[alen] = '\0';
+  char *s = strchr(gxattrv, '.');
+  char *q = NULL;
+  ASSERT_NE(q, s);
+  ASSERT_LT(s, gxattrv + alen);
+  ASSERT_EQ('.', *s);
+  *s = '\0';
+  utime_t btime = utime_t(strtoull(gxattrv, NULL, 10), strtoull(s + 1, NULL, 10));
+  *s = '.';  // restore for later strcmp
+
+  // file within the snapshot should carry the same btime
+  sprintf(c_temp, "/.snap/test_snap_xattr_snap_%d/%s", mypid, test_snap_xattr_file);
+
+  int alen2 = ceph_getxattr(cmount, c_temp, "ceph.snap.btime", (void *)gxattrv2, xbuflen);
+  ASSERT_EQ(alen, alen2);
+  ASSERT_EQ(0, strncmp(gxattrv, gxattrv2, alen));
+
+  // non-snap file shouldn't carry the xattr
+  alen = ceph_getxattr(cmount, test_snap_xattr_file, "ceph.snap.btime", (void *)gxattrv2, xbuflen);
+  ASSERT_EQ(-CEPHFS_ENODATA, alen);
+
+  // create a second snapshot
+  sprintf(c_temp, "/.snap/test_snap_xattr_snap2_%d", mypid);
+  ASSERT_EQ(0, ceph_mkdir(cmount, c_temp, 0777));
+
+  // check that the btime for the newer snapshot is > older
+  alen = ceph_getxattr(cmount, c_temp, "ceph.snap.btime", (void *)gxattrv2, xbuflen);
+  ASSERT_LT(0, alen);
+  ASSERT_LT(alen, xbuflen);
+  gxattrv2[alen] = '\0';
+  s = strchr(gxattrv2, '.');
+  ASSERT_NE(q, s);
+  ASSERT_LT(s, gxattrv2 + alen);
+  ASSERT_EQ('.', *s);
+  *s = '\0';
+  utime_t new_btime = utime_t(strtoull(gxattrv2, NULL, 10), strtoull(s + 1, NULL, 10));
+  #ifndef _WIN32
+  // This assertion sometimes fails on Windows, possibly due to the clock precision.
+  ASSERT_LT(btime, new_btime);
+  #endif
+
+  // listxattr() shouldn't return snap.btime vxattr
+  char xattrlist[512];
+  int len = ceph_listxattr(cmount, test_snap_xattr_file, xattrlist, sizeof(xattrlist));
+  ASSERT_GE(sizeof(xattrlist), (size_t)len);
+  char *p = xattrlist;
+  int found = 0;
+  while (len > 0) {
+    if (strcmp(p, "ceph.snap.btime") == 0)
+      found++;
+    len -= strlen(p) + 1;
+    p += strlen(p) + 1;
+  }
+  ASSERT_EQ(found, 0);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Lseek) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  char c_path[1024];
+  sprintf(c_path, "test_lseek_%d", getpid());
+  int fd = ceph_open(cmount, c_path, O_RDWR|O_CREAT|O_TRUNC, 0666);
+  ASSERT_LT(0, fd);
+
+  const char *out_buf = "hello world";
+  size_t size = strlen(out_buf);
+  ASSERT_EQ(ceph_write(cmount, fd, out_buf, size, 0), (int)size);
+
+  /* basic SEEK_SET/END/CUR tests */
+  ASSERT_EQ(0, ceph_lseek(cmount, fd, 0, SEEK_SET));
+  ASSERT_EQ(size, ceph_lseek(cmount, fd, 0, SEEK_END));
+  ASSERT_EQ(0, ceph_lseek(cmount, fd, -size, SEEK_CUR));
+
+  /* Test basic functionality and out of bounds conditions for SEEK_HOLE/DATA */
+#ifdef SEEK_HOLE
+  ASSERT_EQ(size, ceph_lseek(cmount, fd, 0, SEEK_HOLE));
+  ASSERT_EQ(-CEPHFS_ENXIO, ceph_lseek(cmount, fd, -1, SEEK_HOLE));
+  ASSERT_EQ(-CEPHFS_ENXIO, ceph_lseek(cmount, fd, size + 1, SEEK_HOLE));
+#endif
+#ifdef SEEK_DATA
+  ASSERT_EQ(0, ceph_lseek(cmount, fd, 0, SEEK_DATA));
+  ASSERT_EQ(-CEPHFS_ENXIO, ceph_lseek(cmount, fd, -1, SEEK_DATA));
+  ASSERT_EQ(-CEPHFS_ENXIO, ceph_lseek(cmount, fd, size + 1, SEEK_DATA));
+#endif
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SnapInfoOnNonSnapshot) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  struct snap_info info;
+  ASSERT_EQ(-CEPHFS_EINVAL, ceph_get_snap_info(cmount, "/", &info));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, EmptySnapInfo) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_path[64];
+  char snap_path[PATH_MAX];
+  sprintf(dir_path, "/dir0_%d", getpid());
+  sprintf(snap_path, "%s/.snap/snap0_%d", dir_path, getpid());
+
+  ASSERT_EQ(0, ceph_mkdir(cmount, dir_path, 0755));
+  // snapshot without custom metadata
+  ASSERT_EQ(0, ceph_mkdir(cmount, snap_path, 0755));
+
+  struct snap_info info;
+  ASSERT_EQ(0, ceph_get_snap_info(cmount, snap_path, &info));
+  ASSERT_GT(info.id, 0);
+  ASSERT_EQ(info.nr_snap_metadata, 0);
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, snap_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SnapInfo) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_path[64];
+  char snap_name[64];
+  char snap_path[PATH_MAX];
+  sprintf(dir_path, "/dir0_%d", getpid());
+  sprintf(snap_name, "snap0_%d", getpid());
+  sprintf(snap_path, "%s/.snap/%s", dir_path, snap_name);
+
+  ASSERT_EQ(0, ceph_mkdir(cmount, dir_path, 0755));
+  // snapshot with custom metadata
+  struct snap_metadata snap_meta[] = {{"foo", "bar"},{"this", "that"},{"abcdefg", "12345"}};
+  ASSERT_EQ(0, ceph_mksnap(cmount, dir_path, snap_name, 0755, snap_meta, std::size(snap_meta)));
+
+  struct snap_info info;
+  ASSERT_EQ(0, ceph_get_snap_info(cmount, snap_path, &info));
+  ASSERT_GT(info.id, 0);
+  ASSERT_EQ(info.nr_snap_metadata, std::size(snap_meta));
+  for (size_t i = 0; i < info.nr_snap_metadata; ++i) {
+    auto &k1 = info.snap_metadata[i].key;
+    auto &v1 = info.snap_metadata[i].value;
+    bool found = false;
+    for (size_t j = 0; j < info.nr_snap_metadata; ++j) {
+      auto &k2 = snap_meta[j].key;
+      auto &v2 = snap_meta[j].value;
+      if (strncmp(k1, k2, strlen(k1)) == 0 && strncmp(v1, v2, strlen(v1)) == 0) {
+        found = true;
+        break;
+      }
+    }
+    ASSERT_TRUE(found);
+  }
+  ceph_free_snap_info_buffer(&info);
+
+  ASSERT_EQ(0, ceph_rmsnap(cmount, dir_path, snap_name));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LookupInoMDSDir) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  Inode *inode;
+  auto ino = inodeno_t(0x100); /* rank 0 ~mdsdir */
+  ASSERT_EQ(-CEPHFS_ESTALE, ceph_ll_lookup_inode(cmount, ino, &inode));
+  ino = inodeno_t(0x600); /* rank 0 first stray dir */
+  ASSERT_EQ(-CEPHFS_ESTALE, ceph_ll_lookup_inode(cmount, ino, &inode));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LookupVino) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_path[64];
+  char snap_name[64];
+  char snapdir_path[128];
+  char snap_path[256];
+  char file_path[PATH_MAX];
+  char snap_file[PATH_MAX];
+  sprintf(dir_path, "/dir0_%d", getpid());
+  sprintf(snap_name, "snap0_%d", getpid());
+  sprintf(file_path, "%s/file_%d", dir_path, getpid());
+  sprintf(snapdir_path, "%s/.snap", dir_path);
+  sprintf(snap_path, "%s/%s", snapdir_path, snap_name);
+  sprintf(snap_file, "%s/file_%d", snap_path, getpid());
+
+  ASSERT_EQ(0, ceph_mkdir(cmount, dir_path, 0755));
+  int fd = ceph_open(cmount, file_path, O_WRONLY|O_CREAT, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_mksnap(cmount, dir_path, snap_name, 0755, nullptr, 0));
+
+  // record vinos for all of them
+  struct ceph_statx stx;
+  ASSERT_EQ(0, ceph_statx(cmount, dir_path, &stx, CEPH_STATX_INO, 0));
+  vinodeno_t dir_vino(stx.stx_ino, stx.stx_dev);
+
+  ASSERT_EQ(0, ceph_statx(cmount, file_path, &stx, CEPH_STATX_INO, 0));
+  vinodeno_t file_vino(stx.stx_ino, stx.stx_dev);
+
+  ASSERT_EQ(0, ceph_statx(cmount, snapdir_path, &stx, CEPH_STATX_INO, 0));
+  vinodeno_t snapdir_vino(stx.stx_ino, stx.stx_dev);
+
+  ASSERT_EQ(0, ceph_statx(cmount, snap_path, &stx, CEPH_STATX_INO, 0));
+  vinodeno_t snap_vino(stx.stx_ino, stx.stx_dev);
+
+  ASSERT_EQ(0, ceph_statx(cmount, snap_file, &stx, CEPH_STATX_INO, 0));
+  vinodeno_t snap_file_vino(stx.stx_ino, stx.stx_dev);
+
+  // Remount
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, NULL));
+
+  // Find them all
+  Inode *inode;
+  ASSERT_EQ(0, ceph_ll_lookup_vino(cmount, dir_vino, &inode));
+  ceph_ll_put(cmount, inode);
+  ASSERT_EQ(0, ceph_ll_lookup_vino(cmount, file_vino, &inode));
+  ceph_ll_put(cmount, inode);
+  ASSERT_EQ(0, ceph_ll_lookup_vino(cmount, snapdir_vino, &inode));
+  ceph_ll_put(cmount, inode);
+  ASSERT_EQ(0, ceph_ll_lookup_vino(cmount, snap_vino, &inode));
+  ceph_ll_put(cmount, inode);
+  ASSERT_EQ(0, ceph_ll_lookup_vino(cmount, snap_file_vino, &inode));
+  ceph_ll_put(cmount, inode);
+
+  // cleanup
+  ASSERT_EQ(0, ceph_rmsnap(cmount, dir_path, snap_name));
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Openat) {
+  pid_t mypid = getpid();
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  char c_rel_dir[64];
+  char c_dir[128];
+  sprintf(c_rel_dir, "open_test_%d", mypid);
+  sprintf(c_dir, "/%s", c_rel_dir);
+  ASSERT_EQ(0, ceph_mkdir(cmount, c_dir, 0777));
+
+  int root_fd = ceph_open(cmount, "/", O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, root_fd);
+
+  int dir_fd = ceph_openat(cmount, root_fd, c_rel_dir, O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, dir_fd);
+
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_statxat(cmount, root_fd, c_rel_dir, &stx, 0, 0), 0);
+  ASSERT_EQ(stx.stx_mode & S_IFMT, S_IFDIR);
+
+  char c_rel_path[64];
+  char c_path[256];
+  sprintf(c_rel_path, "created_file_%d", mypid);
+  sprintf(c_path, "%s/created_file_%d", c_dir, mypid);
+  int file_fd = ceph_openat(cmount, dir_fd, c_rel_path, O_RDONLY | O_CREAT, 0666);
+  ASSERT_LE(0, file_fd);
+
+  ASSERT_EQ(ceph_statxat(cmount, dir_fd, c_rel_path, &stx, 0, 0), 0);
+  ASSERT_EQ(stx.stx_mode & S_IFMT, S_IFREG);
+
+  ASSERT_EQ(0, ceph_close(cmount, file_fd));
+  ASSERT_EQ(0, ceph_close(cmount, dir_fd));
+  ASSERT_EQ(0, ceph_close(cmount, root_fd));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, c_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, c_dir));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Statxat) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_name[64];
+  char rel_file_name_1[128];
+  char rel_file_name_2[256];
+
+  char dir_path[512];
+  char file_path[1024];
+
+  // relative paths for *at() calls
+  sprintf(dir_name, "dir0_%d", getpid());
+  sprintf(rel_file_name_1, "file_%d", getpid());
+  sprintf(rel_file_name_2, "%s/%s", dir_name, rel_file_name_1);
+
+  sprintf(dir_path, "/%s", dir_name);
+  sprintf(file_path, "%s/%s", dir_path, rel_file_name_1);
+
+  ASSERT_EQ(0, ceph_mkdir(cmount, dir_path, 0755));
+  int fd = ceph_open(cmount, file_path, O_WRONLY|O_CREAT, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  struct ceph_statx stx;
+
+  // test relative to root
+  fd = ceph_open(cmount, "/", O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(ceph_statxat(cmount, fd, dir_name, &stx, 0, 0), 0);
+  ASSERT_EQ(stx.stx_mode & S_IFMT, S_IFDIR);
+  ASSERT_EQ(ceph_statxat(cmount, fd, rel_file_name_2, &stx, 0, 0), 0);
+  ASSERT_EQ(stx.stx_mode & S_IFMT, S_IFREG);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  // test relative to dir
+  fd = ceph_open(cmount, dir_path, O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(ceph_statxat(cmount, fd, rel_file_name_1, &stx, 0, 0), 0);
+  ASSERT_EQ(stx.stx_mode & S_IFMT, S_IFREG);
+
+  // delete the dirtree, recreate and verify
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ASSERT_EQ(0, ceph_mkdir(cmount, dir_path, 0755));
+  int fd1 = ceph_open(cmount, file_path, O_WRONLY|O_CREAT, 0666);
+  ASSERT_LE(0, fd1);
+  ASSERT_EQ(0, ceph_close(cmount, fd1));
+  ASSERT_EQ(ceph_statxat(cmount, fd, rel_file_name_1, &stx, 0, 0), -CEPHFS_ENOENT);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, StatxatATFDCWD) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_name[64];
+  char rel_file_name_1[128];
+
+  char dir_path[512];
+  char file_path[1024];
+
+  // relative paths for *at() calls
+  sprintf(dir_name, "dir0_%d", getpid());
+  sprintf(rel_file_name_1, "file_%d", getpid());
+
+  sprintf(dir_path, "/%s", dir_name);
+  sprintf(file_path, "%s/%s", dir_path, rel_file_name_1);
+
+  ASSERT_EQ(0, ceph_mkdir(cmount, dir_path, 0755));
+  int fd = ceph_open(cmount, file_path, O_WRONLY|O_CREAT, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  struct ceph_statx stx;
+  // chdir and test with CEPHFS_AT_FDCWD
+  ASSERT_EQ(0, ceph_chdir(cmount, dir_path));
+  ASSERT_EQ(ceph_statxat(cmount, CEPHFS_AT_FDCWD, rel_file_name_1, &stx, 0, 0), 0);
+  ASSERT_EQ(stx.stx_mode & S_IFMT, S_IFREG);
+
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Fdopendir) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char foostr[256];
+  sprintf(foostr, "/dir_ls%d", mypid);
+  ASSERT_EQ(ceph_mkdir(cmount, foostr, 0777), 0);
+
+  char bazstr[512];
+  sprintf(bazstr, "%s/elif", foostr);
+  int fd  = ceph_open(cmount, bazstr, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  fd = ceph_open(cmount, foostr, O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+  struct ceph_dir_result *ls_dir = NULL;
+  ASSERT_EQ(ceph_fdopendir(cmount, fd, &ls_dir), 0);
+
+  // not guaranteed to get . and .. first, but its a safe assumption in this case
+  struct dirent *result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "elif");
+
+  ASSERT_TRUE(ceph_readdir(cmount, ls_dir) == NULL);
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_closedir(cmount, ls_dir));
+  ASSERT_EQ(0, ceph_unlink(cmount, bazstr));
+  ASSERT_EQ(0, ceph_rmdir(cmount, foostr));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, FdopendirATFDCWD) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char foostr[256];
+  sprintf(foostr, "/dir_ls%d", mypid);
+  ASSERT_EQ(ceph_mkdir(cmount, foostr, 0777), 0);
+
+  char bazstr[512];
+  sprintf(bazstr, "%s/elif", foostr);
+  int fd  = ceph_open(cmount, bazstr, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  ASSERT_EQ(0, ceph_chdir(cmount, foostr));
+  struct ceph_dir_result *ls_dir = NULL;
+  ASSERT_EQ(ceph_fdopendir(cmount, CEPHFS_AT_FDCWD, &ls_dir), 0);
+
+  // not guaranteed to get . and .. first, but its a safe assumption in this case
+  struct dirent *result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "elif");
+
+  ASSERT_TRUE(ceph_readdir(cmount, ls_dir) == NULL);
+
+  ASSERT_EQ(0, ceph_closedir(cmount, ls_dir));
+  ASSERT_EQ(0, ceph_unlink(cmount, bazstr));
+  ASSERT_EQ(0, ceph_rmdir(cmount, foostr));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, FdopendirReaddirTestWithDelete) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char foostr[256];
+  sprintf(foostr, "/dir_ls%d", mypid);
+  ASSERT_EQ(ceph_mkdir(cmount, foostr, 0777), 0);
+
+  char bazstr[512];
+  sprintf(bazstr, "%s/elif", foostr);
+  int fd  = ceph_open(cmount, bazstr, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  fd = ceph_open(cmount, foostr, O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+  struct ceph_dir_result *ls_dir = NULL;
+  ASSERT_EQ(ceph_fdopendir(cmount, fd, &ls_dir), 0);
+
+  ASSERT_EQ(0, ceph_unlink(cmount, bazstr));
+  ASSERT_EQ(0, ceph_rmdir(cmount, foostr));
+
+  // not guaranteed to get . and .. first, but its a safe assumption
+  // in this case. also, note that we may or may not get other
+  // entries.
+  struct dirent *result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, ".");
+  result = ceph_readdir(cmount, ls_dir);
+  ASSERT_TRUE(result != NULL);
+  ASSERT_STREQ(result->d_name, "..");
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_closedir(cmount, ls_dir));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, FdopendirOnNonDir) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char foostr[256];
+  sprintf(foostr, "/dir_ls%d", mypid);
+  ASSERT_EQ(ceph_mkdir(cmount, foostr, 0777), 0);
+
+  char bazstr[512];
+  sprintf(bazstr, "%s/file", foostr);
+  int fd  = ceph_open(cmount, bazstr, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+
+  struct ceph_dir_result *ls_dir = NULL;
+  ASSERT_EQ(ceph_fdopendir(cmount, fd, &ls_dir), -CEPHFS_ENOTDIR);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, bazstr));
+  ASSERT_EQ(0, ceph_rmdir(cmount, foostr));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Mkdirat) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path1[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path1, "/%s", dir_name);
+
+  char dir_path2[512];
+  char rel_dir_path2[512];
+  sprintf(dir_path2, "%s/dir_%d", dir_path1, mypid);
+  sprintf(rel_dir_path2, "%s/dir_%d", dir_name, mypid);
+
+  int fd = ceph_open(cmount, "/", O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+
+  ASSERT_EQ(0, ceph_mkdirat(cmount, fd, dir_name, 0777));
+  ASSERT_EQ(0, ceph_mkdirat(cmount, fd, rel_dir_path2, 0666));
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path2));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path1));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, MkdiratATFDCWD) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path1[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path1, "/%s", dir_name);
+
+  char dir_path2[512];
+  sprintf(dir_path2, "%s/dir_%d", dir_path1, mypid);
+
+  ASSERT_EQ(0, ceph_mkdirat(cmount, CEPHFS_AT_FDCWD, dir_name, 0777));
+
+  ASSERT_EQ(0, ceph_chdir(cmount, dir_path1));
+  ASSERT_EQ(0, ceph_mkdirat(cmount, CEPHFS_AT_FDCWD, dir_name, 0666));
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path2));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path1));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Readlinkat) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512];
+  sprintf(rel_file_path, "%s/elif", dir_name);
+  sprintf(file_path, "%s/elif", dir_path);
+  int fd  = ceph_open(cmount, file_path, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  char link_path[128];
+  char rel_link_path[64];
+  sprintf(rel_link_path, "linkfile_%d", mypid);
+  sprintf(link_path, "/%s", rel_link_path);
+  ASSERT_EQ(0, ceph_symlink(cmount, rel_file_path, link_path));
+
+  fd = ceph_open(cmount, "/", O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+  size_t target_len = strlen(rel_file_path);
+  char target[target_len+1];
+  ASSERT_EQ(target_len, ceph_readlinkat(cmount, fd, rel_link_path, target, target_len));
+  target[target_len] = '\0';
+  ASSERT_EQ(0, memcmp(target, rel_file_path, target_len));
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_unlink(cmount, link_path));
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, ReadlinkatATFDCWD) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "./elif";
+  sprintf(file_path, "%s/elif", dir_path);
+  int fd  = ceph_open(cmount, file_path, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  char link_path[PATH_MAX];
+  char rel_link_path[1024];
+  sprintf(rel_link_path, "./linkfile_%d", mypid);
+  sprintf(link_path, "%s/%s", dir_path, rel_link_path);
+  ASSERT_EQ(0, ceph_symlink(cmount, rel_file_path, link_path));
+
+  ASSERT_EQ(0, ceph_chdir(cmount, dir_path));
+  size_t target_len = strlen(rel_file_path);
+  char target[target_len+1];
+  ASSERT_EQ(target_len, ceph_readlinkat(cmount, CEPHFS_AT_FDCWD, rel_link_path, target, target_len));
+  target[target_len] = '\0';
+  ASSERT_EQ(0, memcmp(target, rel_file_path, target_len));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, link_path));
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Symlinkat) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512];
+  sprintf(rel_file_path, "%s/elif", dir_name);
+  sprintf(file_path, "%s/elif", dir_path);
+
+  int fd  = ceph_open(cmount, file_path, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  char link_path[128];
+  char rel_link_path[64];
+  sprintf(rel_link_path, "linkfile_%d", mypid);
+  sprintf(link_path, "/%s", rel_link_path);
+
+  fd = ceph_open(cmount, "/", O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_symlinkat(cmount, rel_file_path, fd, rel_link_path));
+
+  size_t target_len = strlen(rel_file_path);
+  char target[target_len+1];
+  ASSERT_EQ(target_len, ceph_readlinkat(cmount, fd, rel_link_path, target, target_len));
+  target[target_len] = '\0';
+  ASSERT_EQ(0, memcmp(target, rel_file_path, target_len));
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_unlink(cmount, link_path));
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SymlinkatATFDCWD) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "./elif";
+  sprintf(file_path, "%s/elif", dir_path);
+  int fd  = ceph_open(cmount, file_path, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  char link_path[PATH_MAX];
+  char rel_link_path[1024];
+  sprintf(rel_link_path, "./linkfile_%d", mypid);
+  sprintf(link_path, "%s/%s", dir_path, rel_link_path);
+  ASSERT_EQ(0, ceph_chdir(cmount, dir_path));
+  ASSERT_EQ(0, ceph_symlinkat(cmount, rel_file_path, CEPHFS_AT_FDCWD, rel_link_path));
+
+  size_t target_len = strlen(rel_file_path);
+  char target[target_len+1];
+  ASSERT_EQ(target_len, ceph_readlinkat(cmount, CEPHFS_AT_FDCWD, rel_link_path, target, target_len));
+  target[target_len] = '\0';
+  ASSERT_EQ(0, memcmp(target, rel_file_path, target_len));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, link_path));
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Unlinkat) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "elif";
+  sprintf(file_path, "%s/elif", dir_path);
+
+  int fd  = ceph_open(cmount, file_path, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  fd = ceph_open(cmount, dir_path, O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(-CEPHFS_ENOTDIR, ceph_unlinkat(cmount, fd, rel_file_path, AT_REMOVEDIR));
+  ASSERT_EQ(0, ceph_unlinkat(cmount, fd, rel_file_path, 0));
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  fd = ceph_open(cmount, "/", O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_EQ(-CEPHFS_EISDIR, ceph_unlinkat(cmount, fd, dir_name, 0));
+  ASSERT_EQ(0, ceph_unlinkat(cmount, fd, dir_name, AT_REMOVEDIR));
+  ASSERT_LE(0, fd);
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, UnlinkatATFDCWD) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "elif";
+  sprintf(file_path, "%s/elif", dir_path);
+
+  int fd  = ceph_open(cmount, file_path, O_CREAT|O_RDONLY, 0666);
+  ASSERT_LE(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  ASSERT_EQ(0, ceph_chdir(cmount, dir_path));
+  ASSERT_EQ(-CEPHFS_ENOTDIR, ceph_unlinkat(cmount, CEPHFS_AT_FDCWD, rel_file_path, AT_REMOVEDIR));
+  ASSERT_EQ(0, ceph_unlinkat(cmount, CEPHFS_AT_FDCWD, rel_file_path, 0));
+
+  ASSERT_EQ(0, ceph_chdir(cmount, "/"));
+  ASSERT_EQ(-CEPHFS_EISDIR, ceph_unlinkat(cmount, CEPHFS_AT_FDCWD, dir_name, 0));
+  ASSERT_EQ(0, ceph_unlinkat(cmount, CEPHFS_AT_FDCWD, dir_name, AT_REMOVEDIR));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Chownat) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "elif";
+  sprintf(file_path, "%s/elif", dir_path);
+  int fd = ceph_open(cmount, file_path, O_CREAT|O_RDWR, 0666);
+  ASSERT_LE(0, fd);
+
+  // set perms to readable and writeable only by owner
+  ASSERT_EQ(ceph_fchmod(cmount, fd, 0600), 0);
+  ceph_close(cmount, fd);
+
+  fd = ceph_open(cmount, dir_path, O_DIRECTORY | O_RDONLY, 0);
+  // change ownership to nobody -- we assume nobody exists and id is always 65534
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "0"), 0);
+  ASSERT_EQ(ceph_chownat(cmount, fd, rel_file_path, 65534, 65534, 0), 0);
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "1"), 0);
+  ceph_close(cmount, fd);
+
+  // "nobody" will be ignored on Windows
+  #ifndef _WIN32
+  fd = ceph_open(cmount, file_path, O_RDWR, 0);
+  ASSERT_EQ(fd, -CEPHFS_EACCES);
+  #endif
+
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "0"), 0);
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "1"), 0);
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, ChownatATFDCWD) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "elif";
+  sprintf(file_path, "%s/elif", dir_path);
+  int fd = ceph_open(cmount, file_path, O_CREAT|O_RDWR, 0666);
+  ASSERT_LE(0, fd);
+
+  // set perms to readable and writeable only by owner
+  ASSERT_EQ(ceph_fchmod(cmount, fd, 0600), 0);
+  ceph_close(cmount, fd);
+
+  ASSERT_EQ(0, ceph_chdir(cmount, dir_path));
+  // change ownership to nobody -- we assume nobody exists and id is always 65534
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "0"), 0);
+  ASSERT_EQ(ceph_chownat(cmount, CEPHFS_AT_FDCWD, rel_file_path, 65534, 65534, 0), 0);
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "1"), 0);
+
+  // "nobody" will be ignored on Windows
+  #ifndef _WIN32
+  fd = ceph_open(cmount, file_path, O_RDWR, 0);
+  ASSERT_EQ(fd, -CEPHFS_EACCES);
+  #endif
+
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "0"), 0);
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(ceph_conf_set(cmount, "client_permissions", "1"), 0);
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Chmodat) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "elif";
+  sprintf(file_path, "%s/elif", dir_path);
+  int fd = ceph_open(cmount, file_path, O_CREAT|O_RDWR, 0666);
+  ASSERT_LE(0, fd);
+  const char *bytes = "foobarbaz";
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  fd = ceph_open(cmount, dir_path, O_DIRECTORY | O_RDONLY, 0);
+
+  // set perms to read but can't write
+  ASSERT_EQ(ceph_chmodat(cmount, fd, rel_file_path, 0400, 0), 0);
+  ASSERT_EQ(ceph_open(cmount, file_path, O_RDWR, 0), -CEPHFS_EACCES);
+
+  // reset back to writeable
+  ASSERT_EQ(ceph_chmodat(cmount, fd, rel_file_path, 0600, 0), 0);
+  int fd2 = ceph_open(cmount, file_path, O_RDWR, 0);
+  ASSERT_LE(0, fd2);
+
+  ASSERT_EQ(0, ceph_close(cmount, fd2));
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, ChmodatATFDCWD) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, "/"), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "elif";
+  sprintf(file_path, "%s/elif", dir_path);
+  int fd = ceph_open(cmount, file_path, O_CREAT|O_RDWR, 0666);
+  ASSERT_LE(0, fd);
+  const char *bytes = "foobarbaz";
+  ASSERT_EQ(ceph_write(cmount, fd, bytes, strlen(bytes), 0), (int)strlen(bytes));
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  // set perms to read but can't write
+  ASSERT_EQ(0, ceph_chdir(cmount, dir_path));
+  ASSERT_EQ(ceph_chmodat(cmount, CEPHFS_AT_FDCWD, rel_file_path, 0400, 0), 0);
+  ASSERT_EQ(ceph_open(cmount, file_path, O_RDWR, 0), -CEPHFS_EACCES);
+
+  // reset back to writeable
+  ASSERT_EQ(ceph_chmodat(cmount, CEPHFS_AT_FDCWD, rel_file_path, 0600, 0), 0);
+  int fd2 = ceph_open(cmount, file_path, O_RDWR, 0);
+  ASSERT_LE(0, fd2);
+  ASSERT_EQ(0, ceph_close(cmount, fd2));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, Utimensat) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "elif";
+  sprintf(file_path, "%s/elif", dir_path);
+  int fd = ceph_open(cmount, file_path, O_CREAT|O_RDWR, 0666);
+  ASSERT_LE(0, fd);
+
+  struct timespec times[2];
+  get_current_time_timespec(times);
+
+  fd = ceph_open(cmount, dir_path, O_DIRECTORY | O_RDONLY, 0);
+  ASSERT_LE(0, fd);
+  EXPECT_EQ(0, ceph_utimensat(cmount, fd, rel_file_path, times, 0));
+  ceph_close(cmount, fd);
+
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_statx(cmount, file_path, &stx,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME, 0), 0);
+  ASSERT_EQ(utime_t(stx.stx_atime), utime_t(times[0]));
+  ASSERT_EQ(utime_t(stx.stx_mtime), utime_t(times[1]));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, UtimensatATFDCWD) {
+  pid_t mypid = getpid();
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  char file_path[512];
+  char rel_file_path[512] = "elif";
+  sprintf(file_path, "%s/elif", dir_path);
+  int fd = ceph_open(cmount, file_path, O_CREAT|O_RDWR, 0666);
+  ASSERT_LE(0, fd);
+
+  struct timespec times[2];
+  get_current_time_timespec(times);
+
+  ASSERT_EQ(0, ceph_chdir(cmount, dir_path));
+  EXPECT_EQ(0, ceph_utimensat(cmount, CEPHFS_AT_FDCWD, rel_file_path, times, 0));
+
+  struct ceph_statx stx;
+  ASSERT_EQ(ceph_statx(cmount, file_path, &stx,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME, 0), 0);
+  ASSERT_EQ(utime_t(stx.stx_atime), utime_t(times[0]));
+  ASSERT_EQ(utime_t(stx.stx_mtime), utime_t(times[1]));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LookupMdsPrivateInos) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  Inode *inode;
+  for (int ino = 0; ino < MDS_INO_SYSTEM_BASE; ino++) {
+    if (MDS_IS_PRIVATE_INO(ino)) {
+      ASSERT_EQ(-CEPHFS_ESTALE, ceph_ll_lookup_inode(cmount, ino, &inode));
+    } else if (ino == CEPH_INO_ROOT || ino == CEPH_INO_GLOBAL_SNAPREALM) {
+      ASSERT_EQ(0, ceph_ll_lookup_inode(cmount, ino, &inode));
+      ceph_ll_put(cmount, inode);
+    } else if (ino == CEPH_INO_LOST_AND_FOUND) {
+      // the ino 3 will only exists after the recovery tool ran, so
+      // it may return -CEPHFS_ESTALE with a fresh fs cluster
+      int r = ceph_ll_lookup_inode(cmount, ino, &inode);
+      if (r == 0) {
+        ceph_ll_put(cmount, inode);
+      } else {
+        ASSERT_TRUE(r == -CEPHFS_ESTALE);
+      }
+    } else {
+      // currently the ino 0 and 4~99 is not useded yet.
+      ASSERT_EQ(-CEPHFS_ESTALE, ceph_ll_lookup_inode(cmount, ino, &inode));
+    }
+  }
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SetMountTimeoutPostMount) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  ASSERT_EQ(-CEPHFS_EINVAL, ceph_set_mount_timeout(cmount, 5));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SetMountTimeout) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_set_mount_timeout(cmount, 5));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, FsCrypt) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char test_xattr_file[NAME_MAX];
+  sprintf(test_xattr_file, "test_fscrypt_%d", getpid());
+  int fd = ceph_open(cmount, test_xattr_file, O_RDWR|O_CREAT, 0666);
+  ASSERT_GT(fd, 0);
+
+  ASSERT_EQ(0, ceph_fsetxattr(cmount, fd, "ceph.fscrypt.auth", "foo", 3, CEPH_XATTR_CREATE));
+  ASSERT_EQ(0, ceph_fsetxattr(cmount, fd, "ceph.fscrypt.file", "foo", 3, CEPH_XATTR_CREATE));
+
+  char buf[64];
+  ASSERT_EQ(3, ceph_fgetxattr(cmount, fd, "ceph.fscrypt.auth", buf, sizeof(buf)));
+  ASSERT_EQ(3, ceph_fgetxattr(cmount, fd, "ceph.fscrypt.file", buf, sizeof(buf)));
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ASSERT_EQ(0, ceph_mount(cmount, NULL));
+
+  fd = ceph_open(cmount, test_xattr_file, O_RDWR, 0666);
+  ASSERT_GT(fd, 0);
+  ASSERT_EQ(3, ceph_fgetxattr(cmount, fd, "ceph.fscrypt.auth", buf, sizeof(buf)));
+  ASSERT_EQ(3, ceph_fgetxattr(cmount, fd, "ceph.fscrypt.file", buf, sizeof(buf)));
+
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SnapdirAttrs) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  char snap_dir_path[512];
+
+  pid_t mypid = getpid();
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  sprintf(snap_dir_path, "%s/.snap", dir_path);
+
+  Inode *dir, *root;
+  struct ceph_statx stx_dir;
+  struct ceph_statx stx_snap_dir;
+  struct ceph_statx stx_root_snap_dir;
+  UserPerm *perms = ceph_mount_perms(cmount);
+
+  ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
+  ASSERT_EQ(ceph_ll_mkdir(cmount, root, dir_name, 0755, &dir, &stx_dir, 0, 0, perms), 0);
+
+  ASSERT_EQ(ceph_statx(cmount, dir_path, &stx_dir,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME|CEPH_STATX_MODE|CEPH_STATX_MODE|CEPH_STATX_GID|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME|CEPH_STATX_MODE|CEPH_STATX_MODE|CEPH_STATX_GID|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(ceph_statx(cmount, "/.snap", &stx_root_snap_dir,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME|CEPH_STATX_MODE|CEPH_STATX_MODE|CEPH_STATX_GID|CEPH_STATX_VERSION, 0), 0);
+
+  // these should match the parent directories attrs
+  ASSERT_EQ(stx_dir.stx_mode, stx_snap_dir.stx_mode);
+  ASSERT_EQ(stx_dir.stx_uid, stx_snap_dir.stx_uid);
+  ASSERT_EQ(stx_dir.stx_gid, stx_snap_dir.stx_gid);
+  ASSERT_EQ(utime_t(stx_dir.stx_atime), utime_t(stx_snap_dir.stx_atime));
+  // these should match the closest snaprealm ancestor (root in this
+  // case) attrs
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_mtime), utime_t(stx_snap_dir.stx_mtime));
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_ctime), utime_t(stx_snap_dir.stx_ctime));
+  ASSERT_EQ(stx_root_snap_dir.stx_version, stx_snap_dir.stx_version);
+
+  // chown  -- for this we need to be "root"
+  UserPerm *rootcred = ceph_userperm_new(0, 0, 0, NULL);
+  ASSERT_TRUE(rootcred);
+  stx_dir.stx_uid++;
+  stx_dir.stx_gid++;
+  ASSERT_EQ(ceph_ll_setattr(cmount, dir, &stx_dir, CEPH_SETATTR_UID|CEPH_SETATTR_GID, rootcred), 0);
+
+  memset(&stx_dir, 0, sizeof(stx_dir));
+  memset(&stx_snap_dir, 0, sizeof(stx_snap_dir));
+
+  ASSERT_EQ(ceph_statx(cmount, dir_path, &stx_dir,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME|CEPH_STATX_MODE|CEPH_STATX_MODE|CEPH_STATX_GID|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir,
+                       CEPH_STATX_MTIME|CEPH_STATX_ATIME|CEPH_STATX_MODE|CEPH_STATX_MODE|CEPH_STATX_GID|CEPH_STATX_VERSION, 0), 0);
+
+  ASSERT_EQ(stx_dir.stx_mode, stx_snap_dir.stx_mode);
+  ASSERT_EQ(stx_dir.stx_uid, stx_snap_dir.stx_uid);
+  ASSERT_EQ(stx_dir.stx_gid, stx_snap_dir.stx_gid);
+  ASSERT_EQ(utime_t(stx_dir.stx_atime), utime_t(stx_snap_dir.stx_atime));
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_mtime), utime_t(stx_snap_dir.stx_mtime));
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_ctime), utime_t(stx_snap_dir.stx_ctime));
+  ASSERT_EQ(stx_root_snap_dir.stx_version, stx_snap_dir.stx_version);
+
+  ASSERT_EQ(ceph_ll_rmdir(cmount, root, dir_name, rootcred), 0);
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SnapdirAttrsOnSnapCreate) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  char snap_dir_path[512];
+
+  pid_t mypid = getpid();
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  sprintf(snap_dir_path, "%s/.snap", dir_path);
+
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  struct ceph_statx stx_dir;
+  struct ceph_statx stx_snap_dir;
+  struct ceph_statx stx_root_snap_dir;
+  ASSERT_EQ(ceph_statx(cmount, dir_path, &stx_dir, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(ceph_statx(cmount, "/.snap", &stx_root_snap_dir, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_mtime), utime_t(stx_snap_dir.stx_mtime));
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_ctime), utime_t(stx_snap_dir.stx_ctime));
+  ASSERT_EQ(stx_root_snap_dir.stx_version, stx_snap_dir.stx_version);
+
+  char snap_path[1024];
+  sprintf(snap_path, "%s/snap_a", snap_dir_path);
+  ASSERT_EQ(ceph_mkdir(cmount, snap_path, 0777), 0);
+
+  struct ceph_statx stx_snap_dir_1;
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir_1, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_LT(utime_t(stx_root_snap_dir.stx_mtime), utime_t(stx_snap_dir_1.stx_mtime));
+  ASSERT_LT(utime_t(stx_root_snap_dir.stx_ctime), utime_t(stx_snap_dir_1.stx_ctime));
+  ASSERT_LT(stx_root_snap_dir.stx_version, stx_snap_dir_1.stx_version);
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, snap_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ceph_shutdown(cmount);
+}
+
+
+TEST(LibCephFS, SnapdirAttrsOnSnapDelete) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  char snap_dir_path[512];
+
+  pid_t mypid = getpid();
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  sprintf(snap_dir_path, "%s/.snap", dir_path);
+
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  struct ceph_statx stx_dir;
+  struct ceph_statx stx_snap_dir;
+  struct ceph_statx stx_root_snap_dir;
+  ASSERT_EQ(ceph_statx(cmount, dir_path, &stx_dir, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(ceph_statx(cmount, "/.snap", &stx_root_snap_dir, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_mtime), utime_t(stx_snap_dir.stx_mtime));
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_ctime), utime_t(stx_snap_dir.stx_mtime));
+  ASSERT_EQ(stx_root_snap_dir.stx_version, stx_snap_dir.stx_version);
+
+  char snap_path[1024];
+  sprintf(snap_path, "%s/snap_a", snap_dir_path);
+  ASSERT_EQ(ceph_mkdir(cmount, snap_path, 0777), 0);
+
+  struct ceph_statx stx_snap_dir_1;
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir_1, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_LT(utime_t(stx_root_snap_dir.stx_mtime), utime_t(stx_snap_dir_1.stx_mtime));
+  ASSERT_LT(utime_t(stx_root_snap_dir.stx_ctime), utime_t(stx_snap_dir_1.stx_ctime));
+  ASSERT_LT(stx_root_snap_dir.stx_version, stx_snap_dir_1.stx_version);
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, snap_path));
+
+  struct ceph_statx stx_snap_dir_2;
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir_2, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_LT(utime_t(stx_snap_dir_1.stx_mtime), utime_t(stx_snap_dir_2.stx_mtime));
+  ASSERT_LT(utime_t(stx_snap_dir_1.stx_ctime), utime_t(stx_snap_dir_2.stx_ctime));
+  ASSERT_LT(stx_snap_dir_1.stx_version, stx_snap_dir_2.stx_version);
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, SnapdirAttrsOnSnapRename) {
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(ceph_create(&cmount, NULL), 0);
+  ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(ceph_mount(cmount, NULL), 0);
+
+  char dir_name[128];
+  char dir_path[256];
+  char snap_dir_path[512];
+
+  pid_t mypid = getpid();
+  sprintf(dir_name, "dir_%d", mypid);
+  sprintf(dir_path, "/%s", dir_name);
+  sprintf(snap_dir_path, "%s/.snap", dir_path);
+
+  ASSERT_EQ(ceph_mkdir(cmount, dir_path, 0777), 0);
+
+  struct ceph_statx stx_dir;
+  struct ceph_statx stx_snap_dir;
+  struct ceph_statx stx_root_snap_dir;
+  ASSERT_EQ(ceph_statx(cmount, dir_path, &stx_dir, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(ceph_statx(cmount, "/.snap", &stx_root_snap_dir, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_mtime), utime_t(stx_snap_dir.stx_mtime));
+  ASSERT_EQ(utime_t(stx_root_snap_dir.stx_ctime), utime_t(stx_snap_dir.stx_ctime));
+  ASSERT_EQ(stx_root_snap_dir.stx_version, stx_snap_dir.stx_version);
+
+  char snap_path[1024];
+  sprintf(snap_path, "%s/snap_a", snap_dir_path);
+  ASSERT_EQ(ceph_mkdir(cmount, snap_path, 0777), 0);
+
+  struct ceph_statx stx_snap_dir_1;
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir_1, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_LT(utime_t(stx_root_snap_dir.stx_mtime), utime_t(stx_snap_dir_1.stx_mtime));
+  ASSERT_LT(utime_t(stx_root_snap_dir.stx_ctime), utime_t(stx_snap_dir_1.stx_ctime));
+  ASSERT_LT(stx_root_snap_dir.stx_version, stx_snap_dir_1.stx_version);
+
+  char snap_path_r[1024];
+  sprintf(snap_path_r, "%s/snap_b", snap_dir_path);
+  ASSERT_EQ(ceph_rename(cmount, snap_path, snap_path_r), 0);
+
+  struct ceph_statx stx_snap_dir_2;
+  ASSERT_EQ(ceph_statx(cmount, snap_dir_path, &stx_snap_dir_2, CEPH_STATX_MTIME|CEPH_STATX_CTIME|CEPH_STATX_VERSION, 0), 0);
+  ASSERT_LT(utime_t(stx_snap_dir_1.stx_mtime), utime_t(stx_snap_dir_2.stx_mtime));
+  ASSERT_LT(utime_t(stx_snap_dir_1.stx_ctime), utime_t(stx_snap_dir_2.stx_ctime));
+  ASSERT_LT(stx_snap_dir_1.stx_version, stx_snap_dir_2.stx_version);
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, snap_path_r));
+  ASSERT_EQ(0, ceph_rmdir(cmount, dir_path));
+  ASSERT_EQ(0, ceph_unmount(cmount));
+  ceph_shutdown(cmount);
+}
diff --git a/src/test/libcephfs/vxattr.cc b/src/test/libcephfs/vxattr.cc
new file mode 100644
index 000000000..4d9eaf5e4
--- /dev/null
+++ b/src/test/libcephfs/vxattr.cc
@@ -0,0 +1,385 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2021 Red Hat Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/compat.h"
+#include "gtest/gtest.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include "mds/mdstypes.h"
+#include "include/stat.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <sys/uio.h>
+#include <sys/time.h>
+#include <string.h>
+
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+#include "common/Clock.h"
+#include "common/ceph_json.h"
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#endif
+
+#include <fmt/format.h>
+#include <map>
+#include <vector>
+#include <thread>
+#include <regex>
+#include <string>
+
+using namespace std;
+
+TEST(LibCephFS, LayoutVerifyDefaultLayout) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, "test/d0/subdir", 0777));
+
+  {
+    char value[1024] = "";
+    int r = 0;
+
+    // check for default layout
+    r = ceph_getxattr(cmount, "/", "ceph.dir.layout.json", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    std::clog << "layout:" << value << std::endl;
+    ASSERT_STRNE((char*)NULL, strstr(value, "\"inheritance\": \"@default\""));
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0/subdir"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test"));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LayoutSetAndVerifyNewAndInheritedLayout) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, "test/d0/subdir", 0777));
+
+  std::string pool_name_set;
+
+  {
+    char value[1024] = "";
+    int r = 0;
+
+    r = ceph_getxattr(cmount, "/", "ceph.dir.layout.json", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+
+    JSONParser json_parser;
+    ASSERT_EQ(json_parser.parse(value, r), 1);
+    ASSERT_EQ(json_parser.is_object(), 1);
+
+    std::string pool_name;
+
+    JSONDecoder::decode_json("pool_name", pool_name, &json_parser, true);
+
+    pool_name_set = pool_name;
+
+    // set a new layout
+    std::string new_layout;
+    new_layout += "{";
+    new_layout += "\"stripe_unit\": 65536, ";
+    new_layout += "\"stripe_count\": 1, ";
+    new_layout += "\"object_size\": 65536, ";
+    new_layout += "\"pool_name\": \"" + pool_name + "\"";
+    new_layout +=  "}";
+
+    ASSERT_EQ(0, ceph_setxattr(cmount, "test/d0", "ceph.dir.layout.json", (void*)new_layout.c_str(), new_layout.length(), XATTR_CREATE));
+  }
+
+  {
+    char value[1024] = "";
+    int r = 0;
+
+    r = ceph_getxattr(cmount, "test/d0", "ceph.dir.layout.json", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    std::clog << "layout:" << value << std::endl;
+
+    JSONParser json_parser;
+    ASSERT_EQ(json_parser.parse(value, r), 1);
+    ASSERT_EQ(json_parser.is_object(), 1);
+
+    int64_t object_size;
+    int64_t stripe_unit;
+    int64_t stripe_count;
+    std::string pool_name;
+    std::string inheritance;
+
+    JSONDecoder::decode_json("pool_name", pool_name, &json_parser, true);
+    JSONDecoder::decode_json("object_size", object_size, &json_parser, true);
+    JSONDecoder::decode_json("stripe_unit", stripe_unit, &json_parser, true);
+    JSONDecoder::decode_json("stripe_count", stripe_count, &json_parser, true);
+    JSONDecoder::decode_json("inheritance", inheritance, &json_parser, true);
+
+    // now verify the layout
+    ASSERT_EQ(pool_name.compare(pool_name_set), 0);
+    ASSERT_EQ(object_size, 65536);
+    ASSERT_EQ(stripe_unit, 65536);
+    ASSERT_EQ(stripe_count, 1);
+    ASSERT_EQ(inheritance.compare("@set"), 0);
+  }
+
+  {
+    char value[1024] = "";
+    int r = 0;
+
+    JSONParser json_parser;
+    std::string inheritance;
+
+    // now check that the subdir layout is inherited
+    r = ceph_getxattr(cmount, "test/d0/subdir", "ceph.dir.layout.json", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    std::clog << "layout:" << value << std::endl;
+    ASSERT_EQ(json_parser.parse(value, r), 1);
+    ASSERT_EQ(json_parser.is_object(), 1);
+    JSONDecoder::decode_json("inheritance", inheritance, &json_parser, true);
+    ASSERT_EQ(inheritance.compare("@inherited"), 0);
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0/subdir"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test"));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LayoutSetBadJSON) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, "test/d0/subdir", 0777));
+
+  {
+    // set a new layout and verify the same
+    const char *new_layout = "" // bad json without starting brace
+      "\"stripe_unit\": 65536, "
+      "\"stripe_count\": 1, "
+      "\"object_size\": 65536, "
+      "\"pool_name\": \"cephfs.a.data\", "
+      "}";
+    // try to set a malformed JSON, eg. without an open brace
+    ASSERT_EQ(-CEPHFS_EINVAL, ceph_setxattr(cmount, "test/d0", "ceph.dir.layout.json", (void*)new_layout, strlen(new_layout), XATTR_CREATE));
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0/subdir"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test"));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LayoutSetBadPoolName) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, "test/d0/subdir", 0777));
+
+  {
+    // try setting a bad pool name
+    ASSERT_EQ(-CEPHFS_EINVAL, ceph_setxattr(cmount, "test/d0", "ceph.dir.layout.pool_name", (void*)"UglyPoolName", 12, XATTR_CREATE));
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0/subdir"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test"));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LayoutSetBadPoolId) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, "test/d0/subdir", 0777));
+
+  {
+    // try setting a bad pool id
+    ASSERT_EQ(-CEPHFS_EINVAL, ceph_setxattr(cmount, "test/d0", "ceph.dir.layout.pool_id", (void*)"300", 3, XATTR_CREATE));
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0/subdir"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test"));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, LayoutSetInvalidFieldName) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, "test/d0/subdir", 0777));
+
+  {
+    // try to set in invalid field
+    ASSERT_EQ(-CEPHFS_ENODATA, ceph_setxattr(cmount, "test/d0", "ceph.dir.layout.bad_field", (void*)"300", 3, XATTR_CREATE));
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0/subdir"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d0"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test"));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetAndSetDirPin) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, "test/d1", 0777));
+
+  {
+    char value[1024] = "";
+    int r = ceph_getxattr(cmount, "test/d1", "ceph.dir.pin", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    ASSERT_STREQ("-1", value);
+  }
+
+  {
+    char value[1024] = "";
+    int r = -1;
+
+    ASSERT_EQ(0, ceph_setxattr(cmount, "test/d1", "ceph.dir.pin", (void*)"1", 1, XATTR_CREATE));
+
+    r = ceph_getxattr(cmount, "test/d1", "ceph.dir.pin", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    ASSERT_STREQ("1", value);
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d1"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test"));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetAndSetDirDistribution) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, "test/d2", 0777));
+
+  {
+    char value[1024] = "";
+    int r = ceph_getxattr(cmount, "test/d2", "ceph.dir.pin.distributed", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    ASSERT_STREQ("0", value);
+  }
+
+  {
+    char value[1024] = "";
+    int r = -1;
+
+    ASSERT_EQ(0, ceph_setxattr(cmount, "test/d2", "ceph.dir.pin.distributed", (void*)"1", 1, XATTR_CREATE));
+
+    r = ceph_getxattr(cmount, "test/d2", "ceph.dir.pin.distributed", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    ASSERT_STREQ("1", value);
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d2"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test"));
+
+  ceph_shutdown(cmount);
+}
+
+TEST(LibCephFS, GetAndSetDirRandom) {
+
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, "test/d3", 0777));
+
+  {
+    char value[1024] = "";
+    int r = ceph_getxattr(cmount, "test/d3", "ceph.dir.pin.random", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    ASSERT_STREQ("0", value);
+  }
+
+  {
+    double val = (double)1.0/(double)128.0;
+    std::stringstream ss;
+    ss << val;
+    ASSERT_EQ(0, ceph_setxattr(cmount, "test/d3", "ceph.dir.pin.random", (void*)ss.str().c_str(), strlen(ss.str().c_str()), XATTR_CREATE));
+
+    char value[1024] = "";
+    int r = -1;
+
+    r = ceph_getxattr(cmount, "test/d3", "ceph.dir.pin.random", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    ASSERT_STREQ(ss.str().c_str(), value);
+  }
+
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test/d3"));
+  ASSERT_EQ(0, ceph_rmdir(cmount, "test"));
+
+  ceph_shutdown(cmount);
+}