Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 493 insertions, 0 deletions

diff --git a/src/test/os/TestLFNIndex.cc b/src/test/os/TestLFNIndex.cc
new file mode 100644
index 000000000..d74de739d
--- /dev/null
+++ b/src/test/os/TestLFNIndex.cc
@@ -0,0 +1,493 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Library Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Library Public License for more details.
+ *
+ */
+
+#include <stdio.h>
+#include <signal.h>
+#include "os/filestore/LFNIndex.h"
+#include "os/filestore/chain_xattr.h"
+#include "common/ceph_argparse.h"
+#include "global/global_init.h"
+#include <gtest/gtest.h>
+
+class TestWrapLFNIndex : public LFNIndex {
+public:
+  TestWrapLFNIndex(CephContext* cct,
+		   coll_t collection,
+		   const char *base_path,
+		   uint32_t index_version)
+    : LFNIndex(cct, collection, base_path, index_version) {}
+
+  uint32_t collection_version() override {
+    return index_version;
+  }
+
+  int cleanup() override { return 0; }
+
+  int _split(
+		     uint32_t match,                           
+		     uint32_t bits,                            
+		     CollectionIndex* dest
+		     ) override { return 0; }
+  int _merge(
+		     uint32_t bits,
+		     CollectionIndex* dest
+		     ) override { return 0; }
+
+  void test_generate_and_parse(const ghobject_t &hoid, const std::string &mangled_expected) {
+    const std::string mangled_name = lfn_generate_object_name(hoid);
+    EXPECT_EQ(mangled_expected, mangled_name);
+    ghobject_t hoid_parsed;
+    EXPECT_EQ(0, lfn_parse_object_name(mangled_name, &hoid_parsed));
+    EXPECT_EQ(hoid, hoid_parsed);
+  }
+
+protected:
+  int _init() override { return 0; }
+
+  int _created(
+		       const vector<string> &path,
+		       const ghobject_t &hoid,
+		       const string &mangled_name 
+		       ) override { return 0; }
+
+  int _remove(
+		      const vector<string> &path,
+		      const ghobject_t &hoid,
+		      const string &mangled_name
+		      ) override { return 0; }
+
+  int _lookup(
+		      const ghobject_t &hoid,
+		      vector<string> *path,
+		      string *mangled_name,
+		      int *exists		 
+		      ) override { return 0; }
+
+  int _collection_list_partial(
+				       const ghobject_t &start,
+				       const ghobject_t &end,
+				       int max_count,
+				       vector<ghobject_t> *ls,
+				       ghobject_t *next
+				       ) override { return 0; }
+  int _pre_hash_collection(
+                                   uint32_t pg_num,
+                                   uint64_t expected_num_objs
+                                  ) override { return 0; }
+
+};
+
+class TestHASH_INDEX_TAG : public TestWrapLFNIndex, public ::testing::Test {
+public:
+  TestHASH_INDEX_TAG()
+    : TestWrapLFNIndex(g_ceph_context, coll_t(), "PATH_1",
+		       CollectionIndex::HASH_INDEX_TAG) {
+  }
+};
+
+TEST_F(TestHASH_INDEX_TAG, generate_and_parse_name) {
+  const vector<string> path;
+  const std::string key;
+  uint64_t hash = 0xABABABAB;
+  uint64_t pool = -1;
+
+  test_generate_and_parse(ghobject_t(hobject_t(object_t(".A/B_\\C.D"), key, CEPH_NOSNAP, hash, pool, "")),
+			  "\\.A\\sB_\\\\C.D_head_ABABABAB");
+  test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")),
+			  "\\dA_head_ABABABAB");
+}
+
+class TestHASH_INDEX_TAG_2 : public TestWrapLFNIndex, public ::testing::Test {
+public:
+  TestHASH_INDEX_TAG_2()
+    : TestWrapLFNIndex(g_ceph_context,
+		       coll_t(), "PATH_1", CollectionIndex::HASH_INDEX_TAG_2) {
+  }
+};
+
+TEST_F(TestHASH_INDEX_TAG_2, generate_and_parse_name) {
+  const vector<string> path;
+  const std::string key("KEY");
+  uint64_t hash = 0xABABABAB;
+  uint64_t pool = -1;
+
+  {
+    std::string name(".XA/B_\\C.D");
+    name[1] = '\0';
+    ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""));
+
+    test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB");
+  }
+  test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")),
+			  "\\dA_KEY_head_ABABABAB");
+}
+
+class TestHOBJECT_WITH_POOL : public TestWrapLFNIndex, public ::testing::Test {
+public:
+  TestHOBJECT_WITH_POOL()
+    : TestWrapLFNIndex(g_ceph_context, coll_t(),
+		       "PATH_1", CollectionIndex::HOBJECT_WITH_POOL) {
+  }
+};
+
+TEST_F(TestHOBJECT_WITH_POOL, generate_and_parse_name) {
+  const vector<string> path;
+  const std::string key("KEY");
+  uint64_t hash = 0xABABABAB;
+  uint64_t pool = 0xCDCDCDCD;
+  int64_t gen = 0xefefefefef;
+  shard_id_t shard_id(0xb);
+
+  {
+    std::string name(".XA/B_\\C.D");
+    name[1] = '\0';
+    ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""));
+    hoid.hobj.nspace = "NSPACE";
+
+    test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd");
+  }
+  {
+    ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, ""));
+    hoid.hobj.nspace = "NSPACE";
+
+    test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd");
+  }
+  {
+    std::string name(".XA/B_\\C.D");
+    name[1] = '\0';
+    ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id);
+    hoid.hobj.nspace = "NSPACE";
+
+    test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b");
+  }
+  {
+    ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id);
+    hoid.hobj.nspace = "NSPACE";
+
+    test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b");
+  }
+}
+
+class TestLFNIndex : public TestWrapLFNIndex, public ::testing::Test {
+public:
+  TestLFNIndex()
+    : TestWrapLFNIndex(g_ceph_context, coll_t(), "PATH_1",
+		       CollectionIndex::HOBJECT_WITH_POOL) {
+  }
+
+  void SetUp() override {
+    ::chmod("PATH_1", 0700);
+    ASSERT_EQ(0, ::system("rm -fr PATH_1"));
+    ASSERT_EQ(0, ::mkdir("PATH_1", 0700));
+  }
+
+  void TearDown() override {
+    ASSERT_EQ(0, ::system("rm -fr PATH_1"));
+  }
+};
+
+TEST_F(TestLFNIndex, remove_object) {
+  const vector<string> path;
+
+  //
+  // small object name removal
+  //
+  {
+    std::string mangled_name;
+    int exists = 666;
+    ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP)));
+
+    EXPECT_EQ(0, ::chmod("PATH_1", 0000));
+    if (getuid() != 0) {
+      EXPECT_EQ(-EACCES, remove_object(path, hoid));
+    }
+    EXPECT_EQ(0, ::chmod("PATH_1", 0700));
+    EXPECT_EQ(-ENOENT, remove_object(path, hoid));
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    const std::string pathname("PATH_1/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, remove_object(path, hoid));
+    EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+  }
+  //
+  // long object name removal of a single file
+  //
+  {
+    std::string mangled_name;
+    int exists;
+    const std::string object_name(1024, 'A');
+    ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
+
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_EQ(0, exists);
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    std::string pathname("PATH_1/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, created(hoid, pathname.c_str()));
+
+    EXPECT_EQ(0, remove_object(path, hoid));
+    EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+  }
+
+  //
+  // long object name removal of the last file 
+  //
+  {
+    std::string mangled_name;
+    int exists;
+    const std::string object_name(1024, 'A');
+    ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
+
+    //
+    //   PATH_1/AAA..._0_long => does not match long object name
+    //
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_EQ(0, exists);
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    std::string pathname("PATH_1/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, created(hoid, pathname.c_str()));
+    string LFN_ATTR = "user.cephos.lfn";
+    if (index_version != HASH_INDEX_TAG) {
+      char buf[100];
+      snprintf(buf, sizeof(buf), "%d", index_version);
+      LFN_ATTR += string(buf);
+    }
+    const std::string object_name_1 = object_name + "SUFFIX";
+    EXPECT_EQ(object_name_1.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_1.c_str(), object_name_1.size()));
+
+    //
+    //   PATH_1/AAA..._1_long => matches long object name
+    //
+    std::string mangled_name_1;
+    exists = 666;
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_1, &exists));
+    EXPECT_NE(std::string::npos, mangled_name_1.find("1_long"));
+    EXPECT_EQ(0, exists);
+    std::string pathname_1("PATH_1/" + mangled_name_1);
+    auto retvalue = ::creat(pathname_1.c_str(), 0600);
+    ceph_assert(retvalue > 2);
+    EXPECT_EQ(0, ::close(retvalue));
+    EXPECT_EQ(0, created(hoid, pathname_1.c_str()));
+
+    //
+    // remove_object skips PATH_1/AAA..._0_long and removes PATH_1/AAA..._1_long
+    //
+    EXPECT_EQ(0, remove_object(path, hoid));
+    EXPECT_EQ(0, ::access(pathname.c_str(), 0));
+    EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+    EXPECT_EQ(0, ::unlink(pathname.c_str()));
+  }
+
+  //
+  // long object name removal of a file in the middle of the list
+  //
+  {
+    std::string mangled_name;
+    int exists;
+    const std::string object_name(1024, 'A');
+    ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
+
+    //
+    //   PATH_1/AAA..._0_long => matches long object name
+    //
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_EQ(0, exists);
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    std::string pathname("PATH_1/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, created(hoid, pathname.c_str()));
+    //
+    //   PATH_1/AAA..._1_long => matches long object name
+    //
+    std::string mangled_name_1 = mangled_name;
+    mangled_name_1.replace(mangled_name_1.find("0_long"), 6, "1_long");
+    const std::string pathname_1("PATH_1/" + mangled_name_1);
+    const std::string cmd("cp -a " + pathname + " " + pathname_1);
+    EXPECT_EQ(0, ::system(cmd.c_str()));
+    const string ATTR = "user.MARK";
+    EXPECT_EQ((unsigned)1, (unsigned)chain_setxattr(pathname_1.c_str(), ATTR.c_str(), "Y", 1));
+
+    //
+    // remove_object replaces the file to be removed with the last from the
+    // collision list. In this case it replaces
+    //    PATH_1/AAA..._0_long
+    // with
+    //    PATH_1/AAA..._1_long
+    //
+    EXPECT_EQ(0, remove_object(path, hoid));
+    EXPECT_EQ(0, ::access(pathname.c_str(), 0));
+    char buffer[1] = { 0, };
+    EXPECT_EQ((unsigned)1, (unsigned)chain_getxattr(pathname.c_str(), ATTR.c_str(), buffer, 1));
+    EXPECT_EQ('Y', buffer[0]);
+    EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+  }
+}
+
+TEST_F(TestLFNIndex, get_mangled_name) {
+  const vector<string> path;
+
+  //
+  // small object name
+  //
+  {
+    std::string mangled_name;
+    int exists = 666;
+    ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP)));
+
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("ABC__head"));
+    EXPECT_EQ(std::string::npos, mangled_name.find("0_long"));
+    EXPECT_EQ(0, exists);
+    const std::string pathname("PATH_1/" + mangled_name);
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("ABC__head"));
+    EXPECT_EQ(1, exists);
+    EXPECT_EQ(0, ::unlink(pathname.c_str()));
+  }
+  //
+  // long object name
+  //
+  {
+    std::string mangled_name;
+    int exists;
+    const std::string object_name(1024, 'A');
+    ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
+
+    //
+    // long version of the mangled name and no matching
+    // file exists 
+    //
+    mangled_name.clear();
+    exists = 666;
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    EXPECT_EQ(0, exists);
+
+    const std::string pathname("PATH_1/" + mangled_name);
+
+    //
+    // if a file by the same name exists but does not have the
+    // expected extended attribute, it is silently removed 
+    //
+    mangled_name.clear();
+    exists = 666;
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    EXPECT_EQ(0, exists);
+    EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
+    EXPECT_EQ(ENOENT, errno);
+
+    //
+    // if a file by the same name exists but does not have the
+    // expected extended attribute, and cannot be removed, 
+    // return on error
+    //
+    mangled_name.clear();
+    exists = 666;
+    EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
+    EXPECT_EQ(0, ::chmod("PATH_1", 0500));
+    if (getuid() != 0) {
+      EXPECT_EQ(-EACCES, get_mangled_name(path, hoid, &mangled_name, &exists));
+    }
+    EXPECT_EQ("", mangled_name);
+    EXPECT_EQ(666, exists);
+    EXPECT_EQ(0, ::chmod("PATH_1", 0700));
+    EXPECT_EQ(0, ::unlink(pathname.c_str()));
+
+    //
+    // long version of the mangled name and a file
+    // exists by that name and contains the long object name
+    //
+    mangled_name.clear();
+    exists = 666;
+    auto retvalue = ::creat(pathname.c_str(), 0600);
+    ceph_assert(retvalue > 2);
+    EXPECT_EQ(0, ::close(retvalue));
+    EXPECT_EQ(0, created(hoid, pathname.c_str()));
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
+    EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
+    EXPECT_EQ(1, exists);
+    EXPECT_EQ(0, ::access(pathname.c_str(), 0));
+
+    //
+    // long version of the mangled name and a file exists by that name
+    // and contains a long object name with the same prefix but they
+    // are not identical and it so happens that their SHA1 is
+    // identical : a collision number is used to differentiate them
+    //
+    string LFN_ATTR = "user.cephos.lfn";
+    if (index_version != HASH_INDEX_TAG) {
+      char buf[100];
+      snprintf(buf, sizeof(buf), "%d", index_version);
+      LFN_ATTR += string(buf);
+    }
+    const std::string object_name_same_prefix = object_name + "SUFFIX";
+    EXPECT_EQ(object_name_same_prefix.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_same_prefix.c_str(), object_name_same_prefix.size()));
+    std::string mangled_name_same_prefix;
+    exists = 666;
+    EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_same_prefix, &exists));
+    EXPECT_NE(std::string::npos, mangled_name_same_prefix.find("1_long"));
+    EXPECT_EQ(0, exists);
+    
+    EXPECT_EQ(0, ::unlink(pathname.c_str()));
+  }
+}
+
+int main(int argc, char **argv) {
+  int fd = ::creat("detect", 0600);
+  if (fd < 0){
+    cerr << "failed to create file detect" << std::endl;
+    return EXIT_FAILURE;
+  }
+  int ret = chain_fsetxattr(fd, "user.test", "A", 1);
+  ::close(fd);
+  ::unlink("detect");
+  if (ret < 0) {
+    cerr << "SKIP LFNIndex because unable to test for xattr" << std::endl;
+  } else {
+    vector<const char*> args;
+    argv_to_vec(argc, (const char **)argv, args);
+
+    auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
+			   CODE_ENVIRONMENT_UTILITY,
+			   CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+    common_init_finish(g_ceph_context);
+
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+  }
+}
+
+/* 
+ * Local Variables:
+ * compile-command: "cd ../.. ; 
+ *   make unittest_lfnindex && 
+ *   valgrind --tool=memcheck ./unittest_lfnindex \
+ *   # --gtest_filter=TestLFNIndex.* --log-to-stderr=true --debug-filestore=20"
+ * End:
+ */