summaryrefslogtreecommitdiffstats
path: root/src/test/cls_rgw
diff options
context:
space:
mode:
Diffstat (limited to 'src/test/cls_rgw')
-rw-r--r--src/test/cls_rgw/CMakeLists.txt24
-rw-r--r--src/test/cls_rgw/test_cls_rgw.cc1342
-rw-r--r--src/test/cls_rgw/test_cls_rgw_stats.cc646
3 files changed, 2012 insertions, 0 deletions
diff --git a/src/test/cls_rgw/CMakeLists.txt b/src/test/cls_rgw/CMakeLists.txt
new file mode 100644
index 000000000..67b8beb6c
--- /dev/null
+++ b/src/test/cls_rgw/CMakeLists.txt
@@ -0,0 +1,24 @@
+if(${WITH_RADOSGW})
+ add_executable(ceph_test_cls_rgw
+ test_cls_rgw.cc
+ )
+ target_link_libraries(ceph_test_cls_rgw
+ cls_rgw_client
+ librados
+ global
+ ${UNITTEST_LIBS}
+ ${EXTRALIBS}
+ ${BLKID_LIBRARIES}
+ ${CMAKE_DL_LIBS}
+ radostest-cxx)
+ install(TARGETS
+ ceph_test_cls_rgw
+ DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+ add_executable(ceph_test_cls_rgw_stats test_cls_rgw_stats.cc
+ $<TARGET_OBJECTS:unit-main>)
+ target_link_libraries(ceph_test_cls_rgw_stats cls_rgw_client global
+ librados ${UNITTEST_LIBS} radostest-cxx)
+ install(TARGETS ceph_test_cls_rgw_stats DESTINATION ${CMAKE_INSTALL_BINDIR})
+endif(${WITH_RADOSGW})
+
diff --git a/src/test/cls_rgw/test_cls_rgw.cc b/src/test/cls_rgw/test_cls_rgw.cc
new file mode 100644
index 000000000..bf60dfdd0
--- /dev/null
+++ b/src/test/cls_rgw/test_cls_rgw.cc
@@ -0,0 +1,1342 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/types.h"
+#include "cls/rgw/cls_rgw_client.h"
+#include "cls/rgw/cls_rgw_ops.h"
+
+#include "gtest/gtest.h"
+#include "test/librados/test_cxx.h"
+#include "global/global_context.h"
+#include "common/ceph_context.h"
+
+#include <errno.h>
+#include <string>
+#include <vector>
+#include <map>
+#include <set>
+
+using namespace std;
+using namespace librados;
+
+// creates a temporary pool and initializes an IoCtx shared by all tests
+class cls_rgw : public ::testing::Test {
+ static librados::Rados rados;
+ static std::string pool_name;
+ protected:
+ static librados::IoCtx ioctx;
+
+ static void SetUpTestCase() {
+ pool_name = get_temp_pool_name();
+ /* create pool */
+ ASSERT_EQ("", create_one_pool_pp(pool_name, rados));
+ ASSERT_EQ(0, rados.ioctx_create(pool_name.c_str(), ioctx));
+ }
+ static void TearDownTestCase() {
+ /* remove pool */
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(pool_name, rados));
+ }
+};
+librados::Rados cls_rgw::rados;
+std::string cls_rgw::pool_name;
+librados::IoCtx cls_rgw::ioctx;
+
+
+string str_int(string s, int i)
+{
+ char buf[32];
+ snprintf(buf, sizeof(buf), "-%d", i);
+ s.append(buf);
+
+ return s;
+}
+
+void test_stats(librados::IoCtx& ioctx, string& oid, RGWObjCategory category, uint64_t num_entries, uint64_t total_size)
+{
+ map<int, struct rgw_cls_list_ret> results;
+ map<int, string> oids;
+ oids[0] = oid;
+ ASSERT_EQ(0, CLSRGWIssueGetDirHeader(ioctx, oids, results, 8)());
+
+ uint64_t entries = 0;
+ uint64_t size = 0;
+ map<int, struct rgw_cls_list_ret>::iterator iter = results.begin();
+ for (; iter != results.end(); ++iter) {
+ entries += (iter->second).dir.header.stats[category].num_entries;
+ size += (iter->second).dir.header.stats[category].total_size;
+ }
+ ASSERT_EQ(total_size, size);
+ ASSERT_EQ(num_entries, entries);
+}
+
+void index_prepare(librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op,
+ string& tag, const cls_rgw_obj_key& key, string& loc,
+ uint16_t bi_flags = 0, bool log_op = true)
+{
+ ObjectWriteOperation op;
+ rgw_zone_set zones_trace;
+ cls_rgw_bucket_prepare_op(op, index_op, tag, key, loc, log_op, bi_flags, zones_trace);
+ ASSERT_EQ(0, ioctx.operate(oid, &op));
+}
+
+void index_complete(librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op,
+ string& tag, int epoch, const cls_rgw_obj_key& key,
+ rgw_bucket_dir_entry_meta& meta, uint16_t bi_flags = 0,
+ bool log_op = true)
+{
+ ObjectWriteOperation op;
+ rgw_bucket_entry_ver ver;
+ ver.pool = ioctx.get_id();
+ ver.epoch = epoch;
+ meta.accounted_size = meta.size;
+ cls_rgw_bucket_complete_op(op, index_op, tag, ver, key, meta, nullptr, log_op, bi_flags, nullptr);
+ ASSERT_EQ(0, ioctx.operate(oid, &op));
+ if (!key.instance.empty()) {
+ bufferlist olh_tag;
+ olh_tag.append(tag);
+ rgw_zone_set zone_set;
+ ASSERT_EQ(0, cls_rgw_bucket_link_olh(ioctx, oid, key, olh_tag,
+ false, tag, &meta, epoch,
+ ceph::real_time{}, true, true, zone_set));
+ }
+}
+
+TEST_F(cls_rgw, index_basic)
+{
+ string bucket_oid = str_int("bucket", 0);
+
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+
+ uint64_t epoch = 1;
+
+ uint64_t obj_size = 1024;
+
+#define NUM_OBJS 10
+ for (int i = 0; i < NUM_OBJS; i++) {
+ cls_rgw_obj_key obj = str_int("obj", i);
+ string tag = str_int("tag", i);
+ string loc = str_int("loc", i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, i, obj_size * i);
+
+ rgw_bucket_dir_entry_meta meta;
+ meta.category = RGWObjCategory::None;
+ meta.size = obj_size;
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta);
+ }
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS,
+ obj_size * NUM_OBJS);
+}
+
+TEST_F(cls_rgw, index_multiple_obj_writers)
+{
+ string bucket_oid = str_int("bucket", 1);
+
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+
+ uint64_t obj_size = 1024;
+
+ cls_rgw_obj_key obj = str_int("obj", 0);
+ string loc = str_int("loc", 0);
+ /* multi prepare on a single object */
+ for (int i = 0; i < NUM_OBJS; i++) {
+ string tag = str_int("tag", i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, 0, 0);
+ }
+
+ for (int i = NUM_OBJS; i > 0; i--) {
+ string tag = str_int("tag", i - 1);
+
+ rgw_bucket_dir_entry_meta meta;
+ meta.category = RGWObjCategory::None;
+ meta.size = obj_size * i;
+
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, i, obj, meta);
+
+ /* verify that object size doesn't change, as we went back with epoch */
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, 1,
+ obj_size * NUM_OBJS);
+ }
+}
+
+TEST_F(cls_rgw, index_remove_object)
+{
+ string bucket_oid = str_int("bucket", 2);
+
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+
+ uint64_t obj_size = 1024;
+ uint64_t total_size = 0;
+
+ int epoch = 0;
+
+ /* prepare multiple objects */
+ for (int i = 0; i < NUM_OBJS; i++) {
+ cls_rgw_obj_key obj = str_int("obj", i);
+ string tag = str_int("tag", i);
+ string loc = str_int("loc", i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, i, total_size);
+
+ rgw_bucket_dir_entry_meta meta;
+ meta.category = RGWObjCategory::None;
+ meta.size = i * obj_size;
+ total_size += i * obj_size;
+
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, i + 1, total_size);
+ }
+
+ int i = NUM_OBJS / 2;
+ string tag_remove = "tag-rm";
+ string tag_modify = "tag-mod";
+ cls_rgw_obj_key obj = str_int("obj", i);
+ string loc = str_int("loc", i);
+
+ /* prepare both removal and modification on the same object */
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc);
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, obj, loc);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS, total_size);
+
+ rgw_bucket_dir_entry_meta meta;
+
+ /* complete object removal */
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta);
+
+ /* verify stats correct */
+ total_size -= i * obj_size;
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS - 1, total_size);
+
+ meta.size = 512;
+ meta.category = RGWObjCategory::None;
+
+ /* complete object modification */
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta);
+
+ /* verify stats correct */
+ total_size += meta.size;
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS, total_size);
+
+
+ /* prepare both removal and modification on the same object, this time we'll
+ * first complete modification then remove*/
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, obj, loc);
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_modify, obj, loc);
+
+ /* complete modification */
+ total_size -= meta.size;
+ meta.size = i * obj_size * 2;
+ meta.category = RGWObjCategory::None;
+
+ /* complete object modification */
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag_modify, ++epoch, obj, meta);
+
+ /* verify stats correct */
+ total_size += meta.size;
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS, total_size);
+
+ /* complete object removal */
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag_remove, ++epoch, obj, meta);
+
+ /* verify stats correct */
+ total_size -= meta.size;
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, NUM_OBJS - 1,
+ total_size);
+}
+
+TEST_F(cls_rgw, index_suggest)
+{
+ string bucket_oid = str_int("suggest", 1);
+ {
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+ }
+ uint64_t total_size = 0;
+
+ int epoch = 0;
+
+ int num_objs = 100;
+
+ uint64_t obj_size = 1024;
+
+ /* create multiple objects */
+ for (int i = 0; i < num_objs; i++) {
+ cls_rgw_obj_key obj = str_int("obj", i);
+ string tag = str_int("tag", i);
+ string loc = str_int("loc", i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, i, total_size);
+
+ rgw_bucket_dir_entry_meta meta;
+ meta.category = RGWObjCategory::None;
+ meta.size = obj_size;
+ total_size += meta.size;
+
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, i + 1, total_size);
+ }
+
+ /* prepare (without completion) some of the objects */
+ for (int i = 0; i < num_objs; i += 2) {
+ cls_rgw_obj_key obj = str_int("obj", i);
+ string tag = str_int("tag-prepare", i);
+ string loc = str_int("loc", i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, num_objs, total_size);
+ }
+
+ int actual_num_objs = num_objs;
+ /* remove half of the objects */
+ for (int i = num_objs / 2; i < num_objs; i++) {
+ cls_rgw_obj_key obj = str_int("obj", i);
+ string tag = str_int("tag-rm", i);
+ string loc = str_int("loc", i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, actual_num_objs, total_size);
+
+ rgw_bucket_dir_entry_meta meta;
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag, ++epoch, obj, meta);
+
+ total_size -= obj_size;
+ actual_num_objs--;
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, actual_num_objs, total_size);
+ }
+
+ bufferlist updates;
+
+ for (int i = 0; i < num_objs; i += 2) {
+ cls_rgw_obj_key obj = str_int("obj", i);
+ string tag = str_int("tag-rm", i);
+ string loc = str_int("loc", i);
+
+ rgw_bucket_dir_entry dirent;
+ dirent.key.name = obj.name;
+ dirent.locator = loc;
+ dirent.exists = (i < num_objs / 2); // we removed half the objects
+ dirent.meta.size = 1024;
+ dirent.meta.accounted_size = 1024;
+
+ char suggest_op = (i < num_objs / 2 ? CEPH_RGW_UPDATE : CEPH_RGW_REMOVE);
+ cls_rgw_encode_suggestion(suggest_op, dirent, updates);
+ }
+
+ map<int, string> bucket_objs;
+ bucket_objs[0] = bucket_oid;
+ int r = CLSRGWIssueSetTagTimeout(ioctx, bucket_objs, 8 /* max aio */, 1)();
+ ASSERT_EQ(0, r);
+
+ sleep(1);
+
+ /* suggest changes! */
+ {
+ ObjectWriteOperation op;
+ cls_rgw_suggest_changes(op, updates);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+ }
+ /* suggest changes twice! */
+ {
+ ObjectWriteOperation op;
+ cls_rgw_suggest_changes(op, updates);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+ }
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, num_objs / 2, total_size);
+}
+
+static void list_entries(librados::IoCtx& ioctx,
+ const std::string& oid,
+ uint32_t num_entries,
+ std::map<int, rgw_cls_list_ret>& results)
+{
+ std::map<int, std::string> oids = { {0, oid} };
+ cls_rgw_obj_key start_key;
+ string empty_prefix;
+ string empty_delimiter;
+ ASSERT_EQ(0, CLSRGWIssueBucketList(ioctx, start_key, empty_prefix,
+ empty_delimiter, num_entries,
+ true, oids, results, 1)());
+}
+
+TEST_F(cls_rgw, index_suggest_complete)
+{
+ string bucket_oid = str_int("suggest", 2);
+ {
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+ }
+
+ cls_rgw_obj_key obj = str_int("obj", 0);
+ string tag = str_int("tag-prepare", 0);
+ string loc = str_int("loc", 0);
+
+ // prepare entry
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+
+ // list entry before completion
+ rgw_bucket_dir_entry dirent;
+ {
+ std::map<int, rgw_cls_list_ret> listing;
+ list_entries(ioctx, bucket_oid, 1, listing);
+ ASSERT_EQ(1, listing.size());
+ const auto& entries = listing.begin()->second.dir.m;
+ ASSERT_EQ(1, entries.size());
+ dirent = entries.begin()->second;
+ ASSERT_EQ(obj, dirent.key);
+ }
+ // complete entry
+ {
+ rgw_bucket_dir_entry_meta meta;
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, 1, obj, meta);
+ }
+ // suggest removal of listed entry
+ {
+ bufferlist updates;
+ cls_rgw_encode_suggestion(CEPH_RGW_REMOVE, dirent, updates);
+
+ ObjectWriteOperation op;
+ cls_rgw_suggest_changes(op, updates);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+ }
+ // list entry again, verify that suggested removal was not applied
+ {
+ std::map<int, rgw_cls_list_ret> listing;
+ list_entries(ioctx, bucket_oid, 1, listing);
+ ASSERT_EQ(1, listing.size());
+ const auto& entries = listing.begin()->second.dir.m;
+ ASSERT_EQ(1, entries.size());
+ EXPECT_TRUE(entries.begin()->second.exists);
+ }
+}
+
+/*
+ * This case is used to test whether get_obj_vals will
+ * return all validate utf8 objnames and filter out those
+ * in BI_PREFIX_CHAR private namespace.
+ */
+TEST_F(cls_rgw, index_list)
+{
+ string bucket_oid = str_int("bucket", 4);
+
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+
+ uint64_t epoch = 1;
+ uint64_t obj_size = 1024;
+ const int num_objs = 4;
+ const string keys[num_objs] = {
+ /* single byte utf8 character */
+ { static_cast<char>(0x41) },
+ /* double byte utf8 character */
+ { static_cast<char>(0xCF), static_cast<char>(0x8F) },
+ /* treble byte utf8 character */
+ { static_cast<char>(0xDF), static_cast<char>(0x8F), static_cast<char>(0x8F) },
+ /* quadruble byte utf8 character */
+ { static_cast<char>(0xF7), static_cast<char>(0x8F), static_cast<char>(0x8F), static_cast<char>(0x8F) },
+ };
+
+ for (int i = 0; i < num_objs; i++) {
+ string obj = keys[i];
+ string tag = str_int("tag", i);
+ string loc = str_int("loc", i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc,
+ 0 /* bi_flags */, false /* log_op */);
+
+ rgw_bucket_dir_entry_meta meta;
+ meta.category = RGWObjCategory::None;
+ meta.size = obj_size;
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta,
+ 0 /* bi_flags */, false /* log_op */);
+ }
+
+ map<string, bufferlist> entries;
+ /* insert 998 omap key starts with BI_PREFIX_CHAR,
+ * so bucket list first time will get one key before 0x80 and one key after */
+ for (int i = 0; i < 998; ++i) {
+ char buf[10];
+ snprintf(buf, sizeof(buf), "%c%s%d", 0x80, "1000_", i);
+ entries.emplace(string{buf}, bufferlist{});
+ }
+ ioctx.omap_set(bucket_oid, entries);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None,
+ num_objs, obj_size * num_objs);
+
+ map<int, string> oids = { {0, bucket_oid} };
+ map<int, struct rgw_cls_list_ret> list_results;
+ cls_rgw_obj_key start_key("", "");
+ string empty_prefix;
+ string empty_delimiter;
+ int r = CLSRGWIssueBucketList(ioctx, start_key,
+ empty_prefix, empty_delimiter,
+ 1000, true, oids, list_results, 1)();
+ ASSERT_EQ(r, 0);
+ ASSERT_EQ(1u, list_results.size());
+
+ auto it = list_results.begin();
+ auto m = (it->second).dir.m;
+
+ ASSERT_EQ(4u, m.size());
+ int i = 0;
+ for(auto it2 = m.cbegin(); it2 != m.cend(); it2++, i++) {
+ ASSERT_EQ(it2->first.compare(keys[i]), 0);
+ }
+}
+
+
+/*
+ * This case is used to test when bucket index list that includes a
+ * delimiter can handle the first chunk ending in a delimiter.
+ */
+TEST_F(cls_rgw, index_list_delimited)
+{
+ string bucket_oid = str_int("bucket", 7);
+
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+
+ uint64_t epoch = 1;
+ uint64_t obj_size = 1024;
+ const int file_num_objs = 5;
+ const int dir_num_objs = 1005;
+
+ std::vector<std::string> file_prefixes =
+ { "a", "c", "e", "g", "i", "k", "m", "o", "q", "s", "u" };
+ std::vector<std::string> dir_prefixes =
+ { "b/", "d/", "f/", "h/", "j/", "l/", "n/", "p/", "r/", "t/" };
+
+ rgw_bucket_dir_entry_meta meta;
+ meta.category = RGWObjCategory::None;
+ meta.size = obj_size;
+
+ // create top-level files
+ for (const auto& p : file_prefixes) {
+ for (int i = 0; i < file_num_objs; i++) {
+ string tag = str_int("tag", i);
+ string loc = str_int("loc", i);
+ const string obj = str_int(p, i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc,
+ 0 /* bi_flags */, false /* log_op */);
+
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta,
+ 0 /* bi_flags */, false /* log_op */);
+ }
+ }
+
+ // create large directories
+ for (const auto& p : dir_prefixes) {
+ for (int i = 0; i < dir_num_objs; i++) {
+ string tag = str_int("tag", i);
+ string loc = str_int("loc", i);
+ const string obj = p + str_int("f", i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc,
+ 0 /* bi_flags */, false /* log_op */);
+
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta,
+ 0 /* bi_flags */, false /* log_op */);
+ }
+ }
+
+ map<int, string> oids = { {0, bucket_oid} };
+ map<int, struct rgw_cls_list_ret> list_results;
+ cls_rgw_obj_key start_key("", "");
+ const string empty_prefix;
+ const string delimiter = "/";
+ int r = CLSRGWIssueBucketList(ioctx, start_key,
+ empty_prefix, delimiter,
+ 1000, true, oids, list_results, 1)();
+ ASSERT_EQ(r, 0);
+ ASSERT_EQ(1u, list_results.size()) <<
+ "Because we only have one bucket index shard, we should "
+ "only get one list_result.";
+
+ auto it = list_results.begin();
+ auto id_entry_map = it->second.dir.m;
+ bool truncated = it->second.is_truncated;
+
+ // the cls code will make 4 tries to get 1000 entries; however
+ // because each of the subdirectories is so large, each attempt will
+ // only retrieve the first part of the subdirectory
+
+ ASSERT_EQ(48u, id_entry_map.size()) <<
+ "We should get 40 top-level entries and the tops of 8 \"subdirectories\".";
+ ASSERT_EQ(true, truncated) << "We did not get all entries.";
+
+ ASSERT_EQ("a-0", id_entry_map.cbegin()->first);
+ ASSERT_EQ("p/", id_entry_map.crbegin()->first);
+
+ // now let's get the rest of the entries
+
+ list_results.clear();
+
+ cls_rgw_obj_key start_key2("p/", "");
+ r = CLSRGWIssueBucketList(ioctx, start_key2,
+ empty_prefix, delimiter,
+ 1000, true, oids, list_results, 1)();
+ ASSERT_EQ(r, 0);
+
+ it = list_results.begin();
+ id_entry_map = it->second.dir.m;
+ truncated = it->second.is_truncated;
+
+ ASSERT_EQ(17u, id_entry_map.size()) <<
+ "We should get 15 top-level entries and the tops of 2 \"subdirectories\".";
+ ASSERT_EQ(false, truncated) << "We now have all entries.";
+
+ ASSERT_EQ("q-0", id_entry_map.cbegin()->first);
+ ASSERT_EQ("u-4", id_entry_map.crbegin()->first);
+}
+
+
+TEST_F(cls_rgw, bi_list)
+{
+ string bucket_oid = str_int("bucket", 5);
+
+ CephContext *cct = reinterpret_cast<CephContext *>(ioctx.cct());
+
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+
+ const std::string empty_name_filter;
+ uint64_t max = 10;
+ std::list<rgw_cls_bi_entry> entries;
+ bool is_truncated;
+ std::string marker;
+
+ int ret = cls_rgw_bi_list(ioctx, bucket_oid, empty_name_filter, marker, max,
+ &entries, &is_truncated);
+ ASSERT_EQ(ret, 0);
+ ASSERT_EQ(entries.size(), 0u) <<
+ "The listing of an empty bucket as 0 entries.";
+ ASSERT_EQ(is_truncated, false) <<
+ "The listing of an empty bucket is not truncated.";
+
+ uint64_t epoch = 1;
+ uint64_t obj_size = 1024;
+ const uint64_t num_objs = 35;
+
+ for (uint64_t i = 0; i < num_objs; i++) {
+ string obj = str_int(i % 4 ? "obj" : "об'єкт", i);
+ string tag = str_int("tag", i);
+ string loc = str_int("loc", i);
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc,
+ RGW_BILOG_FLAG_VERSIONED_OP);
+
+ rgw_bucket_dir_entry_meta meta;
+ meta.category = RGWObjCategory::None;
+ meta.size = obj_size;
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta,
+ RGW_BILOG_FLAG_VERSIONED_OP);
+ }
+
+ ret = cls_rgw_bi_list(ioctx, bucket_oid, empty_name_filter, marker, num_objs + 10,
+ &entries, &is_truncated);
+ ASSERT_EQ(ret, 0);
+ if (is_truncated) {
+ ASSERT_LT(entries.size(), num_objs);
+ } else {
+ ASSERT_EQ(entries.size(), num_objs);
+ }
+
+ uint64_t num_entries = 0;
+
+ is_truncated = true;
+ marker.clear();
+ while(is_truncated) {
+ ret = cls_rgw_bi_list(ioctx, bucket_oid, empty_name_filter, marker, max,
+ &entries, &is_truncated);
+ ASSERT_EQ(ret, 0);
+ if (is_truncated) {
+ ASSERT_LT(entries.size(), num_objs - num_entries);
+ } else {
+ ASSERT_EQ(entries.size(), num_objs - num_entries);
+ }
+ num_entries += entries.size();
+ marker = entries.back().idx;
+ }
+
+ // try with marker as final entry
+ ret = cls_rgw_bi_list(ioctx, bucket_oid, empty_name_filter, marker, max,
+ &entries, &is_truncated);
+ ASSERT_EQ(ret, 0);
+ ASSERT_EQ(entries.size(), 0u);
+ ASSERT_EQ(is_truncated, false);
+
+ if (cct->_conf->osd_max_omap_entries_per_request < 15) {
+ num_entries = 0;
+ max = 15;
+ is_truncated = true;
+ marker.clear();
+ while(is_truncated) {
+ ret = cls_rgw_bi_list(ioctx, bucket_oid, empty_name_filter, marker, max,
+ &entries, &is_truncated);
+ ASSERT_EQ(ret, 0);
+ if (is_truncated) {
+ ASSERT_LT(entries.size(), num_objs - num_entries);
+ } else {
+ ASSERT_EQ(entries.size(), num_objs - num_entries);
+ }
+ num_entries += entries.size();
+ marker = entries.back().idx;
+ }
+
+ // try with marker as final entry
+ ret = cls_rgw_bi_list(ioctx, bucket_oid, empty_name_filter, marker, max,
+ &entries, &is_truncated);
+ ASSERT_EQ(ret, 0);
+ ASSERT_EQ(entries.size(), 0u);
+ ASSERT_EQ(is_truncated, false);
+ }
+
+ // test with name filters; pairs contain filter and expected number of elements returned
+ const std::list<std::pair<const std::string,unsigned>> filters_results =
+ { { str_int("obj", 9), 1 },
+ { str_int("об'єкт", 8), 1 },
+ { str_int("obj", 8), 0 } };
+ for (const auto& filter_result : filters_results) {
+ is_truncated = true;
+ entries.clear();
+ marker.clear();
+
+ ret = cls_rgw_bi_list(ioctx, bucket_oid, filter_result.first, marker, max,
+ &entries, &is_truncated);
+
+ ASSERT_EQ(ret, 0) << "bi list test with name filters should succeed";
+ ASSERT_EQ(entries.size(), filter_result.second) <<
+ "bi list test with filters should return the correct number of results";
+ ASSERT_EQ(is_truncated, false) <<
+ "bi list test with filters should return correct truncation indicator";
+ }
+
+ // test whether combined segment count is correcgt
+ is_truncated = false;
+ entries.clear();
+ marker.clear();
+
+ ret = cls_rgw_bi_list(ioctx, bucket_oid, empty_name_filter, marker, num_objs - 1,
+ &entries, &is_truncated);
+ ASSERT_EQ(ret, 0) << "combined segment count should succeed";
+ ASSERT_EQ(entries.size(), num_objs - 1) <<
+ "combined segment count should return the correct number of results";
+ ASSERT_EQ(is_truncated, true) <<
+ "combined segment count should return correct truncation indicator";
+
+
+ marker = entries.back().idx; // advance marker
+ ret = cls_rgw_bi_list(ioctx, bucket_oid, empty_name_filter, marker, num_objs - 1,
+ &entries, &is_truncated);
+ ASSERT_EQ(ret, 0) << "combined segment count should succeed";
+ ASSERT_EQ(entries.size(), 1) <<
+ "combined segment count should return the correct number of results";
+ ASSERT_EQ(is_truncated, false) <<
+ "combined segment count should return correct truncation indicator";
+}
+
+/* test garbage collection */
+static void create_obj(cls_rgw_obj& obj, int i, int j)
+{
+ char buf[32];
+ snprintf(buf, sizeof(buf), "-%d.%d", i, j);
+ obj.pool = "pool";
+ obj.pool.append(buf);
+ obj.key.name = "oid";
+ obj.key.name.append(buf);
+ obj.loc = "loc";
+ obj.loc.append(buf);
+}
+
+static bool cmp_objs(cls_rgw_obj& obj1, cls_rgw_obj& obj2)
+{
+ return (obj1.pool == obj2.pool) &&
+ (obj1.key == obj2.key) &&
+ (obj1.loc == obj2.loc);
+}
+
+
+TEST_F(cls_rgw, gc_set)
+{
+ /* add chains */
+ string oid = "obj";
+ for (int i = 0; i < 10; i++) {
+ char buf[32];
+ snprintf(buf, sizeof(buf), "chain-%d", i);
+ string tag = buf;
+ librados::ObjectWriteOperation op;
+ cls_rgw_gc_obj_info info;
+
+ cls_rgw_obj obj1, obj2;
+ create_obj(obj1, i, 1);
+ create_obj(obj2, i, 2);
+ info.chain.objs.push_back(obj1);
+ info.chain.objs.push_back(obj2);
+
+ op.create(false); // create object
+
+ info.tag = tag;
+ cls_rgw_gc_set_entry(op, 0, info);
+
+ ASSERT_EQ(0, ioctx.operate(oid, &op));
+ }
+
+ bool truncated;
+ list<cls_rgw_gc_obj_info> entries;
+ string marker;
+ string next_marker;
+
+ /* list chains, verify truncated */
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries, &truncated, next_marker));
+ ASSERT_EQ(8, (int)entries.size());
+ ASSERT_EQ(1, truncated);
+
+ entries.clear();
+ next_marker.clear();
+
+ /* list all chains, verify not truncated */
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, true, entries, &truncated, next_marker));
+ ASSERT_EQ(10, (int)entries.size());
+ ASSERT_EQ(0, truncated);
+
+ /* verify all chains are valid */
+ list<cls_rgw_gc_obj_info>::iterator iter = entries.begin();
+ for (int i = 0; i < 10; i++, ++iter) {
+ cls_rgw_gc_obj_info& entry = *iter;
+
+ /* create expected chain name */
+ char buf[32];
+ snprintf(buf, sizeof(buf), "chain-%d", i);
+ string tag = buf;
+
+ /* verify chain name as expected */
+ ASSERT_EQ(entry.tag, tag);
+
+ /* verify expected num of objects in chain */
+ ASSERT_EQ(2, (int)entry.chain.objs.size());
+
+ list<cls_rgw_obj>::iterator oiter = entry.chain.objs.begin();
+ cls_rgw_obj obj1, obj2;
+
+ /* create expected objects */
+ create_obj(obj1, i, 1);
+ create_obj(obj2, i, 2);
+
+ /* assign returned object names */
+ cls_rgw_obj& ret_obj1 = *oiter++;
+ cls_rgw_obj& ret_obj2 = *oiter;
+
+ /* verify objects are as expected */
+ ASSERT_EQ(1, (int)cmp_objs(obj1, ret_obj1));
+ ASSERT_EQ(1, (int)cmp_objs(obj2, ret_obj2));
+ }
+}
+
+TEST_F(cls_rgw, gc_list)
+{
+ /* add chains */
+ string oid = "obj";
+ for (int i = 0; i < 10; i++) {
+ char buf[32];
+ snprintf(buf, sizeof(buf), "chain-%d", i);
+ string tag = buf;
+ librados::ObjectWriteOperation op;
+ cls_rgw_gc_obj_info info;
+
+ cls_rgw_obj obj1, obj2;
+ create_obj(obj1, i, 1);
+ create_obj(obj2, i, 2);
+ info.chain.objs.push_back(obj1);
+ info.chain.objs.push_back(obj2);
+
+ op.create(false); // create object
+
+ info.tag = tag;
+ cls_rgw_gc_set_entry(op, 0, info);
+
+ ASSERT_EQ(0, ioctx.operate(oid, &op));
+ }
+
+ bool truncated;
+ list<cls_rgw_gc_obj_info> entries;
+ list<cls_rgw_gc_obj_info> entries2;
+ string marker;
+ string next_marker;
+
+ /* list chains, verify truncated */
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries, &truncated, next_marker));
+ ASSERT_EQ(8, (int)entries.size());
+ ASSERT_EQ(1, truncated);
+
+ marker = next_marker;
+ next_marker.clear();
+
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries2, &truncated, next_marker));
+ ASSERT_EQ(2, (int)entries2.size());
+ ASSERT_EQ(0, truncated);
+
+ entries.splice(entries.end(), entries2);
+
+ /* verify all chains are valid */
+ list<cls_rgw_gc_obj_info>::iterator iter = entries.begin();
+ for (int i = 0; i < 10; i++, ++iter) {
+ cls_rgw_gc_obj_info& entry = *iter;
+
+ /* create expected chain name */
+ char buf[32];
+ snprintf(buf, sizeof(buf), "chain-%d", i);
+ string tag = buf;
+
+ /* verify chain name as expected */
+ ASSERT_EQ(entry.tag, tag);
+
+ /* verify expected num of objects in chain */
+ ASSERT_EQ(2, (int)entry.chain.objs.size());
+
+ list<cls_rgw_obj>::iterator oiter = entry.chain.objs.begin();
+ cls_rgw_obj obj1, obj2;
+
+ /* create expected objects */
+ create_obj(obj1, i, 1);
+ create_obj(obj2, i, 2);
+
+ /* assign returned object names */
+ cls_rgw_obj& ret_obj1 = *oiter++;
+ cls_rgw_obj& ret_obj2 = *oiter;
+
+ /* verify objects are as expected */
+ ASSERT_EQ(1, (int)cmp_objs(obj1, ret_obj1));
+ ASSERT_EQ(1, (int)cmp_objs(obj2, ret_obj2));
+ }
+}
+
+TEST_F(cls_rgw, gc_defer)
+{
+ librados::IoCtx ioctx;
+ librados::Rados rados;
+
+ string gc_pool_name = get_temp_pool_name();
+ /* create pool */
+ ASSERT_EQ("", create_one_pool_pp(gc_pool_name, rados));
+ ASSERT_EQ(0, rados.ioctx_create(gc_pool_name.c_str(), ioctx));
+
+ string oid = "obj";
+ string tag = "mychain";
+
+ librados::ObjectWriteOperation op;
+ cls_rgw_gc_obj_info info;
+
+ op.create(false);
+
+ info.tag = tag;
+
+ /* create chain */
+ cls_rgw_gc_set_entry(op, 0, info);
+
+ ASSERT_EQ(0, ioctx.operate(oid, &op));
+
+ bool truncated;
+ list<cls_rgw_gc_obj_info> entries;
+ string marker;
+ string next_marker;
+
+ /* list chains, verify num entries as expected */
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated, next_marker));
+ ASSERT_EQ(1, (int)entries.size());
+ ASSERT_EQ(0, truncated);
+
+ librados::ObjectWriteOperation op2;
+
+ /* defer chain */
+ cls_rgw_gc_defer_entry(op2, 5, tag);
+ ASSERT_EQ(0, ioctx.operate(oid, &op2));
+
+ entries.clear();
+ next_marker.clear();
+
+ /* verify list doesn't show deferred entry (this may fail if cluster is thrashing) */
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated, next_marker));
+ ASSERT_EQ(0, (int)entries.size());
+ ASSERT_EQ(0, truncated);
+
+ /* wait enough */
+ sleep(5);
+ next_marker.clear();
+
+ /* verify list shows deferred entry */
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated, next_marker));
+ ASSERT_EQ(1, (int)entries.size());
+ ASSERT_EQ(0, truncated);
+
+ librados::ObjectWriteOperation op3;
+ vector<string> tags;
+ tags.push_back(tag);
+
+ /* remove chain */
+ cls_rgw_gc_remove(op3, tags);
+ ASSERT_EQ(0, ioctx.operate(oid, &op3));
+
+ entries.clear();
+ next_marker.clear();
+
+ /* verify entry was removed */
+ ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated, next_marker));
+ ASSERT_EQ(0, (int)entries.size());
+ ASSERT_EQ(0, truncated);
+
+ /* remove pool */
+ ioctx.close();
+ ASSERT_EQ(0, destroy_one_pool_pp(gc_pool_name, rados));
+}
+
+auto populate_usage_log_info(std::string user, std::string payer, int total_usage_entries)
+{
+ rgw_usage_log_info info;
+
+ for (int i=0; i < total_usage_entries; i++){
+ auto bucket = str_int("bucket", i);
+ info.entries.emplace_back(rgw_usage_log_entry(user, payer, bucket));
+ }
+
+ return info;
+}
+
+auto gen_usage_log_info(std::string payer, std::string bucket, int total_usage_entries)
+{
+ rgw_usage_log_info info;
+ for (int i=0; i < total_usage_entries; i++){
+ auto user = str_int("user", i);
+ info.entries.emplace_back(rgw_usage_log_entry(user, payer, bucket));
+ }
+
+ return info;
+}
+
+TEST_F(cls_rgw, usage_basic)
+{
+ string oid="usage.1";
+ string user="user1";
+ uint64_t start_epoch{0}, end_epoch{(uint64_t) -1};
+ int total_usage_entries = 512;
+ uint64_t max_entries = 2000;
+ string payer;
+
+ auto info = populate_usage_log_info(user, payer, total_usage_entries);
+ ObjectWriteOperation op;
+ cls_rgw_usage_log_add(op, info);
+ ASSERT_EQ(0, ioctx.operate(oid, &op));
+
+ string read_iter;
+ map <rgw_user_bucket, rgw_usage_log_entry> usage, usage2;
+ bool truncated;
+
+
+ int ret = cls_rgw_usage_log_read(ioctx, oid, user, "", start_epoch, end_epoch,
+ max_entries, read_iter, usage, &truncated);
+ // read the entries, and see that we have all the added entries
+ ASSERT_EQ(0, ret);
+ ASSERT_FALSE(truncated);
+ ASSERT_EQ(static_cast<uint64_t>(total_usage_entries), usage.size());
+
+ // delete and read to assert that we've deleted all the values
+ ASSERT_EQ(0, cls_rgw_usage_log_trim(ioctx, oid, user, "", start_epoch, end_epoch));
+
+
+ ret = cls_rgw_usage_log_read(ioctx, oid, user, "", start_epoch, end_epoch,
+ max_entries, read_iter, usage2, &truncated);
+ ASSERT_EQ(0, ret);
+ ASSERT_EQ(0u, usage2.size());
+
+ // add and read to assert that bucket option is valid for usage reading
+ string bucket1 = "bucket-usage-1";
+ string bucket2 = "bucket-usage-2";
+ info = gen_usage_log_info(payer, bucket1, 100);
+ cls_rgw_usage_log_add(op, info);
+ ASSERT_EQ(0, ioctx.operate(oid, &op));
+
+ info = gen_usage_log_info(payer, bucket2, 100);
+ cls_rgw_usage_log_add(op, info);
+ ASSERT_EQ(0, ioctx.operate(oid, &op));
+ ret = cls_rgw_usage_log_read(ioctx, oid, "", bucket1, start_epoch, end_epoch,
+ max_entries, read_iter, usage2, &truncated);
+ ASSERT_EQ(0, ret);
+ ASSERT_EQ(100u, usage2.size());
+
+ // delete and read to assert that bucket option is valid for usage trim
+ ASSERT_EQ(0, cls_rgw_usage_log_trim(ioctx, oid, "", bucket1, start_epoch, end_epoch));
+
+ ret = cls_rgw_usage_log_read(ioctx, oid, "", bucket1, start_epoch, end_epoch,
+ max_entries, read_iter, usage2, &truncated);
+ ASSERT_EQ(0, ret);
+ ASSERT_EQ(0u, usage2.size());
+ ASSERT_EQ(0, cls_rgw_usage_log_trim(ioctx, oid, "", bucket2, start_epoch, end_epoch));
+}
+
+TEST_F(cls_rgw, usage_clear_no_obj)
+{
+ string user="user1";
+ string oid="usage.10";
+ librados::ObjectWriteOperation op;
+ cls_rgw_usage_log_clear(op);
+ int ret = ioctx.operate(oid, &op);
+ ASSERT_EQ(0, ret);
+
+}
+
+TEST_F(cls_rgw, usage_clear)
+{
+ string user="user1";
+ string payer;
+ string oid="usage.10";
+ librados::ObjectWriteOperation op;
+ int max_entries=2000;
+
+ auto info = populate_usage_log_info(user, payer, max_entries);
+
+ cls_rgw_usage_log_add(op, info);
+ ASSERT_EQ(0, ioctx.operate(oid, &op));
+
+ ObjectWriteOperation op2;
+ cls_rgw_usage_log_clear(op2);
+ int ret = ioctx.operate(oid, &op2);
+ ASSERT_EQ(0, ret);
+
+ map <rgw_user_bucket, rgw_usage_log_entry> usage;
+ bool truncated;
+ uint64_t start_epoch{0}, end_epoch{(uint64_t) -1};
+ string read_iter;
+ ret = cls_rgw_usage_log_read(ioctx, oid, user, "", start_epoch, end_epoch,
+ max_entries, read_iter, usage, &truncated);
+ ASSERT_EQ(0, ret);
+ ASSERT_EQ(0u, usage.size());
+}
+
+static int bilog_list(librados::IoCtx& ioctx, const std::string& oid,
+ cls_rgw_bi_log_list_ret *result)
+{
+ int retcode = 0;
+ librados::ObjectReadOperation op;
+ cls_rgw_bilog_list(op, "", 128, result, &retcode);
+ int ret = ioctx.operate(oid, &op, nullptr);
+ if (ret < 0) {
+ return ret;
+ }
+ return retcode;
+}
+
+static int bilog_trim(librados::IoCtx& ioctx, const std::string& oid,
+ const std::string& start_marker,
+ const std::string& end_marker)
+{
+ librados::ObjectWriteOperation op;
+ cls_rgw_bilog_trim(op, start_marker, end_marker);
+ return ioctx.operate(oid, &op);
+}
+
+TEST_F(cls_rgw, bi_log_trim)
+{
+ string bucket_oid = str_int("bucket", 6);
+
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+
+ // create 10 versioned entries. this generates instance and olh bi entries,
+ // allowing us to check that bilog trim doesn't remove any of those
+ for (int i = 0; i < 10; i++) {
+ cls_rgw_obj_key obj{str_int("obj", i), "inst"};
+ string tag = str_int("tag", i);
+ string loc = str_int("loc", i);
+
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+ rgw_bucket_dir_entry_meta meta;
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, 1, obj, meta);
+ }
+ // bi list
+ {
+ list<rgw_cls_bi_entry> entries;
+ bool truncated{false};
+ ASSERT_EQ(0, cls_rgw_bi_list(ioctx, bucket_oid, "", "", 128,
+ &entries, &truncated));
+ // prepare/complete/instance/olh entry for each
+ EXPECT_EQ(40u, entries.size());
+ EXPECT_FALSE(truncated);
+ }
+ // bilog list
+ vector<rgw_bi_log_entry> bilog1;
+ {
+ cls_rgw_bi_log_list_ret bilog;
+ ASSERT_EQ(0, bilog_list(ioctx, bucket_oid, &bilog));
+ // complete/olh entry for each
+ EXPECT_EQ(20u, bilog.entries.size());
+
+ bilog1.assign(std::make_move_iterator(bilog.entries.begin()),
+ std::make_move_iterator(bilog.entries.end()));
+ }
+ // trim front of bilog
+ {
+ const std::string from = "";
+ const std::string to = bilog1[0].id;
+ ASSERT_EQ(0, bilog_trim(ioctx, bucket_oid, from, to));
+ cls_rgw_bi_log_list_ret bilog;
+ ASSERT_EQ(0, bilog_list(ioctx, bucket_oid, &bilog));
+ EXPECT_EQ(19u, bilog.entries.size());
+ EXPECT_EQ(bilog1[1].id, bilog.entries.begin()->id);
+ ASSERT_EQ(-ENODATA, bilog_trim(ioctx, bucket_oid, from, to));
+ }
+ // trim back of bilog
+ {
+ const std::string from = bilog1[18].id;
+ const std::string to = "9";
+ ASSERT_EQ(0, bilog_trim(ioctx, bucket_oid, from, to));
+ cls_rgw_bi_log_list_ret bilog;
+ ASSERT_EQ(0, bilog_list(ioctx, bucket_oid, &bilog));
+ EXPECT_EQ(18u, bilog.entries.size());
+ EXPECT_EQ(bilog1[18].id, bilog.entries.rbegin()->id);
+ ASSERT_EQ(-ENODATA, bilog_trim(ioctx, bucket_oid, from, to));
+ }
+ // trim middle of bilog
+ {
+ const std::string from = bilog1[13].id;
+ const std::string to = bilog1[14].id;
+ ASSERT_EQ(0, bilog_trim(ioctx, bucket_oid, from, to));
+ cls_rgw_bi_log_list_ret bilog;
+ ASSERT_EQ(0, bilog_list(ioctx, bucket_oid, &bilog));
+ EXPECT_EQ(17u, bilog.entries.size());
+ ASSERT_EQ(-ENODATA, bilog_trim(ioctx, bucket_oid, from, to));
+ }
+ // trim full bilog
+ {
+ const std::string from = "";
+ const std::string to = "9";
+ ASSERT_EQ(0, bilog_trim(ioctx, bucket_oid, from, to));
+ cls_rgw_bi_log_list_ret bilog;
+ ASSERT_EQ(0, bilog_list(ioctx, bucket_oid, &bilog));
+ EXPECT_EQ(0u, bilog.entries.size());
+ ASSERT_EQ(-ENODATA, bilog_trim(ioctx, bucket_oid, from, to));
+ }
+ // bi list should be the same
+ {
+ list<rgw_cls_bi_entry> entries;
+ bool truncated{false};
+ ASSERT_EQ(0, cls_rgw_bi_list(ioctx, bucket_oid, "", "", 128,
+ &entries, &truncated));
+ EXPECT_EQ(40u, entries.size());
+ EXPECT_FALSE(truncated);
+ }
+}
+
+TEST_F(cls_rgw, index_racing_removes)
+{
+ string bucket_oid = str_int("bucket", 8);
+
+ ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ ASSERT_EQ(0, ioctx.operate(bucket_oid, &op));
+
+ int epoch = 0;
+ rgw_bucket_dir_entry dirent;
+ rgw_bucket_dir_entry_meta meta;
+
+ // prepare/complete add for single object
+ const cls_rgw_obj_key obj{"obj"};
+ std::string loc = "loc";
+ {
+ std::string tag = "tag-add";
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, ++epoch, obj, meta);
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, 1, 0);
+ }
+
+ // list to verify no pending ops
+ {
+ std::map<int, rgw_cls_list_ret> results;
+ list_entries(ioctx, bucket_oid, 1, results);
+ ASSERT_EQ(1, results.size());
+ const auto& entries = results.begin()->second.dir.m;
+ ASSERT_EQ(1, entries.size());
+ dirent = std::move(entries.begin()->second);
+ ASSERT_EQ(obj, dirent.key);
+ ASSERT_TRUE(dirent.exists);
+ ASSERT_TRUE(dirent.pending_map.empty());
+ }
+
+ // prepare three racing removals
+ std::string tag1 = "tag-rm1";
+ std::string tag2 = "tag-rm2";
+ std::string tag3 = "tag-rm3";
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag1, obj, loc);
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag2, obj, loc);
+ index_prepare(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag3, obj, loc);
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, 1, 0);
+
+ // complete on tag2
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_DEL, tag2, ++epoch, obj, meta);
+ {
+ std::map<int, rgw_cls_list_ret> results;
+ list_entries(ioctx, bucket_oid, 1, results);
+ ASSERT_EQ(1, results.size());
+ const auto& entries = results.begin()->second.dir.m;
+ ASSERT_EQ(1, entries.size());
+ dirent = std::move(entries.begin()->second);
+ ASSERT_EQ(obj, dirent.key);
+ ASSERT_FALSE(dirent.exists);
+ ASSERT_FALSE(dirent.pending_map.empty());
+ }
+
+ // cancel on tag1
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_CANCEL, tag1, ++epoch, obj, meta);
+ {
+ std::map<int, rgw_cls_list_ret> results;
+ list_entries(ioctx, bucket_oid, 1, results);
+ ASSERT_EQ(1, results.size());
+ const auto& entries = results.begin()->second.dir.m;
+ ASSERT_EQ(1, entries.size());
+ dirent = std::move(entries.begin()->second);
+ ASSERT_EQ(obj, dirent.key);
+ ASSERT_FALSE(dirent.exists);
+ ASSERT_FALSE(dirent.pending_map.empty());
+ }
+
+ // final cancel on tag3
+ index_complete(ioctx, bucket_oid, CLS_RGW_OP_CANCEL, tag3, ++epoch, obj, meta);
+
+ // verify that the key was removed
+ {
+ std::map<int, rgw_cls_list_ret> results;
+ list_entries(ioctx, bucket_oid, 1, results);
+ EXPECT_EQ(1, results.size());
+ const auto& entries = results.begin()->second.dir.m;
+ ASSERT_EQ(0, entries.size());
+ }
+
+ test_stats(ioctx, bucket_oid, RGWObjCategory::None, 0, 0);
+}
diff --git a/src/test/cls_rgw/test_cls_rgw_stats.cc b/src/test/cls_rgw/test_cls_rgw_stats.cc
new file mode 100644
index 000000000..004ccc6d1
--- /dev/null
+++ b/src/test/cls_rgw/test_cls_rgw_stats.cc
@@ -0,0 +1,646 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <vector>
+#include <boost/circular_buffer.hpp>
+#include <boost/intrusive/set.hpp>
+#include <gtest/gtest.h>
+#include "cls/rgw/cls_rgw_client.h"
+#include "common/debug.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "common/random_string.h"
+#include "global/global_context.h"
+#include "test/librados/test_cxx.h"
+
+#define dout_subsys ceph_subsys_rgw
+#define dout_context g_ceph_context
+
+
+// simulation parameters:
+
+// total number of index operations to prepare
+constexpr size_t max_operations = 2048;
+// total number of object names. each operation chooses one at random
+constexpr size_t max_entries = 32;
+// maximum number of pending operations. once the limit is reached, the oldest
+// pending operation is finished before another can start
+constexpr size_t max_pending = 16;
+// object size is randomly distributed between 0 and 4M
+constexpr size_t max_object_size = 4*1024*1024;
+// multipart upload threshold
+constexpr size_t max_part_size = 1024*1024;
+
+
+// create/destroy a pool that's shared by all tests in the process
+struct RadosEnv : public ::testing::Environment {
+ static std::optional<std::string> pool_name;
+ public:
+ static librados::Rados rados;
+ static librados::IoCtx ioctx;
+
+ void SetUp() override {
+ // create pool
+ std::string name = get_temp_pool_name();
+ ASSERT_EQ("", create_one_pool_pp(name, rados));
+ pool_name = name;
+ ASSERT_EQ(rados.ioctx_create(name.c_str(), ioctx), 0);
+ }
+ void TearDown() override {
+ ioctx.close();
+ if (pool_name) {
+ ASSERT_EQ(destroy_one_pool_pp(*pool_name, rados), 0);
+ }
+ }
+};
+std::optional<std::string> RadosEnv::pool_name;
+librados::Rados RadosEnv::rados;
+librados::IoCtx RadosEnv::ioctx;
+
+auto *const rados_env = ::testing::AddGlobalTestEnvironment(new RadosEnv);
+
+
+std::ostream& operator<<(std::ostream& out, const rgw_bucket_category_stats& c) {
+ return out << "{count=" << c.num_entries << " size=" << c.total_size << '}';
+}
+
+
+// librados helpers
+rgw_bucket_entry_ver last_version(librados::IoCtx& ioctx)
+{
+ rgw_bucket_entry_ver ver;
+ ver.pool = ioctx.get_id();
+ ver.epoch = ioctx.get_last_version();
+ return ver;
+}
+
+int index_init(librados::IoCtx& ioctx, const std::string& oid)
+{
+ librados::ObjectWriteOperation op;
+ cls_rgw_bucket_init_index(op);
+ return ioctx.operate(oid, &op);
+}
+
+int index_prepare(librados::IoCtx& ioctx, const std::string& oid,
+ const cls_rgw_obj_key& key, const std::string& tag,
+ RGWModifyOp type)
+{
+ librados::ObjectWriteOperation op;
+ const std::string loc; // empty
+ constexpr bool log_op = false;
+ constexpr int flags = 0;
+ rgw_zone_set zones;
+ cls_rgw_bucket_prepare_op(op, type, tag, key, loc, log_op, flags, zones);
+ return ioctx.operate(oid, &op);
+}
+
+int index_complete(librados::IoCtx& ioctx, const std::string& oid,
+ const cls_rgw_obj_key& key, const std::string& tag,
+ RGWModifyOp type, const rgw_bucket_entry_ver& ver,
+ const rgw_bucket_dir_entry_meta& meta,
+ std::list<cls_rgw_obj_key>* remove_objs)
+{
+ librados::ObjectWriteOperation op;
+ constexpr bool log_op = false;
+ constexpr int flags = 0;
+ constexpr rgw_zone_set* zones = nullptr;
+ cls_rgw_bucket_complete_op(op, type, tag, ver, key, meta,
+ remove_objs, log_op, flags, zones);
+ return ioctx.operate(oid, &op);
+}
+
+void read_stats(librados::IoCtx& ioctx, const std::string& oid,
+ rgw_bucket_dir_stats& stats)
+{
+ auto oids = std::map<int, std::string>{{0, oid}};
+ std::map<int, rgw_cls_list_ret> results;
+ ASSERT_EQ(0, CLSRGWIssueGetDirHeader(ioctx, oids, results, 8)());
+ ASSERT_EQ(1, results.size());
+ stats = std::move(results.begin()->second.dir.header.stats);
+}
+
+static void account_entry(rgw_bucket_dir_stats& stats,
+ const rgw_bucket_dir_entry_meta& meta)
+{
+ rgw_bucket_category_stats& c = stats[meta.category];
+ c.num_entries++;
+ c.total_size += meta.accounted_size;
+ c.total_size_rounded += cls_rgw_get_rounded_size(meta.accounted_size);
+ c.actual_size += meta.size;
+}
+
+static void unaccount_entry(rgw_bucket_dir_stats& stats,
+ const rgw_bucket_dir_entry_meta& meta)
+{
+ rgw_bucket_category_stats& c = stats[meta.category];
+ c.num_entries--;
+ c.total_size -= meta.accounted_size;
+ c.total_size_rounded -= cls_rgw_get_rounded_size(meta.accounted_size);
+ c.actual_size -= meta.size;
+}
+
+
+// a map of cached dir entries representing the expected state of cls_rgw
+struct object : rgw_bucket_dir_entry, boost::intrusive::set_base_hook<> {
+ explicit object(const cls_rgw_obj_key& key) {
+ this->key = key;
+ }
+};
+
+struct object_key {
+ using type = cls_rgw_obj_key;
+ const type& operator()(const object& o) const { return o.key; }
+};
+
+using object_map_base = boost::intrusive::set<object,
+ boost::intrusive::key_of_value<object_key>>;
+
+struct object_map : object_map_base {
+ ~object_map() {
+ clear_and_dispose(std::default_delete<object>{});
+ }
+};
+
+
+// models a bucket index operation, starting with cls_rgw_bucket_prepare_op().
+// stores all of the state necessary to complete the operation, either with
+// cls_rgw_bucket_complete_op() or cls_rgw_suggest_changes(). uploads larger
+// than max_part_size are modeled as multipart uploads
+struct operation {
+ RGWModifyOp type;
+ cls_rgw_obj_key key;
+ std::string tag;
+ rgw_bucket_entry_ver ver;
+ std::string upload_id; // empty unless multipart
+ rgw_bucket_dir_entry_meta meta;
+};
+
+
+class simulator {
+ public:
+ simulator(librados::IoCtx& ioctx, std::string oid)
+ : ioctx(ioctx), oid(std::move(oid)), pending(max_pending)
+ {
+ // generate a set of object keys. each operation chooses one at random
+ keys.reserve(max_entries);
+ for (size_t i = 0; i < max_entries; i++) {
+ keys.emplace_back(gen_rand_alphanumeric_upper(g_ceph_context, 12));
+ }
+ }
+
+ void run();
+
+ private:
+ void start();
+ int try_start(const cls_rgw_obj_key& key,
+ const std::string& tag);
+
+ void finish(const operation& op);
+ void complete(const operation& op, RGWModifyOp type);
+ void suggest(const operation& op, char suggestion);
+
+ int init_multipart(const operation& op);
+ void complete_multipart(const operation& op);
+
+ object_map::iterator find_or_create(const cls_rgw_obj_key& key);
+
+ librados::IoCtx& ioctx;
+ std::string oid;
+
+ std::vector<cls_rgw_obj_key> keys;
+ object_map objects;
+ boost::circular_buffer<operation> pending;
+ rgw_bucket_dir_stats stats;
+};
+
+
+void simulator::run()
+{
+ // init the bucket index object
+ ASSERT_EQ(0, index_init(ioctx, oid));
+ // run the simulation for N steps
+ for (size_t i = 0; i < max_operations; i++) {
+ if (pending.full()) {
+ // if we're at max_pending, finish the oldest operation
+ auto& op = pending.front();
+ finish(op);
+ pending.pop_front();
+
+ // verify bucket stats
+ rgw_bucket_dir_stats stored_stats;
+ read_stats(ioctx, oid, stored_stats);
+
+ const rgw_bucket_dir_stats& expected_stats = stats;
+ ASSERT_EQ(expected_stats, stored_stats);
+ }
+
+ // initiate the next operation
+ start();
+
+ // verify bucket stats
+ rgw_bucket_dir_stats stored_stats;
+ read_stats(ioctx, oid, stored_stats);
+
+ const rgw_bucket_dir_stats& expected_stats = stats;
+ ASSERT_EQ(expected_stats, stored_stats);
+ }
+}
+
+object_map::iterator simulator::find_or_create(const cls_rgw_obj_key& key)
+{
+ object_map::insert_commit_data commit;
+ auto result = objects.insert_check(key, std::less<cls_rgw_obj_key>{}, commit);
+ if (result.second) { // inserting new entry
+ auto p = new object(key);
+ result.first = objects.insert_commit(*p, commit);
+ }
+ return result.first;
+}
+
+int simulator::try_start(const cls_rgw_obj_key& key, const std::string& tag)
+{
+ // choose randomly betwen create and delete
+ const auto type = static_cast<RGWModifyOp>(
+ ceph::util::generate_random_number<size_t, size_t>(CLS_RGW_OP_ADD,
+ CLS_RGW_OP_DEL));
+ auto op = operation{type, key, tag};
+
+ op.meta.category = RGWObjCategory::Main;
+ op.meta.size = op.meta.accounted_size =
+ ceph::util::generate_random_number(1, max_object_size);
+
+ if (type == CLS_RGW_OP_ADD && op.meta.size > max_part_size) {
+ // simulate multipart for uploads over the max_part_size threshold
+ op.upload_id = gen_rand_alphanumeric_upper(g_ceph_context, 12);
+
+ int r = init_multipart(op);
+ if (r != 0) {
+ derr << "> failed to prepare multipart upload key=" << key
+ << " upload=" << op.upload_id << " tag=" << tag
+ << " type=" << type << ": " << cpp_strerror(r) << dendl;
+ return r;
+ }
+
+ dout(1) << "> prepared multipart upload key=" << key
+ << " upload=" << op.upload_id << " tag=" << tag
+ << " type=" << type << " size=" << op.meta.size << dendl;
+ } else {
+ // prepare operation
+ int r = index_prepare(ioctx, oid, op.key, op.tag, op.type);
+ if (r != 0) {
+ derr << "> failed to prepare operation key=" << key
+ << " tag=" << tag << " type=" << type
+ << ": " << cpp_strerror(r) << dendl;
+ return r;
+ }
+
+ dout(1) << "> prepared operation key=" << key
+ << " tag=" << tag << " type=" << type
+ << " size=" << op.meta.size << dendl;
+ }
+ op.ver = last_version(ioctx);
+
+ ceph_assert(!pending.full());
+ pending.push_back(std::move(op));
+ return 0;
+}
+
+void simulator::start()
+{
+ // choose a random object key
+ const size_t index = ceph::util::generate_random_number(0, keys.size() - 1);
+ const auto& key = keys[index];
+ // generate a random tag
+ const auto tag = gen_rand_alphanumeric_upper(g_ceph_context, 12);
+
+ // retry until success. failures don't count towards max_operations
+ while (try_start(key, tag) != 0)
+ ;
+}
+
+void simulator::finish(const operation& op)
+{
+ if (op.type == CLS_RGW_OP_ADD && !op.upload_id.empty()) {
+ // multipart uploads either complete or abort based on part uploads
+ complete_multipart(op);
+ return;
+ }
+
+ // complete most operations, but finish some with cancel or dir suggest
+ constexpr int cancel_percent = 10;
+ constexpr int suggest_update_percent = 10;
+ constexpr int suggest_remove_percent = 10;
+
+ int result = ceph::util::generate_random_number(0, 99);
+ if (result < cancel_percent) {
+ complete(op, CLS_RGW_OP_CANCEL);
+ return;
+ }
+ result -= cancel_percent;
+ if (result < suggest_update_percent) {
+ suggest(op, CEPH_RGW_UPDATE);
+ return;
+ }
+ result -= suggest_update_percent;
+ if (result < suggest_remove_percent) {
+ suggest(op, CEPH_RGW_REMOVE);
+ return;
+ }
+ complete(op, op.type);
+}
+
+void simulator::complete(const operation& op, RGWModifyOp type)
+{
+ int r = index_complete(ioctx, oid, op.key, op.tag, type,
+ op.ver, op.meta, nullptr);
+ if (r != 0) {
+ derr << "< failed to complete operation key=" << op.key
+ << " tag=" << op.tag << " type=" << op.type
+ << " size=" << op.meta.size
+ << ": " << cpp_strerror(r) << dendl;
+ return;
+ }
+
+ if (type == CLS_RGW_OP_CANCEL) {
+ dout(1) << "< canceled operation key=" << op.key
+ << " tag=" << op.tag << " type=" << op.type
+ << " size=" << op.meta.size << dendl;
+ } else if (type == CLS_RGW_OP_ADD) {
+ auto obj = find_or_create(op.key);
+ if (obj->exists) {
+ unaccount_entry(stats, obj->meta);
+ }
+ obj->exists = true;
+ obj->meta = op.meta;
+ account_entry(stats, obj->meta);
+ dout(1) << "< completed write operation key=" << op.key
+ << " tag=" << op.tag << " type=" << type
+ << " size=" << op.meta.size << dendl;
+ } else {
+ ceph_assert(type == CLS_RGW_OP_DEL);
+ auto obj = objects.find(op.key, std::less<cls_rgw_obj_key>{});
+ if (obj != objects.end()) {
+ if (obj->exists) {
+ unaccount_entry(stats, obj->meta);
+ }
+ objects.erase_and_dispose(obj, std::default_delete<object>{});
+ }
+ dout(1) << "< completed delete operation key=" << op.key
+ << " tag=" << op.tag << " type=" << type << dendl;
+ }
+}
+
+void simulator::suggest(const operation& op, char suggestion)
+{
+ // read and decode the current dir entry
+ rgw_cls_bi_entry bi_entry;
+ int r = cls_rgw_bi_get(ioctx, oid, BIIndexType::Plain, op.key, &bi_entry);
+ if (r != 0) {
+ derr << "< no bi entry to suggest for operation key=" << op.key
+ << " tag=" << op.tag << " type=" << op.type
+ << " size=" << op.meta.size
+ << ": " << cpp_strerror(r) << dendl;
+ return;
+ }
+ ASSERT_EQ(bi_entry.type, BIIndexType::Plain);
+
+ rgw_bucket_dir_entry entry;
+ auto p = bi_entry.data.cbegin();
+ ASSERT_NO_THROW(decode(entry, p));
+
+ ASSERT_EQ(entry.key, op.key);
+
+ // clear pending info and write it back; this cancels those pending
+ // operations (we'll see EINVAL when we try to complete them), but dir
+ // suggest is ignored unless the pending_map is empty
+ entry.pending_map.clear();
+
+ bi_entry.data.clear();
+ encode(entry, bi_entry.data);
+
+ r = cls_rgw_bi_put(ioctx, oid, bi_entry);
+ ASSERT_EQ(0, r);
+
+ // now suggest changes for this entry
+ entry.ver = last_version(ioctx);
+ entry.exists = (suggestion == CEPH_RGW_UPDATE);
+ entry.meta = op.meta;
+
+ bufferlist update;
+ cls_rgw_encode_suggestion(suggestion, entry, update);
+
+ librados::ObjectWriteOperation write_op;
+ cls_rgw_suggest_changes(write_op, update);
+ r = ioctx.operate(oid, &write_op);
+ if (r != 0) {
+ derr << "< failed to suggest operation key=" << op.key
+ << " tag=" << op.tag << " type=" << op.type
+ << " size=" << op.meta.size
+ << ": " << cpp_strerror(r) << dendl;
+ return;
+ }
+
+ // update our cache accordingly
+ if (suggestion == CEPH_RGW_UPDATE) {
+ auto obj = find_or_create(op.key);
+ if (obj->exists) {
+ unaccount_entry(stats, obj->meta);
+ }
+ obj->exists = true;
+ obj->meta = op.meta;
+ account_entry(stats, obj->meta);
+ dout(1) << "< suggested update operation key=" << op.key
+ << " tag=" << op.tag << " type=" << op.type
+ << " size=" << op.meta.size << dendl;
+ } else {
+ ceph_assert(suggestion == CEPH_RGW_REMOVE);
+ auto obj = objects.find(op.key, std::less<cls_rgw_obj_key>{});
+ if (obj != objects.end()) {
+ if (obj->exists) {
+ unaccount_entry(stats, obj->meta);
+ }
+ objects.erase_and_dispose(obj, std::default_delete<object>{});
+ }
+ dout(1) << "< suggested remove operation key=" << op.key
+ << " tag=" << op.tag << " type=" << op.type << dendl;
+ }
+}
+
+int simulator::init_multipart(const operation& op)
+{
+ // create (not just prepare) the meta object
+ const auto meta_key = cls_rgw_obj_key{
+ fmt::format("_multipart_{}.2~{}.meta", op.key.name, op.upload_id)};
+ const std::string empty_tag; // empty tag enables complete without prepare
+ const rgw_bucket_entry_ver empty_ver;
+ rgw_bucket_dir_entry_meta meta_meta;
+ meta_meta.category = RGWObjCategory::MultiMeta;
+ int r = index_complete(ioctx, oid, meta_key, empty_tag, CLS_RGW_OP_ADD,
+ empty_ver, meta_meta, nullptr);
+ if (r != 0) {
+ derr << " < failed to create multipart meta key=" << meta_key
+ << ": " << cpp_strerror(r) << dendl;
+ return r;
+ } else {
+ // account for meta object
+ auto obj = find_or_create(meta_key);
+ if (obj->exists) {
+ unaccount_entry(stats, obj->meta);
+ }
+ obj->exists = true;
+ obj->meta = meta_meta;
+ account_entry(stats, obj->meta);
+ }
+
+ // prepare part uploads
+ std::list<cls_rgw_obj_key> remove_objs;
+ size_t part_id = 0;
+
+ size_t remaining = op.meta.size;
+ while (remaining > max_part_size) {
+ remaining -= max_part_size;
+ const auto part_size = std::min(remaining, max_part_size);
+ const auto part_key = cls_rgw_obj_key{
+ fmt::format("_multipart_{}.2~{}.{}", op.key.name, op.upload_id, part_id)};
+ part_id++;
+
+ r = index_prepare(ioctx, oid, part_key, op.tag, op.type);
+ if (r != 0) {
+ // if part prepare fails, remove the meta object and remove_objs
+ [[maybe_unused]] int ignored =
+ index_complete(ioctx, oid, meta_key, empty_tag, CLS_RGW_OP_DEL,
+ empty_ver, meta_meta, &remove_objs);
+ derr << " > failed to prepare part key=" << part_key
+ << " size=" << part_size << dendl;
+ return r; // return the error from prepare
+ }
+ dout(1) << " > prepared part key=" << part_key
+ << " size=" << part_size << dendl;
+ remove_objs.push_back(part_key);
+ }
+ return 0;
+}
+
+void simulator::complete_multipart(const operation& op)
+{
+ const std::string empty_tag; // empty tag enables complete without prepare
+ const rgw_bucket_entry_ver empty_ver;
+
+ // try to finish part uploads
+ size_t part_id = 0;
+ std::list<cls_rgw_obj_key> remove_objs;
+
+ RGWModifyOp type = op.type; // OP_ADD, or OP_CANCEL for abort
+
+ size_t remaining = op.meta.size;
+ while (remaining > max_part_size) {
+ remaining -= max_part_size;
+ const auto part_size = std::min(remaining, max_part_size);
+ const auto part_key = cls_rgw_obj_key{
+ fmt::format("_multipart_{}.2~{}.{}", op.key.name, op.upload_id, part_id)};
+ part_id++;
+
+ // cancel 10% of part uploads (and abort the multipart upload)
+ constexpr int cancel_percent = 10;
+ const int result = ceph::util::generate_random_number(0, 99);
+ if (result < cancel_percent) {
+ type = CLS_RGW_OP_CANCEL; // abort multipart
+ dout(1) << " < canceled part key=" << part_key
+ << " size=" << part_size << dendl;
+ } else {
+ rgw_bucket_dir_entry_meta meta;
+ meta.category = op.meta.category;
+ meta.size = meta.accounted_size = part_size;
+
+ int r = index_complete(ioctx, oid, part_key, op.tag, op.type,
+ empty_ver, meta, nullptr);
+ if (r != 0) {
+ derr << " < failed to complete part key=" << part_key
+ << " size=" << meta.size << ": " << cpp_strerror(r) << dendl;
+ type = CLS_RGW_OP_CANCEL; // abort multipart
+ } else {
+ dout(1) << " < completed part key=" << part_key
+ << " size=" << meta.size << dendl;
+ // account for successful part upload
+ auto obj = find_or_create(part_key);
+ if (obj->exists) {
+ unaccount_entry(stats, obj->meta);
+ }
+ obj->exists = true;
+ obj->meta = meta;
+ account_entry(stats, obj->meta);
+ }
+ }
+ remove_objs.push_back(part_key);
+ }
+
+ // delete the multipart meta object
+ const auto meta_key = cls_rgw_obj_key{
+ fmt::format("_multipart_{}.2~{}.meta", op.key.name, op.upload_id)};
+ rgw_bucket_dir_entry_meta meta_meta;
+ meta_meta.category = RGWObjCategory::MultiMeta;
+
+ int r = index_complete(ioctx, oid, meta_key, empty_tag, CLS_RGW_OP_DEL,
+ empty_ver, meta_meta, nullptr);
+ if (r != 0) {
+ derr << " < failed to remove multipart meta key=" << meta_key
+ << ": " << cpp_strerror(r) << dendl;
+ } else {
+ // unaccount for meta object
+ auto obj = objects.find(meta_key, std::less<cls_rgw_obj_key>{});
+ if (obj != objects.end()) {
+ if (obj->exists) {
+ unaccount_entry(stats, obj->meta);
+ }
+ objects.erase_and_dispose(obj, std::default_delete<object>{});
+ }
+ }
+
+ // create or cancel the head object
+ r = index_complete(ioctx, oid, op.key, empty_tag, type,
+ empty_ver, op.meta, &remove_objs);
+ if (r != 0) {
+ derr << "< failed to complete multipart upload key=" << op.key
+ << " upload=" << op.upload_id << " tag=" << op.tag
+ << " type=" << type << " size=" << op.meta.size
+ << ": " << cpp_strerror(r) << dendl;
+ return;
+ }
+
+ if (type == CLS_RGW_OP_ADD) {
+ dout(1) << "< completed multipart upload key=" << op.key
+ << " upload=" << op.upload_id << " tag=" << op.tag
+ << " type=" << op.type << " size=" << op.meta.size << dendl;
+
+ // account for head stats
+ auto obj = find_or_create(op.key);
+ if (obj->exists) {
+ unaccount_entry(stats, obj->meta);
+ }
+ obj->exists = true;
+ obj->meta = op.meta;
+ account_entry(stats, obj->meta);
+ } else {
+ dout(1) << "< canceled multipart upload key=" << op.key
+ << " upload=" << op.upload_id << " tag=" << op.tag
+ << " type=" << op.type << " size=" << op.meta.size << dendl;
+ }
+
+ // unaccount for remove_objs
+ for (const auto& part_key : remove_objs) {
+ auto obj = objects.find(part_key, std::less<cls_rgw_obj_key>{});
+ if (obj != objects.end()) {
+ if (obj->exists) {
+ unaccount_entry(stats, obj->meta);
+ }
+ objects.erase_and_dispose(obj, std::default_delete<object>{});
+ }
+ }
+}
+
+TEST(cls_rgw_stats, simulate)
+{
+ const char* bucket_oid = __func__;
+ auto sim = simulator{RadosEnv::ioctx, bucket_oid};
+ sim.run();
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-10-30 05:57:45 +0000